From 91472492ce42e580237ffb781a9a33613b8baa19 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Mon, 14 Aug 2023 13:08:53 +0200 Subject: [PATCH 001/379] Build operators regardless of model --- src/operators/operators.jl | 8 ++++++-- src/setup.jl | 4 ++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/src/operators/operators.jl b/src/operators/operators.jl index 97a7610be..887646356 100644 --- a/src/operators/operators.jl +++ b/src/operators/operators.jl @@ -3,7 +3,7 @@ Build operators. """ -function Operators(grid, boundary_conditions, viscosity_model) +function Operators(grid, boundary_conditions) # Averaging operators op_ave = operator_averaging(grid, boundary_conditions) @@ -19,8 +19,12 @@ function Operators(grid, boundary_conditions, viscosity_model) # Regularization modelling - this changes the convective term op_reg = operator_regularization(grid, op_con) + # # Classical turbulence modelling via the diffusive term + # op_vis = operator_viscosity(viscosity_model, grid, boundary_conditions) + # Classical turbulence modelling via the diffusive term - op_vis = operator_viscosity(viscosity_model, grid, boundary_conditions) + # Note: We build turbulent diffusion operator even for laminar model + op_vis = operator_turbulent_diffusion(grid, boundary_conditions) # Post-processing op_pos = operator_postprocessing(grid, boundary_conditions) diff --git a/src/setup.jl b/src/setup.jl index d9c922fa4..51ef41936 100644 --- a/src/setup.jl +++ b/src/setup.jl @@ -41,7 +41,7 @@ function Setup( BoundaryConditions(u_bc, v_bc; dudt_bc, dvdt_bc, bc_type, T = eltype(x)) grid = Grid(x, y; boundary_conditions, order4) force = SteadyBodyForce(bodyforce_u, bodyforce_v, grid) - operators = Operators(grid, boundary_conditions, viscosity_model) + operators = Operators(grid, boundary_conditions) (; grid, boundary_conditions, viscosity_model, convection_model, force, operators) end @@ -137,6 +137,6 @@ function Setup( ) grid = Grid(x, y, z; boundary_conditions, order4) force = SteadyBodyForce(bodyforce_u, bodyforce_v, bodyforce_w, grid) - operators = Operators(grid, boundary_conditions, viscosity_model) + operators = Operators(grid, boundary_conditions) (; grid, boundary_conditions, viscosity_model, convection_model, force, operators) end From 7fd81658a2d602be2968ad24a9accd859d59b3e2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Mon, 14 Aug 2023 13:09:31 +0200 Subject: [PATCH 002/379] Raise error --- src/operators/operator_filter.jl | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/operators/operator_filter.jl b/src/operators/operator_filter.jl index 43f6d6dbf..5376b3f0a 100644 --- a/src/operators/operator_filter.jl +++ b/src/operators/operator_filter.jl @@ -116,4 +116,6 @@ function operator_filter(::Dimension{2}, grid, boundary_conditions, s) end # 3D version -function operator_filter(::Dimension{3}, grid, boundary_conditions) end +function operator_filter(::Dimension{3}, grid, boundary_conditions) + error("Not implemented") +end From 8cfaf0fb696e11327bd33e24085f705929ee8b0c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Wed, 16 Aug 2023 14:54:40 +0200 Subject: [PATCH 003/379] feat(filter): Add type --- src/operators/operator_filter.jl | 30 +++++++++++++++++------------- 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/src/operators/operator_filter.jl b/src/operators/operator_filter.jl index 5376b3f0a..a048f13a5 100644 --- a/src/operators/operator_filter.jl +++ b/src/operators/operator_filter.jl @@ -1,9 +1,10 @@ """ - create_top_hat_p(N, M) + create_top_hat_p([T = Float64], N, M) `N` fine points and `M` coarse points in each dimension. """ -function create_top_hat_p(N, M) +create_top_hat_p(N, M) = create_top_hat_p(Float64, N, M) +function create_top_hat_p(T, N, M) s = N ÷ M @assert s * M == N @@ -17,17 +18,18 @@ function create_top_hat_p(N, M) ijkl = @. s * (i - 1) + k + s * N * (j - 1) + N * (l - 1) - z = fill(1 / s^2, N^2) + z = fill(T(1) / s^2, N^2) sparse(ij[:], ijkl[:], z) end """ - create_top_hat_u(N, M) + create_top_hat_u([T = Float64], N, M) `N` fine points and `M` coarse points in each dimension. """ -function create_top_hat_u(N, M) +create_top_hat_u(N, M) = create_top_hat_u(Float64, N, M) +function create_top_hat_u(T, N, M) s = N ÷ M @assert s * M == N @@ -41,17 +43,18 @@ function create_top_hat_u(N, M) ijkl = @. s * (i - 1) + k + s * N * (j - 1) + N * (l - 1) - z = fill(1 / s, N * M) + z = fill(T(1) / s, N * M) sparse(ij[:], ijkl[:], z, M^2, N^2) end """ - create_top_hat_v(N, M) +create_top_hat_v([T = Float64], N, M) `N` fine points and `M` coarse points in each dimension. """ -function create_top_hat_v(N, M) +create_top_hat_v(N, M) = create_top_hat_v(Float64, N, M) +function create_top_hat_v(T, N, M) s = N ÷ M @assert s * M == N @@ -65,7 +68,7 @@ function create_top_hat_v(N, M) ijkl = @. s * (i - 1) + k + s * N * (j - 1) + N * (l - 1) - z = fill(1 / s, N * M) + z = fill(T(1) / s, N * M) sparse(ij[:], ijkl[:], z, M^2, N^2) end @@ -91,7 +94,8 @@ operator_filter(grid, boundary_conditions, s) = # 2D version function operator_filter(::Dimension{2}, grid, boundary_conditions, s) - (; Nx, Ny, hx, hy) = grid + (; Nx, Ny, hx, hy, x) = grid + T = eltype(x) N = Nx M = N ÷ s @@ -107,9 +111,9 @@ function operator_filter(::Dimension{2}, grid, boundary_conditions, s) ) "Filter assumes periodic boundary conditions" @assert Nx == Ny - Kp = create_top_hat_p(N, M) - Ku = create_top_hat_u(N, M) - Kv = create_top_hat_v(N, M) + Kp = create_top_hat_p(T, N, M) + Ku = create_top_hat_u(T, N, M) + Kv = create_top_hat_v(T, N, M) KV = blockdiag(Ku, Kv) (; KV, Kp) From 16432fed4812b49e167724fca8a249e1701da429 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Wed, 16 Aug 2023 14:55:34 +0200 Subject: [PATCH 004/379] feat(plots): Add image type --- src/processors/real_time_plot.jl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/processors/real_time_plot.jl b/src/processors/real_time_plot.jl index 563d0566d..ab68adf3f 100644 --- a/src/processors/real_time_plot.jl +++ b/src/processors/real_time_plot.jl @@ -94,7 +94,7 @@ function field_plot( lims = @lift begin f = $field - if type == heatmap + if type ∈ (heatmap, image) lims = get_lims(f) elseif type ∈ (contour, contourf) if ≈(extrema(f)...; rtol = 1e-10) @@ -113,8 +113,8 @@ function field_plot( fig = Figure() - if type == heatmap - ax, hm = heatmap(fig[1, 1], xf, yf, field; colorrange = lims) + if type ∈ (heatmap, image) + ax, hm = type(fig[1, 1], xf, yf, field; colorrange = lims) elseif type ∈ (contour, contourf) ax, hm = type( fig[1, 1], From 755df2705aad1e23e6ab22345f97eaddfb47183c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Wed, 16 Aug 2023 14:56:11 +0200 Subject: [PATCH 005/379] feat(plots): Add draft for automatic wait time --- src/processors/real_time_plot.jl | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/src/processors/real_time_plot.jl b/src/processors/real_time_plot.jl index ab68adf3f..cd4ff6c61 100644 --- a/src/processors/real_time_plot.jl +++ b/src/processors/real_time_plot.jl @@ -285,7 +285,7 @@ function energy_spectrum_plot(::Dimension{2}, setup, state; displayfig = true) espec end -function energy_spectrum_plot(::Dimension{3}, setup, state, displayfig = true) +function energy_spectrum_plot(::Dimension{3}, setup, state, displayfig = true, checktime = 0.0001) (; xpp) = setup.grid Kx, Ky, Kz = size(xpp) .÷ 2 kx = 1:(Kx-1) @@ -307,5 +307,22 @@ function energy_spectrum_plot(::Dimension{3}, setup, state, displayfig = true) lines!(ax, krange, 1e6 * krange .^ (-5 / 3); label = "\$k^{-5/3}\$", color = :red) axislegend(ax) displayfig && display(espec) - espec + return espec + + # Make sure the figure is fully rendered before allowing code to continue + if displayfig + render = display(espec) + done_rendering = Ref(false) + on(render.render_tic) do _ + done_rendering[] = true + end + on(state) do s + # State is updated, block code execution until GLMakie has rendered + # figure update + done_rendering[] = false + while !done_rendering[] + sleep(checktime) + end + end + end end From ed0a31d04d94219d36d4555a9f7e55f7706f21d0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Wed, 16 Aug 2023 14:58:33 +0200 Subject: [PATCH 006/379] Add script to hide long sparse arrays --- scratch/censor_sparse.jl | 46 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100644 scratch/censor_sparse.jl diff --git a/scratch/censor_sparse.jl b/scratch/censor_sparse.jl new file mode 100644 index 000000000..f97a2f8f4 --- /dev/null +++ b/scratch/censor_sparse.jl @@ -0,0 +1,46 @@ +using SparseArrays +using LinearAlgebra + +# Hide long sparse arrays +function Base.show(io::IO, _S::SparseArrays.AbstractSparseMatrixCSCInclAdjointAndTranspose) + SparseArrays._checkbuffers(_S) + # can't use `findnz`, because that expects all values not to be #undef + S = _S isa Adjoint || _S isa Transpose ? _S.parent : _S + I = rowvals(S) + K = nonzeros(S) + m, n = size(S) + if _S isa Adjoint + print(io, "adjoint(") + elseif _S isa Transpose + print(io, "transpose(") + end + nn = nnz(S) + if nn > 20 + print(io, "sparse()") + if _S isa Adjoint || _S isa Transpose + print(io, ")") + end + return + end + print(io, "sparse(", I, ", ") + if length(I) == 0 + print(io, eltype(SparseArrays.getcolptr(S)), "[]") + else + print(io, "[") + il = nnz(S) - 1 + for col = 1:size(S, 2), + k = SparseArrays.getcolptr(S)[col]:(SparseArrays.getcolptr(S)[col+1]-1) + + print(io, col) + if il > 0 + print(io, ", ") + il -= 1 + end + end + print(io, "]") + end + print(io, ", ", K, ", ", m, ", ", n, ")") + if _S isa Adjoint || _S isa Transpose + print(io, ")") + end +end From 4558fde56b63ee5c1c95d56f072fc45cfef39fa0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Wed, 16 Aug 2023 15:00:08 +0200 Subject: [PATCH 007/379] Add script --- scratch/filtered.jl | 150 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 150 insertions(+) create mode 100644 scratch/filtered.jl diff --git a/scratch/filtered.jl b/scratch/filtered.jl new file mode 100644 index 000000000..bc14e8693 --- /dev/null +++ b/scratch/filtered.jl @@ -0,0 +1,150 @@ +# Little LSP hack to get function signatures, go #src +# to definition etc. #src +if isdefined(@__MODULE__, :LanguageServer) #src + include("../src/IncompressibleNavierStokes.jl") #src + using .IncompressibleNavierStokes #src +end #src + +using GLMakie +using IncompressibleNavierStokes +using LinearAlgebra + +# Floating point precision +T = Float32 + +# To use CPU: Do not move any arrays +device = identity + +# # To use GPU, use `cu` to move arrays to the GPU. +# # Note: `cu` converts to Float32 +# using CUDA +# device = cu + +# Viscosity model +Re = T(2_000) +laminar = LaminarModel(; Re) +smagorinsky = SmagorinskyModel(; Re, C_s = T(0.173)) + +# A 2D grid is a Cartesian product of two vectors +s = 2 +n_coarse = 256 +n = s * n_coarse + +lims = T(0), T(1) +x = LinRange(lims..., n + 1) +y = LinRange(lims..., n + 1) +plot_grid(x, y) + +x_coarse = x[1:s:end] +y_coarse = y[1:s:end] +plot_grid(x_coarse, y_coarse) + +# Build setup and assemble operators +setup = Setup(x, y; viscosity_model = laminar); +devsetup = device(setup); +setup_coarse = Setup(x_coarse, y_coarse; viscosity_model = laminar); + +# Filter +(; KV, Kp) = operator_filter(setup.grid, setup.boundary_conditions, s); + +# Since the grid is uniform and identical for x and y, we may use a specialized +# spectral pressure solver +pressure_solver = SpectralPressureSolver(setup); +pressure_solver_coarse = SpectralPressureSolver(setup_coarse); + +# Initial conditions +V₀, p₀ = random_field(setup; A = T(10_000_000), σ = T(30), s = 5, pressure_solver); + +filter_saver(setup, KV, Kp; nupdate = 1, bc_vectors = get_bc_vectors(setup, T(0))) = + processor( + function (step_observer) + T = eltype(setup.grid.x) + _V = fill(zeros(T, 0), 0) + _F = fill(zeros(T, 0), 0) + _FG = fill(zeros(T, 0), 0) + _p = fill(zeros(T, 0), 0) + _t = fill(zero(T), 0) + @lift begin + (; V, p, t) = $step_observer + F, = momentum(V, V, p, t, setup; bc_vectors, nopressure = true) + FG, = momentum(V, V, p, t, setup; bc_vectors, nopressure = false) + push!(_V, KV * Array(V)) + push!(_F, KV * Array(F)) + push!(_FG, KV * Array(FG)) + push!(_p, Kp * Array(p)) + push!(_t, t) + end + (; V = _V, F = _F, FG = _FG, p = _p, t = _t) + end; + nupdate, + ) + +# Iteration processors +processors = ( + filter_saver(devsetup, KV, Kp; bc_vectors = device(get_bc_vectors(setup, T(0)))), + field_plotter(devsetup; type = image, nupdate = 1), + # energy_history_plotter(devsetup; nupdate = 20, displayfig = false), + # energy_spectrum_plotter(devsetup; nupdate = 20, displayfig = false), + # animator(devsetup; nupdate = 16), + ## vtk_writer(setup; nupdate = 10, dir = "output/$name", filename = "solution"), + ## field_saver(setup; nupdate = 10), + step_logger(; nupdate = 10), +); + +processors_coarse = ( + field_plotter(device(setup_coarse); type = image, nupdate = 1), + step_logger(; nupdate = 10), +); + +# Time interval +t_start, t_end = tlims = T(0), T(0.1) + +# Solve unsteady problem +V, p, outputs = solve_unsteady( + setup, + V₀, + p₀, + # V, p, + tlims; + Δt = T(2e-4), + processors, + pressure_solver, + inplace = true, + device, +); + +# V₀, p₀ = V, p + +Vbar = KV * V +pbar = Kp * p + +Vbar_nomodel, pbar_nomodel, outputs_lam = solve_unsteady( + setup_coarse, + KV * V₀, + Kp * p₀, + tlims; + Δt = T(2e-4), + processors = processors_coarse, + pressure_solver = pressure_solver_coarse, + inplace = true, + device, +); + +Vbar_smag, pbar_smag, outputs_smag = solve_unsteady( + (; setup_coarse..., viscosity_model = smagorinsky), + KV * V₀, + Kp * p₀, + tlims; + Δt = T(2e-4), + processors = processors_coarse, + pressure_solver = pressure_solver_coarse, + inplace = true, + device, +); + +norm(Vbar_nomodel - Vbar) / norm(Vbar) +norm(Vbar_smag - Vbar) / norm(Vbar) + +# Filtered quantities +filtered = outputs[1] +filtered.V[end] From 54764f7c4e83c402d7babbb5091eda79258e9487 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Wed, 16 Aug 2023 15:00:30 +0200 Subject: [PATCH 008/379] Update ignore --- .gitignore | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index b81b1fa0b..41f11d726 100644 --- a/.gitignore +++ b/.gitignore @@ -5,8 +5,7 @@ /docs/build/ /docs/src/generated/ *output/ -*notebooks/ -examples/test.jl +# scratch/* # Vim swap *.swp From 9ef79b8e1087c0016481da0682cf7c5e8c99a542 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Wed, 16 Aug 2023 16:37:23 +0200 Subject: [PATCH 009/379] Add closures --- Project.toml | 2 ++ src/IncompressibleNavierStokes.jl | 5 +++++ src/closures/cnn.jl | 4 ++++ 3 files changed, 11 insertions(+) create mode 100644 src/closures/cnn.jl diff --git a/Project.toml b/Project.toml index b7ea46fa0..5d22038a4 100644 --- a/Project.toml +++ b/Project.toml @@ -8,8 +8,10 @@ Adapt = "79e6a3ab-5dfb-504d-930d-738a2a938a0e" FFTW = "7a1cc6ca-52ef-59f5-83cd-3a7055c09341" IterativeSolvers = "42fd0dbc-a981-5370-80f2-aaf504508153" LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" +Lux = "b2108857-7c20-44ae-9111-449ecde12c47" Makie = "ee78f7c6-11fb-53f2-987a-cfe4a2b5a57a" Printf = "de0858da-6303-5e67-8744-51eddeeeb8d7" +Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" SparseArrays = "2f01184e-e22b-5df5-ae63-d93ebab69eaf" Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2" WriteVTK = "64499a7a-5c06-52f2-abe2-ccb03c286192" diff --git a/src/IncompressibleNavierStokes.jl b/src/IncompressibleNavierStokes.jl index 71ce39e38..fbbcdf12a 100644 --- a/src/IncompressibleNavierStokes.jl +++ b/src/IncompressibleNavierStokes.jl @@ -9,7 +9,9 @@ using Adapt using FFTW using IterativeSolvers using LinearAlgebra +using Lux using Printf +using Random using SparseArrays using Statistics using WriteVTK: CollectionFile, paraview_collection, vtk_grid, vtk_save @@ -120,6 +122,9 @@ include("postprocess/plot_vorticity.jl") include("postprocess/plot_streamfunction.jl") include("postprocess/save_vtk.jl") +# Closure models +include("closures/cnn.jl") + # Force export SteadyBodyForce diff --git a/src/closures/cnn.jl b/src/closures/cnn.jl new file mode 100644 index 000000000..929f840da --- /dev/null +++ b/src/closures/cnn.jl @@ -0,0 +1,4 @@ +function cnn(setup; channel_augmenter = identity, rng = Random.default_rng()) + NN = Chain( + ) +end From 7ecc411f07a1707313e9c7dfd24bbf75e164bae8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Fri, 18 Aug 2023 17:37:09 +0200 Subject: [PATCH 010/379] feat: Add closure function --- src/momentum/momentum.jl | 28 ++++++++++++++++++++++++---- src/setup.jl | 24 ++++++++++++++++++++++-- 2 files changed, 46 insertions(+), 6 deletions(-) diff --git a/src/momentum/momentum.jl b/src/momentum/momentum.jl index eef27c268..f982edda5 100644 --- a/src/momentum/momentum.jl +++ b/src/momentum/momentum.jl @@ -32,7 +32,14 @@ function momentum( nopressure = false, newton_factor = false, ) - (; viscosity_model, convection_model, force, boundary_conditions, operators) = setup + (; + viscosity_model, + convection_model, + force, + closure_model, + boundary_conditions, + operators, + ) = setup (; G) = operators # Unsteady BC @@ -51,8 +58,11 @@ function momentum( # Body force b = force + # Closure model + cm = closure_model(V) + # Residual in Finite Volume form, including the pressure contribution - F = @. -c + d + b + F = @. -c + d + b + cm # Nopressure = false is the most common situation, in which we return the entire # right-hand side vector @@ -102,7 +112,14 @@ function momentum!( nopressure = false, newton_factor = false, ) - (; viscosity_model, convection_model, force, boundary_conditions, operators) = setup + (; + viscosity_model, + convection_model, + force, + closure_model, + boundary_conditions, + operators, + ) = setup (; G) = setup.operators # Unsteady BC @@ -134,8 +151,11 @@ function momentum!( # Body force b = force + # Closure model + cm = closure_model(V) + # Residual in Finite Volume form, including the pressure contribution - @. F = -c + d + b + @. F = -c + d + b + cm # Nopressure = false is the most common situation, in which we return the entire # right-hand side vector diff --git a/src/setup.jl b/src/setup.jl index 51ef41936..f32708f44 100644 --- a/src/setup.jl +++ b/src/setup.jl @@ -14,6 +14,7 @@ order4 = false, bodyforce_u = (x, y) -> 0, bodyforce_v = (x, y) -> 0, + closure_model = V -> zero(V), ) Create 2D setup. @@ -36,13 +37,22 @@ function Setup( bodyforce_u = (x, y) -> 0, bodyforce_v = (x, y) -> 0, steady_force = true, + closure_model = V -> zero(V), ) boundary_conditions = BoundaryConditions(u_bc, v_bc; dudt_bc, dvdt_bc, bc_type, T = eltype(x)) grid = Grid(x, y; boundary_conditions, order4) force = SteadyBodyForce(bodyforce_u, bodyforce_v, grid) operators = Operators(grid, boundary_conditions) - (; grid, boundary_conditions, viscosity_model, convection_model, force, operators) + (; + grid, + boundary_conditions, + viscosity_model, + convection_model, + force, + closure_model, + operators, + ) end """ @@ -82,6 +92,7 @@ end bodyforce_u = (x, y, z) -> 0, bodyforce_v = (x, y, z) -> 0, bodyforce_w = (x, y, z) -> 0, + closure_model = V -> zero(V), ) Create 3D setup. @@ -124,6 +135,7 @@ function Setup( bodyforce_u = (x, y, z) -> 0, bodyforce_v = (x, y, z) -> 0, bodyforce_w = (x, y, z) -> 0, + closure_model = V -> zero(V), ) boundary_conditions = BoundaryConditions( u_bc, @@ -138,5 +150,13 @@ function Setup( grid = Grid(x, y, z; boundary_conditions, order4) force = SteadyBodyForce(bodyforce_u, bodyforce_v, bodyforce_w, grid) operators = Operators(grid, boundary_conditions) - (; grid, boundary_conditions, viscosity_model, convection_model, force, operators) + (; + grid, + boundary_conditions, + viscosity_model, + convection_model, + force, + closure_model, + operators, + ) end From 8f5a37e45917c365dc3648f7905b7f80c58ac8a3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Fri, 18 Aug 2023 17:38:35 +0200 Subject: [PATCH 011/379] feat: Add CNN closure --- src/IncompressibleNavierStokes.jl | 4 +++ src/closures/cnn.jl | 60 ++++++++++++++++++++++++++++++- 2 files changed, 63 insertions(+), 1 deletion(-) diff --git a/src/IncompressibleNavierStokes.jl b/src/IncompressibleNavierStokes.jl index fbbcdf12a..f256f9935 100644 --- a/src/IncompressibleNavierStokes.jl +++ b/src/IncompressibleNavierStokes.jl @@ -6,6 +6,7 @@ Energy-conserving solvers for the incompressible Navier-Stokes equations. module IncompressibleNavierStokes using Adapt +using ComponentArrays using FFTW using IterativeSolvers using LinearAlgebra @@ -159,6 +160,9 @@ export plot_force, plot_grid, plot_pressure, plot_streamfunction, plot_velocity, plot_vorticity, save_vtk export plotmat +# Closure models +export cnn + # ODE methods export AdamsBashforthCrankNicolsonMethod, OneLegMethod diff --git a/src/closures/cnn.jl b/src/closures/cnn.jl index 929f840da..59a09754d 100644 --- a/src/closures/cnn.jl +++ b/src/closures/cnn.jl @@ -1,4 +1,62 @@ -function cnn(setup; channel_augmenter = identity, rng = Random.default_rng()) +""" + cnn(setup, r, c, σ, b; kwargs...) + +Create CNN closure model. Return a tuple `(closure, Θ)` where `Θ` are the initial +parameters and `closure(V, Θ)` predicts the commutator error. +""" +cnn(setup, r, c, σ, b; kwargs...) = cnn(setup.grid.dimension, setup, r, c, σ, b; kwargs...) + +function cnn( + ::Dimension{2}, + setup, + r, + c, + σ, + b; + channel_augmenter = identity, + rng = Random.default_rng(), +) + (; grid) = setup + (; Nx, Ny, x) = grid + + # For now + T = eltype(x) + @assert T == Float32 + + # Make sure there are two velocity fields in input and output + @assert c[1] == 2 + @assert c[end] == 2 + + # Create convolutional closure model NN = Chain( + # Unflatten and separate u and v velocities + V -> reshape(V, Nx, Ny, 2, :), + + # Add padding so that output has same shape as commutator error + u -> pad_circular(u, sum(r)), + + # Some convolutional layers + ( + Conv((2r[i] + 1, 2r[i] + 1), c[i] => c[i+1], σ[i]; use_bias = b[i]) for + i ∈ eachindex(r) + )..., + + # Flatten to vector + u -> reshape(u, :, size(u, 4)), ) + + # Create parameter vector (empty state) + params, state = Lux.setup(rng, NN) + θ = ComponentArray(params) + + """ + closure(V, θ) + + Compute closure term for given parameters `θ`. + """ + function closure end + closure(V, θ) = first(NN(V, θ, state)) + closure(V::AbstractVector, θ) = reshape(closure(reshape(V, :, 1), θ), :) + + closure, θ end From 87a2e8b0264a80dccdaf692b53aa13ee545a1c02 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Fri, 18 Aug 2023 17:39:24 +0200 Subject: [PATCH 012/379] Add training script --- scratch/train_model.jl | 220 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 220 insertions(+) create mode 100644 scratch/train_model.jl diff --git a/scratch/train_model.jl b/scratch/train_model.jl new file mode 100644 index 000000000..859a70453 --- /dev/null +++ b/scratch/train_model.jl @@ -0,0 +1,220 @@ +# Little LSP hack to get function signatures, go #src +# to definition etc. #src +if isdefined(@__MODULE__, :LanguageServer) #src + include("../src/IncompressibleNavierStokes.jl") #src + using .IncompressibleNavierStokes #src +end #src + +using GLMakie +using IncompressibleNavierStokes +using JLD2 +using LinearAlgebra +using Lux +using Optimisers +using Random +using Zygote + +# Floating point precision +T = Float32 + +# To use CPU: Do not move any arrays +device = identity + +# To use GPU, use `cu` to move arrays to the GPU. +# Note: `cu` converts to Float32 +using CUDA +using LuxCUDA +device = cu + +# Viscosity model +Re = T(2_000) +viscosity_model = LaminarModel(; Re) + +# A 2D grid is a Cartesian product of two vectors +s = 4 +n_les = 128 +n_dns = s * n_les + +lims = T(0), T(1) +x_dns = LinRange(lims..., n_dns + 1) +y_dns = LinRange(lims..., n_dns + 1) +x_les = x_dns[1:s:end] +y_les = y_dns[1:s:end] + +# Build setup and assemble operators +dns = Setup(x_dns, y_dns; viscosity_model); +les = Setup(x_les, y_les; viscosity_model); + +# Filter +(; KV, Kp) = operator_filter(dns.grid, dns.boundary_conditions, s); + +# Since the grid is uniform and identical for x and y, we may use a specialized +# spectral pressure solver +pressure_solver = SpectralPressureSolver(dns); +pressure_solver_les = SpectralPressureSolver(les); + +filter_saver(setup, KV, Kp, Ωbar; nupdate = 1, bc_vectors = get_bc_vectors(setup, T(0))) = + processor( + function (step_observer) + (; Ω) = setup.grid + KVmom = Ωbar * KV * Diagonal(1 ./ Ω) + T = eltype(setup.grid.x) + _V = fill(zeros(T, 0), 0) + _F = fill(zeros(T, 0), 0) + _FG = fill(zeros(T, 0), 0) + _p = fill(zeros(T, 0), 0) + _t = fill(zero(T), 0) + @lift begin + (; V, p, t) = $step_observer + F, = momentum(V, V, p, t, setup; bc_vectors, nopressure = true) + FG, = momentum(V, V, p, t, setup; bc_vectors, nopressure = false) + push!(_V, KV * Array(V)) + push!(_F, KVmom * Array(F)) + push!(_FG, KVmom * Array(FG)) + push!(_p, Kp * Array(p)) + push!(_t, t) + end + (; V = _V, F = _F, FG = _FG, p = _p, t = _t) + end; + nupdate, + ) + +# Time interval, including burn-in time +t_start, t_burn, t_end = T(0.0), T(0.1), T(0.2) + +# Number of time steps +Δt = T(2e-4) +n_t = round(Int, (t_end - t_burn) / Δt) + +# Number of random initial conditions +n_ic = 10 + +# Filtered quantities to store +filtered = (; + V = zeros(T, n_les * n_les * 2, n_t + 1, n_ic), + F = zeros(T, n_les * n_les * 2, n_t + 1, n_ic), + FG = zeros(T, n_les * n_les * 2, n_t + 1, n_ic), + p = zeros(T, n_les * n_les, n_t + 1, n_ic), +) + +Base.summarysize(filtered) / 1e6 + +# Iteration processors +processors = ( + filter_saver( + device(dns), + KV, + Kp, + les.grid.Ω; + bc_vectors = device(get_bc_vectors(dns, t_start)), + ), + step_logger(; nupdate = 10), +); + +for i_ic = 1:n_ic + @info "Generating data for IC $i_ic of $n_ic" + + # Initial conditions + V₀, p₀ = random_field(dns; A = T(10_000_000), σ = T(30), s = 5, pressure_solver) + + # Solve burn-in DNS + @info "Burn-in for IC $i_ic of $n_ic" + V, p, outputs = solve_unsteady( + dns, + V₀, + p₀, + (t_start, t_burn); + Δt = T(2e-4), + processors = (step_logger(; nupdate = 10),), + pressure_solver, + inplace = true, + device, + ) + + # Solve DNS and store filtered quantities + @info "Solving DNS for IC $i_ic of $n_ic" + V, p, outputs = solve_unsteady( + dns, + V, + p, + (t_burn, t_end); + Δt = T(2e-4), + processors, + pressure_solver, + inplace = true, + device, + ) + f = outputs[1] + + # Store result for current IC + filtered.V[:, :, i_ic] = stack(f.V) + filtered.F[:, :, i_ic] = stack(f.F) + filtered.FG[:, :, i_ic] = stack(f.FG) + filtered.p[:, :, i_ic] = stack(f.p) +end + +# jldsave("output/filtered/filtered.jld2"; filtered) +filtered = load("output/filtered/filtered.jld2", "filtered") + +size(filtered.V) + +plot_vorticity(les, filtered.V[:, end, 1], t_burn) + +# Compute commutator errors +bc_vectors = get_bc_vectors(les, t_burn) +commutator_error = zero(filtered.F) +pbar = filtered.p[:, 1, 1] +for i_t = 1:n_t, i_ic = 1:n_ic + @info "Computing commutator error for time $i_t of $n_t, IC $i_ic of $n_ic" + V = filtered.V[:, i_t, i_ic] + F = filtered.F[:, i_t, i_ic] + Fbar, = momentum(V, V, pbar, t_burn, les; bc_vectors, nopressure = true) + commutator_error[:, i_t, i_ic] .= F .- Fbar +end + +norm(commutator_error[:, 1, 1]) / norm(filtered.F[:, 1, 1]) + +closure, θ = cnn( + les, + [5, 5, 5], + [2, 5, 5, 2], + [tanh, tanh, identity], + [true, true, false]; +) + +loss(x, y, θ) = sum(abs2, closure(x, θ) - y) / length(y) + +loss(filtered.V[:, 1, 1], commutator_error[:, 1, 1], θ) + +function create_loss(x, y; nuse = size(x, 2)) + x = reshape(x, size(x, 1), :) + y = reshape(y, size(y, 1), :) + nsample = size(x, 2) + d = ndims(x) + function randloss(θ) + i = Zygote.@ignore sort(shuffle(1:nsample)[1:nuse]) + xuse = Zygote.@ignore device(selectdim(x, d, i)) + yuse = Zygote.@ignore device(selectdim(y, d, i)) + loss(xuse, yuse, θ) + end +end + +randloss = create_loss(filtered.V, commutator_error; nuse = 100) + +devθ₀ = device(θ) + +randloss(devθ) + +first(gradient(randloss, devθ)) + +V_test = device(reshape(filtered.V[:, 1:20, 1:2], :, 40)) +c_test = device(reshape(commutator_error[:, 1:20, 1:2], :, 40)) + +devθ = devθ₀ +opt = Optimisers.setup(Adam(0.005f0), devθ) +for i = 1:200 + g = first(gradient(randloss, devθ)) + opt, devθ = Optimisers.update(opt, devθ, g) + e_test = norm(closure(V_test, devθ) - c_test) / norm(c_test) + @info "Iteration $i\trelative test error: $e_test" +end From a3372b3a706a3ea3207a22b8628799234c7158be Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Fri, 18 Aug 2023 17:40:27 +0200 Subject: [PATCH 013/379] feat(plot): add `image` type (faster than heatmap) --- src/processors/real_time_plot.jl | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/processors/real_time_plot.jl b/src/processors/real_time_plot.jl index cd4ff6c61..97cf8660f 100644 --- a/src/processors/real_time_plot.jl +++ b/src/processors/real_time_plot.jl @@ -114,7 +114,9 @@ function field_plot( fig = Figure() if type ∈ (heatmap, image) - ax, hm = type(fig[1, 1], xf, yf, field; colorrange = lims) + ax, hm = type(fig[1, 1], xf, yf, field; + colormap = :viridis, + colorrange = lims) elseif type ∈ (contour, contourf) ax, hm = type( fig[1, 1], From be7e1d6bba2d1d89a6d14f3e8930c692a47b3375 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Wed, 23 Aug 2023 14:46:16 +0200 Subject: [PATCH 014/379] feat(cnn): idea for cross term channels --- src/closures/cnn.jl | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/closures/cnn.jl b/src/closures/cnn.jl index 59a09754d..e16244208 100644 --- a/src/closures/cnn.jl +++ b/src/closures/cnn.jl @@ -25,6 +25,7 @@ function cnn( # Make sure there are two velocity fields in input and output @assert c[1] == 2 + # @assert c[1] == 4 @assert c[end] == 2 # Create convolutional closure model @@ -32,6 +33,10 @@ function cnn( # Unflatten and separate u and v velocities V -> reshape(V, Nx, Ny, 2, :), + # # uu, uv, vu, vv + # V -> reshape(V, Nx, Ny, 2, 1, :) .* reshape(V, Nx, Ny, 1, 2, :), + # V -> reshape(V, Nx, Ny, 4, :), + # Add padding so that output has same shape as commutator error u -> pad_circular(u, sum(r)), From 23d1dcbbcab29c6993385fd7613a87ff10bdb9c1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Wed, 23 Aug 2023 14:46:51 +0200 Subject: [PATCH 015/379] Add relative error --- src/IncompressibleNavierStokes.jl | 3 ++- src/closures/loss.jl | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) create mode 100644 src/closures/loss.jl diff --git a/src/IncompressibleNavierStokes.jl b/src/IncompressibleNavierStokes.jl index f256f9935..fbcc59633 100644 --- a/src/IncompressibleNavierStokes.jl +++ b/src/IncompressibleNavierStokes.jl @@ -125,6 +125,7 @@ include("postprocess/save_vtk.jl") # Closure models include("closures/cnn.jl") +include("closures/loss.jl") # Force export SteadyBodyForce @@ -161,7 +162,7 @@ export plot_force, export plotmat # Closure models -export cnn +export cnn, relative_error # ODE methods diff --git a/src/closures/loss.jl b/src/closures/loss.jl new file mode 100644 index 000000000..308011f20 --- /dev/null +++ b/src/closures/loss.jl @@ -0,0 +1 @@ +relative_error(x, y) = sum(norm(x - y) / norm(y) for (x, y) ∈ zip(eachcol(x), eachcol(y))) / size(x, 2) From 29daab3f89a6b422651f8fa3f857220c55b33f67 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Wed, 23 Aug 2023 14:49:16 +0200 Subject: [PATCH 016/379] feat(closures): Add FNO --- src/IncompressibleNavierStokes.jl | 3 +- src/closures/fno.jl | 145 ++++++++++++++++++++++++++++++ 2 files changed, 147 insertions(+), 1 deletion(-) create mode 100644 src/closures/fno.jl diff --git a/src/IncompressibleNavierStokes.jl b/src/IncompressibleNavierStokes.jl index fbcc59633..f68fa613e 100644 --- a/src/IncompressibleNavierStokes.jl +++ b/src/IncompressibleNavierStokes.jl @@ -125,6 +125,7 @@ include("postprocess/save_vtk.jl") # Closure models include("closures/cnn.jl") +include("closures/fno.jl") include("closures/loss.jl") # Force @@ -162,7 +163,7 @@ export plot_force, export plotmat # Closure models -export cnn, relative_error +export cnn, fno, relative_error # ODE methods diff --git a/src/closures/fno.jl b/src/closures/fno.jl new file mode 100644 index 000000000..6bdac45bd --- /dev/null +++ b/src/closures/fno.jl @@ -0,0 +1,145 @@ +""" + fno(setup, c, σ, kmax; rng = Random.default_rng(), kwargs...) + +Create FNO closure model. Return a tuple `(closure, θ)` where `θ` are the initial +parameters and `closure(V, θ)` predicts the commutator error. +""" +function fno(setup, c, σ, kmax; rng = Random.default_rng(), kwargs...) + (; grid) = setup + (; dimension) = grid + + if dimension() == 2 + (; Nx, Ny) = grid + _nx = (Nx, Ny) + elseif dimension() == 3 + (; Nx, Ny, Nz) = grid + _nx = (Nx, Ny, Nz) + end + @assert all(==(first(_nx)), _nx) + + # Make sure there are two velocity fields in input and output + @assert c[1] == 2 + @assert c[end] == 2 + + # Create convolutional closure model + NN = Chain( + # Unflatten and separate u and v velocities + V -> reshape(V, _nx..., 2, :), + + # # uu, uv, vu, vv + # V -> reshape(V, Nx, Ny, 2, 1, :) .* reshape(V, Nx, Ny, 1, 2, :), + # V -> reshape(V, Nx, Ny, 4, :), + + # Some Fourier layers + (FourierLayer(dimension, c[i] => c[i+1], kmax; σ = σ[i]) for i ∈ eachindex(r))..., + + # Flatten to vector + u -> reshape(u, 2 * prod(_nx...), :), + ) + + # Create parameter vector (empty state) + params, state = Lux.setup(rng, NN) + θ = ComponentArray(params) + + """ + closure(V, θ) + + Compute closure term for given parameters `θ`. + """ + function closure end + closure(V, θ) = first(NN(V, θ, state)) + closure(V::AbstractVector, θ) = reshape(closure(reshape(V, :, 1), θ), :) + + closure, θ +end + +""" + FourierLayer(dimension, cin => cout, kmax; σ = identity, init_weight = glorot_uniform) + +Fourier layer operating on uniformly discretized functions. + +Some important sizes: + +- `dimension`: Spatial dimension, eg `Dimension(2)` or `Dimension(3)`. +- `(cin, nx..., nsample)`: input size +- `(cout, nx..., nsample)`: output size +- `nx = fill(n, dimension())`: number of points in each spatial dimension +- `n ≥ kmax`: Same number of points in each spatial dimension, must be + larger than cut-off frequency +- `kmax`: Cut-off frequency +- `nsample`: Number of input samples (treated independently) +""" +struct FourierLayer{N,A,F} <: Lux.AbstractExplicitLayer + dimension::Dimension{N} + cin::Int + cout::Int + kmax::Int + σ::A + init_weight::F +end + +FourierLayer( + dimension, + ch::Pair{Int,Int}, + kmax; + σ = identity, + init_weight = glorot_uniform, +) = FourierLayer(dimension, first(ch), last(ch), kmax, σ, init_weight) + +Lux.initialparameters( + rng::AbstractRNG, + (; dimension, kmax, cin, cout, init_weight)::FourierLayer, +) = (; + spatial_weight = init_weight(rng, cout, cin), + spectral_weights = init_weight(rng, cout, cin, fill(kmax + 1, dimension())..., 2), +) +Lux.initialstates(::AbstractRNG, ::FourierLayer) = (;) +Lux.parameterlength((; cin, cout, kmax)::FourierLayer) = + cout * cin + cout * cin * (kmax + 1)^dimension() * 2 +Lux.statelength(::FourierLayer) = 0 + +# Pass inputs through Fourier layer +function ((; cout, cin, kmax, σ)::FourierLayer{N})(x, params, state) + _cin, nx..., nsample = size(x) + @assert _cin == cin "Number of input channels must be compatible with weights" + @assert all(==(first(nx)), nx) "Fourier layer requires same number of grid points in each dimension" + @assert kmax ≤ first(nx) "Fourier layer input must be discretized on at least `kmax` points" + + # Destructure params + # The real and imaginary parts of R are stored in two separate channels + W = params.spatial_weight + R = params.spectral_weights + R = selectdim(R, 3 + N, 1) .+ im .* selectdim(R, 3 + N, 2) + + # Spatial part (applied point-wise) + y = reshape(x, cin, :) + y = W * y + y = reshape(y, cout, nx..., :) + + # Spectral part (applied mode-wise) + # - go to complex-valued spectral space + # - chop off high frequencies + # - multiply with weights mode-wise + # - pad with zeros to restore original shape + # - go back to real valued spatial representation + ikeep = ntuple(Returns(1:kmax+1), N) + z = fft(x, 2:1+N) + z = z[:, ikeep..., :] + z = reshape(z, 1, cin, ikeep..., :) + z = sum(R .* z; dims = 2) + z = reshape(z, cout, ikeep..., :) + z = pad_zeros(z, ntuple(i -> isodd(i) ? 0 : first(nx) - kmax - 1, 2N); dims = 2:1+N) + z = real.(ifft(z, 2:1+N)) + + # Outer layer: Activation over combined spatial and spectral parts + # Note: Even though high frequencies are chopped of in `z` and may + # possibly not be present in the input at all, `σ` creates new high frequencies. + # High frequency functions may thus be represented using a sequence of + # Fourier layers. In this case, the `y`s are the only place where + # information contained in high + # input frequencies survive in a Fourier layer. + v = σ.(y .+ z) + + # Fourier layer does not change state + v, state +end From 84c2b572b3808ac8ca85cb8db3ffe240969dcbbf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Wed, 23 Aug 2023 14:52:03 +0200 Subject: [PATCH 017/379] Update scripts --- scratch/filtered.jl | 40 +++++++++++++++++++++--- scratch/train_model.jl | 70 ++++++++++++++++++++++++++++++++---------- 2 files changed, 90 insertions(+), 20 deletions(-) diff --git a/scratch/filtered.jl b/scratch/filtered.jl index bc14e8693..0f2fa2e91 100644 --- a/scratch/filtered.jl +++ b/scratch/filtered.jl @@ -8,6 +8,7 @@ end #src using GLMakie using IncompressibleNavierStokes using LinearAlgebra +using Lux # Floating point precision T = Float32 @@ -15,10 +16,11 @@ T = Float32 # To use CPU: Do not move any arrays device = identity -# # To use GPU, use `cu` to move arrays to the GPU. -# # Note: `cu` converts to Float32 -# using CUDA -# device = cu +# To use GPU, use `cu` to move arrays to the GPU. +# Note: `cu` converts to Float32 +using CUDA +using LuxCUDA +device = cu # Viscosity model Re = T(2_000) @@ -114,6 +116,7 @@ V, p, outputs = solve_unsteady( ); # V₀, p₀ = V, p +# V, p = V₀, p₀ Vbar = KV * V pbar = Kp * p @@ -148,3 +151,32 @@ norm(Vbar_smag - Vbar) / norm(Vbar) # Filtered quantities filtered = outputs[1] filtered.V[end] + +closure, θ = + cnn(setup_coarse, [5, 5, 5], [2, 5, 5, 2], [tanh, tanh, identity], [true, true, false];) + +@time closure(Vbar, θ); + +cuVbar = cu(Vbar) +cuθ = cu(θ) +@time closure(cuVbar, cuθ); + +size(Vbar) +size(closure(Vbar, θ)) + +using Zygote + +@time gradient(θ -> sum(closure(Vbar, θ)), θ); +@time gradient(θ -> sum(closure(cuVbar, θ)), cuθ); + +Vbar_cnn, pbar_cnn, outputs_cnn = solve_unsteady( + (; setup_coarse..., closure_model = V -> closure(V, 1.0f-3 * cuθ)), + KV * V₀, + Kp * p₀, + tlims; + Δt = T(2e-4), + processors = processors_coarse, + pressure_solver = pressure_solver_coarse, + inplace = true, + device, +); diff --git a/scratch/train_model.jl b/scratch/train_model.jl index 859a70453..8ffd4e7dd 100644 --- a/scratch/train_model.jl +++ b/scratch/train_model.jl @@ -10,6 +10,7 @@ using IncompressibleNavierStokes using JLD2 using LinearAlgebra using Lux +using NNlib using Optimisers using Random using Zygote @@ -17,8 +18,8 @@ using Zygote # Floating point precision T = Float32 -# To use CPU: Do not move any arrays -device = identity +# # To use CPU: Do not move any arrays +# device = identity # To use GPU, use `cu` to move arrays to the GPU. # Note: `cu` converts to Float32 @@ -174,17 +175,25 @@ end norm(commutator_error[:, 1, 1]) / norm(filtered.F[:, 1, 1]) -closure, θ = cnn( +# closure, θ₀ = cnn( +# les, +# [5, 5, 5], +# [2, 8, 8, 2], +# [tanh, tanh, identity], +# [true, true, false]; +# ) + +closure, θ₀ = cnn( les, [5, 5, 5], - [2, 5, 5, 2], - [tanh, tanh, identity], + [2, 8, 8, 2], + [leakyrelu, leakyrelu, identity], [true, true, false]; ) -loss(x, y, θ) = sum(abs2, closure(x, θ) - y) / length(y) +loss(x, y, θ) = sum(abs2, closure(x, θ) - y) / sum(abs2, y) -loss(filtered.V[:, 1, 1], commutator_error[:, 1, 1], θ) +loss(filtered.V[:, 1, 1], commutator_error[:, 1, 1], θ₀) function create_loss(x, y; nuse = size(x, 2)) x = reshape(x, size(x, 1), :) @@ -201,20 +210,49 @@ end randloss = create_loss(filtered.V, commutator_error; nuse = 100) -devθ₀ = device(θ) +# θ = 2f-2 * device(θ₀) +θ = 5.0f-2 * device(θ₀) -randloss(devθ) +randloss(θ) -first(gradient(randloss, devθ)) +first(gradient(randloss, θ)) V_test = device(reshape(filtered.V[:, 1:20, 1:2], :, 40)) c_test = device(reshape(commutator_error[:, 1:20, 1:2], :, 40)) -devθ = devθ₀ -opt = Optimisers.setup(Adam(0.005f0), devθ) -for i = 1:200 - g = first(gradient(randloss, devθ)) - opt, devθ = Optimisers.update(opt, devθ, g) - e_test = norm(closure(V_test, devθ) - c_test) / norm(c_test) +opt = Optimisers.setup(Adam(1.0f-3), θ) + +obs = Observable([(0, T(0))]) +fig = lines(obs) +hlines!([1.0f0]) +obs[] = fill((0, T(0)), 0) +j = 0 +display(fig) + +niter = 1000 +for i = 1:niter + g = first(gradient(randloss, θ)) + opt, θ = Optimisers.update(opt, θ, g) + e_test = norm(closure(V_test, θ) - c_test) / norm(c_test) @info "Iteration $i\trelative test error: $e_test" + obs[] = push!(obs[], (j + i, e_test)) end +j += niter + +# jldsave("output/theta.jld2"; θ = Array(θ)) +# θθ = load("output/theta.jld2") +# θθ = θθ["θ"] +# θθ = cu(θθ) +# θ .= θθ + +relative_error(closure(V_test, θ), c_test) + +CUDA.memory_status() + +GC.gc() + +CUDA.reclaim() + +Array(θ.layer_5) +θ.layer_4.weight +θ.layer_4.bias From c787c4e4d05bbd27e81552e182e30619e0acfc4c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Wed, 23 Aug 2023 14:53:41 +0200 Subject: [PATCH 018/379] fix(FNO): Add missing dependency for padding --- src/IncompressibleNavierStokes.jl | 1 + 1 file changed, 1 insertion(+) diff --git a/src/IncompressibleNavierStokes.jl b/src/IncompressibleNavierStokes.jl index f68fa613e..635447880 100644 --- a/src/IncompressibleNavierStokes.jl +++ b/src/IncompressibleNavierStokes.jl @@ -11,6 +11,7 @@ using FFTW using IterativeSolvers using LinearAlgebra using Lux +using NNlib using Printf using Random using SparseArrays From ce783db16deb3cedf391de6ef5acc24bdfd1b44b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Wed, 23 Aug 2023 15:16:15 +0200 Subject: [PATCH 019/379] fix(FNO): Permute dims before and after --- src/closures/fno.jl | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/src/closures/fno.jl b/src/closures/fno.jl index 6bdac45bd..e754e343c 100644 --- a/src/closures/fno.jl +++ b/src/closures/fno.jl @@ -8,10 +8,12 @@ function fno(setup, c, σ, kmax; rng = Random.default_rng(), kwargs...) (; grid) = setup (; dimension) = grid - if dimension() == 2 + N = dimension() + + if N == 2 (; Nx, Ny) = grid _nx = (Nx, Ny) - elseif dimension() == 3 + elseif N == 3 (; Nx, Ny, Nz) = grid _nx = (Nx, Ny, Nz) end @@ -26,6 +28,9 @@ function fno(setup, c, σ, kmax; rng = Random.default_rng(), kwargs...) # Unflatten and separate u and v velocities V -> reshape(V, _nx..., 2, :), + # Put channels in first dimension + V -> permutedims(V, (N + 1, (1:N)..., N + 2)) + # # uu, uv, vu, vv # V -> reshape(V, Nx, Ny, 2, 1, :) .* reshape(V, Nx, Ny, 1, 2, :), # V -> reshape(V, Nx, Ny, 4, :), @@ -33,6 +38,9 @@ function fno(setup, c, σ, kmax; rng = Random.default_rng(), kwargs...) # Some Fourier layers (FourierLayer(dimension, c[i] => c[i+1], kmax; σ = σ[i]) for i ∈ eachindex(r))..., + # Put channels back after spatial dimensions + u -> permutedims(u, ((2:N+1)..., 1, N + 2)) + # Flatten to vector u -> reshape(u, 2 * prod(_nx...), :), ) From 964f7e91258387570483441d480b5674467f19ea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Wed, 23 Aug 2023 15:24:43 +0200 Subject: [PATCH 020/379] comments(fno): Minor changes --- src/closures/fno.jl | 32 +++++++++++++++++++------------- 1 file changed, 19 insertions(+), 13 deletions(-) diff --git a/src/closures/fno.jl b/src/closures/fno.jl index e754e343c..637ea2d7b 100644 --- a/src/closures/fno.jl +++ b/src/closures/fno.jl @@ -23,7 +23,7 @@ function fno(setup, c, σ, kmax; rng = Random.default_rng(), kwargs...) @assert c[1] == 2 @assert c[end] == 2 - # Create convolutional closure model + # Create FNO closure model NN = Chain( # Unflatten and separate u and v velocities V -> reshape(V, _nx..., 2, :), @@ -68,13 +68,13 @@ Fourier layer operating on uniformly discretized functions. Some important sizes: -- `dimension`: Spatial dimension, eg `Dimension(2)` or `Dimension(3)`. -- `(cin, nx..., nsample)`: input size -- `(cout, nx..., nsample)`: output size -- `nx = fill(n, dimension())`: number of points in each spatial dimension +- `dimension`: Spatial dimension, e.g. `Dimension(2)` or `Dimension(3)`. +- `(cin, nx..., nsample)`: Input size +- `(cout, nx..., nsample)`: Output size +- `nx = fill(n, dimension())`: Number of points in each spatial dimension - `n ≥ kmax`: Same number of points in each spatial dimension, must be - larger than cut-off frequency -- `kmax`: Cut-off frequency + larger than cut-off wavenumber +- `kmax`: Cut-off wavenumber - `nsample`: Number of input samples (treated independently) """ struct FourierLayer{N,A,F} <: Lux.AbstractExplicitLayer @@ -108,6 +108,12 @@ Lux.statelength(::FourierLayer) = 0 # Pass inputs through Fourier layer function ((; cout, cin, kmax, σ)::FourierLayer{N})(x, params, state) + # TODO: Check if this is more efficient for + # size(x) = (cin, nx..., nsample) or + # size(x) = (nx..., cin, nsample) + + # TODO: Set FFT normalization so that layer is truly grid independent + _cin, nx..., nsample = size(x) @assert _cin == cin "Number of input channels must be compatible with weights" @assert all(==(first(nx)), nx) "Fourier layer requires same number of grid points in each dimension" @@ -126,7 +132,7 @@ function ((; cout, cin, kmax, σ)::FourierLayer{N})(x, params, state) # Spectral part (applied mode-wise) # - go to complex-valued spectral space - # - chop off high frequencies + # - chop off high wavenumbers # - multiply with weights mode-wise # - pad with zeros to restore original shape # - go back to real valued spatial representation @@ -140,14 +146,14 @@ function ((; cout, cin, kmax, σ)::FourierLayer{N})(x, params, state) z = real.(ifft(z, 2:1+N)) # Outer layer: Activation over combined spatial and spectral parts - # Note: Even though high frequencies are chopped of in `z` and may - # possibly not be present in the input at all, `σ` creates new high frequencies. - # High frequency functions may thus be represented using a sequence of + # Note: Even though high wavenumbers are chopped off in `z` and may + # possibly not be present in the input at all, `σ` creates new high wavenumbers. + # High wavenumber functions may thus be represented using a sequence of # Fourier layers. In this case, the `y`s are the only place where # information contained in high - # input frequencies survive in a Fourier layer. + # input wavenumbers survive in a Fourier layer. v = σ.(y .+ z) - # Fourier layer does not change state + # Fourier layer does not modify state v, state end From 0e268eb2aba3ba7fefa7add25c0441020ce6832b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Thu, 24 Aug 2023 10:40:55 +0200 Subject: [PATCH 021/379] Update dependencies --- Project.toml | 2 ++ examples/Project.toml | 4 ++++ 2 files changed, 6 insertions(+) diff --git a/Project.toml b/Project.toml index 5d22038a4..929e3e87e 100644 --- a/Project.toml +++ b/Project.toml @@ -5,11 +5,13 @@ version = "0.4.1" [deps] Adapt = "79e6a3ab-5dfb-504d-930d-738a2a938a0e" +ComponentArrays = "b0b7db55-cfe3-40fc-9ded-d10e2dbeff66" FFTW = "7a1cc6ca-52ef-59f5-83cd-3a7055c09341" IterativeSolvers = "42fd0dbc-a981-5370-80f2-aaf504508153" LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" Lux = "b2108857-7c20-44ae-9111-449ecde12c47" Makie = "ee78f7c6-11fb-53f2-987a-cfe4a2b5a57a" +NNlib = "872c559c-99b0-510c-b3b7-b6c96a88d5cd" Printf = "de0858da-6303-5e67-8744-51eddeeeb8d7" Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" SparseArrays = "2f01184e-e22b-5df5-ae63-d93ebab69eaf" diff --git a/examples/Project.toml b/examples/Project.toml index 2fe454328..a16800d51 100644 --- a/examples/Project.toml +++ b/examples/Project.toml @@ -3,4 +3,8 @@ CairoMakie = "13f3f980-e62b-5c42-98c6-ff1f3baf88f0" FFTW = "7a1cc6ca-52ef-59f5-83cd-3a7055c09341" GLMakie = "e9467ef8-e4e7-5192-8a1a-b1aee30e663a" IncompressibleNavierStokes = "5e318141-6589-402b-868d-77d7df8c442e" +JLD2 = "033835bb-8acc-5ee8-8aae-3f567f8a3819" LaTeXStrings = "b964fa9f-0449-5b57-a5c2-d3ea65f4040f" +Observables = "510215fc-4207-5dde-b226-833fc4488ee2" +Optimisers = "3bd65402-5787-11e9-1adc-39752487f4e2" +Zygote = "e88e6eb3-aa80-5325-afca-941959d7151f" From 885235342b3c0e90dda1278ba815f21997872dee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Thu, 24 Aug 2023 15:03:08 +0200 Subject: [PATCH 022/379] fix(FNO): fix and add dense layers --- src/closures/fno.jl | 31 +++++++++++++++++++------------ 1 file changed, 19 insertions(+), 12 deletions(-) diff --git a/src/closures/fno.jl b/src/closures/fno.jl index 637ea2d7b..56d9ce51c 100644 --- a/src/closures/fno.jl +++ b/src/closures/fno.jl @@ -1,10 +1,10 @@ """ - fno(setup, c, σ, kmax; rng = Random.default_rng(), kwargs...) + fno(setup, kmax, c, σ, ψ; rng = Random.default_rng(), kwargs...) Create FNO closure model. Return a tuple `(closure, θ)` where `θ` are the initial parameters and `closure(V, θ)` predicts the commutator error. """ -function fno(setup, c, σ, kmax; rng = Random.default_rng(), kwargs...) +function fno(setup, kmax, c, σ, ψ; rng = Random.default_rng(), kwargs...) (; grid) = setup (; dimension) = grid @@ -21,7 +21,6 @@ function fno(setup, c, σ, kmax; rng = Random.default_rng(), kwargs...) # Make sure there are two velocity fields in input and output @assert c[1] == 2 - @assert c[end] == 2 # Create FNO closure model NN = Chain( @@ -29,20 +28,24 @@ function fno(setup, c, σ, kmax; rng = Random.default_rng(), kwargs...) V -> reshape(V, _nx..., 2, :), # Put channels in first dimension - V -> permutedims(V, (N + 1, (1:N)..., N + 2)) + V -> permutedims(V, (N + 1, (1:N)..., N + 2)), # # uu, uv, vu, vv # V -> reshape(V, Nx, Ny, 2, 1, :) .* reshape(V, Nx, Ny, 1, 2, :), # V -> reshape(V, Nx, Ny, 4, :), # Some Fourier layers - (FourierLayer(dimension, c[i] => c[i+1], kmax; σ = σ[i]) for i ∈ eachindex(r))..., + (FourierLayer(dimension, c[i] => c[i+1], kmax; σ = σ[i]) for i ∈ eachindex(σ))..., + + # Compress with a final dense layer + Dense(c[end] => 2 * c[end], ψ), + Dense(2 * c[end] => 2; use_bias = false), # Put channels back after spatial dimensions - u -> permutedims(u, ((2:N+1)..., 1, N + 2)) + u -> permutedims(u, ((2:N+1)..., 1, N + 2)), # Flatten to vector - u -> reshape(u, 2 * prod(_nx...), :), + u -> reshape(u, 2 * prod(_nx), :), ) # Create parameter vector (empty state) @@ -107,13 +110,15 @@ Lux.parameterlength((; cin, cout, kmax)::FourierLayer) = Lux.statelength(::FourierLayer) = 0 # Pass inputs through Fourier layer -function ((; cout, cin, kmax, σ)::FourierLayer{N})(x, params, state) +function ((; dimension, cout, cin, kmax, σ)::FourierLayer)(x, params, state) # TODO: Check if this is more efficient for # size(x) = (cin, nx..., nsample) or # size(x) = (nx..., cin, nsample) - + # TODO: Set FFT normalization so that layer is truly grid independent + N = dimension() + _cin, nx..., nsample = size(x) @assert _cin == cin "Number of input channels must be compatible with weights" @assert all(==(first(nx)), nx) "Fourier layer requires same number of grid points in each dimension" @@ -137,13 +142,15 @@ function ((; cout, cin, kmax, σ)::FourierLayer{N})(x, params, state) # - pad with zeros to restore original shape # - go back to real valued spatial representation ikeep = ntuple(Returns(1:kmax+1), N) - z = fft(x, 2:1+N) + dims = ntuple(i -> 1 + i, N) + z = fft(x, dims) z = z[:, ikeep..., :] z = reshape(z, 1, cin, ikeep..., :) z = sum(R .* z; dims = 2) z = reshape(z, cout, ikeep..., :) - z = pad_zeros(z, ntuple(i -> isodd(i) ? 0 : first(nx) - kmax - 1, 2N); dims = 2:1+N) - z = real.(ifft(z, 2:1+N)) + # z = pad_zeros(z, ntuple(i -> isodd(i) ? 0 : first(nx) - kmax - 1, 2N); dims = 2:1+N) + z = pad_zeros(z, ntuple(i -> isodd(i) ? 0 : first(nx) - kmax - 1, 2N); dims) + z = real.(ifft(z, dims)) # Outer layer: Activation over combined spatial and spectral parts # Note: Even though high wavenumbers are chopped off in `z` and may From a0af046604ee117789dceb05032a406d662e1c18 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Thu, 24 Aug 2023 18:21:08 +0200 Subject: [PATCH 023/379] refactor(FNO): Put spatial dimensions first --- scratch/train_model.jl | 96 +++++++++++++++++++++++++++++------------- src/closures/fno.jl | 53 +++++++++++------------ 2 files changed, 94 insertions(+), 55 deletions(-) diff --git a/scratch/train_model.jl b/scratch/train_model.jl index 8ffd4e7dd..c3480032a 100644 --- a/scratch/train_model.jl +++ b/scratch/train_model.jl @@ -65,8 +65,7 @@ filter_saver(setup, KV, Kp, Ωbar; nupdate = 1, bc_vectors = get_bc_vectors(setu _FG = fill(zeros(T, 0), 0) _p = fill(zeros(T, 0), 0) _t = fill(zero(T), 0) - @lift begin - (; V, p, t) = $step_observer + on(step_observer) do (; V, p, t) F, = momentum(V, V, p, t, setup; bc_vectors, nopressure = true) FG, = momentum(V, V, p, t, setup; bc_vectors, nopressure = false) push!(_V, KV * Array(V)) @@ -179,43 +178,38 @@ norm(commutator_error[:, 1, 1]) / norm(filtered.F[:, 1, 1]) # les, # [5, 5, 5], # [2, 8, 8, 2], -# [tanh, tanh, identity], +# [leakyrelu, leakyrelu, identity], # [true, true, false]; # ) -closure, θ₀ = cnn( - les, - [5, 5, 5], - [2, 8, 8, 2], - [leakyrelu, leakyrelu, identity], - [true, true, false]; -) +closure, θ₀ = fno(les, 8, [2, 16, 8, 8], [gelu, gelu, identity], gelu) + +length(θ₀) loss(x, y, θ) = sum(abs2, closure(x, θ) - y) / sum(abs2, y) loss(filtered.V[:, 1, 1], commutator_error[:, 1, 1], θ₀) -function create_loss(x, y; nuse = size(x, 2)) +function create_loss(x, y; nuse = size(x, 2), device = identity) x = reshape(x, size(x, 1), :) y = reshape(y, size(y, 1), :) nsample = size(x, 2) d = ndims(x) function randloss(θ) i = Zygote.@ignore sort(shuffle(1:nsample)[1:nuse]) - xuse = Zygote.@ignore device(selectdim(x, d, i)) - yuse = Zygote.@ignore device(selectdim(y, d, i)) + xuse = Zygote.@ignore device(Array(selectdim(x, d, i))) + yuse = Zygote.@ignore device(Array(selectdim(y, d, i))) loss(xuse, yuse, θ) end end -randloss = create_loss(filtered.V, commutator_error; nuse = 100) +randloss = create_loss(filtered.V, commutator_error; nuse = 50, device) -# θ = 2f-2 * device(θ₀) θ = 5.0f-2 * device(θ₀) randloss(θ) -first(gradient(randloss, θ)) +@time first(gradient(randloss, θ)); V_test = device(reshape(filtered.V[:, 1:20, 1:2], :, 40)) c_test = device(reshape(commutator_error[:, 1:20, 1:2], :, 40)) @@ -223,21 +217,27 @@ c_test = device(reshape(commutator_error[:, 1:20, 1:2], :, 40)) opt = Optimisers.setup(Adam(1.0f-3), θ) obs = Observable([(0, T(0))]) -fig = lines(obs) + +fig = lines(obs; axis = (; title = "Relative prediction error", xlabel = "Iteration")) hlines!([1.0f0]) +display(fig) + obs[] = fill((0, T(0)), 0) j = 0 -display(fig) -niter = 1000 +nplot = 10 +niter = 500 for i = 1:niter g = first(gradient(randloss, θ)) opt, θ = Optimisers.update(opt, θ, g) - e_test = norm(closure(V_test, θ) - c_test) / norm(c_test) - @info "Iteration $i\trelative test error: $e_test" - obs[] = push!(obs[], (j + i, e_test)) + if i % nplot == 0 + e_test = norm(closure(V_test, θ) - c_test) / norm(c_test) + @info "Iteration $i\trelative test error: $e_test" + _i = (j += nplot) + obs[] = push!(obs[], (_i, e_test)) + autolimits!(fig.axis) + end end -j += niter # jldsave("output/theta.jld2"; θ = Array(θ)) # θθ = load("output/theta.jld2") @@ -247,12 +247,50 @@ j += niter relative_error(closure(V_test, θ), c_test) -CUDA.memory_status() +size(filtered.V) -GC.gc() +devles = device(les); -CUDA.reclaim() +V_nm, p_nm, outputs_nm = solve_unsteady( + les, + filtered.V[:, 1, 1], + filtered.p[:, 1, 1], + (t_burn, t_end); + Δt = T(2e-4), + processors = ( + step_logger(; nupdate = 10), + field_plotter(devles; type = heatmap, nupdate = 1), + ), + pressure_solver = pressure_solver_les, + inplace = false, + device, +) + +V_fno, p_fno, outputs_fno = solve_unsteady( + (; les..., closure_model = V -> closure(V, θ)), + filtered.V[:, 1, 1], + filtered.p[:, 1, 1], + (t_burn, t_end); + Δt = T(2e-4), + processors = ( + step_logger(; nupdate = 10), + field_plotter(devles; type = heatmap, nupdate = 1), + ), + pressure_solver = pressure_solver_les, + inplace = false, + device, +) -Array(θ.layer_5) -θ.layer_4.weight -θ.layer_4.bias +V = filtered.V[:, end, 1] +p = filtered.p[:, end, 1] + +relative_error(V_nm, V) +relative_error(V_fno, V) + +plot_vorticity(les, V_nm, t_end) +plot_vorticity(les, V_fno, t_end) +plot_vorticity(les, V, t_end) + +CUDA.memory_status() +GC.gc() +CUDA.reclaim() diff --git a/src/closures/fno.jl b/src/closures/fno.jl index 56d9ce51c..74345d065 100644 --- a/src/closures/fno.jl +++ b/src/closures/fno.jl @@ -27,15 +27,15 @@ function fno(setup, kmax, c, σ, ψ; rng = Random.default_rng(), kwargs...) # Unflatten and separate u and v velocities V -> reshape(V, _nx..., 2, :), - # Put channels in first dimension - V -> permutedims(V, (N + 1, (1:N)..., N + 2)), - # # uu, uv, vu, vv # V -> reshape(V, Nx, Ny, 2, 1, :) .* reshape(V, Nx, Ny, 1, 2, :), # V -> reshape(V, Nx, Ny, 4, :), # Some Fourier layers - (FourierLayer(dimension, c[i] => c[i+1], kmax; σ = σ[i]) for i ∈ eachindex(σ))..., + (FourierLayer(dimension, kmax, c[i] => c[i+1]; σ = σ[i]) for i ∈ eachindex(σ))..., + + # Put channels in first dimension + V -> permutedims(V, (N + 1, (1:N)..., N + 2)), # Compress with a final dense layer Dense(c[end] => 2 * c[end], ψ), @@ -65,15 +65,15 @@ function fno(setup, kmax, c, σ, ψ; rng = Random.default_rng(), kwargs...) end """ - FourierLayer(dimension, cin => cout, kmax; σ = identity, init_weight = glorot_uniform) + FourierLayer(dimension, kmax, cin => cout; σ = identity, init_weight = glorot_uniform) Fourier layer operating on uniformly discretized functions. Some important sizes: - `dimension`: Spatial dimension, e.g. `Dimension(2)` or `Dimension(3)`. -- `(cin, nx..., nsample)`: Input size -- `(cout, nx..., nsample)`: Output size +- `(nx..., cin, nsample)`: Input size +- `(nx..., cout, nsample)`: Output size - `nx = fill(n, dimension())`: Number of points in each spatial dimension - `n ≥ kmax`: Same number of points in each spatial dimension, must be larger than cut-off wavenumber @@ -82,35 +82,35 @@ Some important sizes: """ struct FourierLayer{N,A,F} <: Lux.AbstractExplicitLayer dimension::Dimension{N} + kmax::Int cin::Int cout::Int - kmax::Int σ::A init_weight::F end FourierLayer( dimension, - ch::Pair{Int,Int}, - kmax; + kmax, + ch::Pair{Int,Int}; σ = identity, init_weight = glorot_uniform, -) = FourierLayer(dimension, first(ch), last(ch), kmax, σ, init_weight) +) = FourierLayer(dimension, kmax, first(ch), last(ch), σ, init_weight) Lux.initialparameters( rng::AbstractRNG, (; dimension, kmax, cin, cout, init_weight)::FourierLayer, ) = (; spatial_weight = init_weight(rng, cout, cin), - spectral_weights = init_weight(rng, cout, cin, fill(kmax + 1, dimension())..., 2), + spectral_weights = init_weight(rng, fill(kmax + 1, dimension())..., cout, cin, 2), ) Lux.initialstates(::AbstractRNG, ::FourierLayer) = (;) -Lux.parameterlength((; cin, cout, kmax)::FourierLayer) = - cout * cin + cout * cin * (kmax + 1)^dimension() * 2 +Lux.parameterlength((; kmax, cin, cout)::FourierLayer) = + cout * cin + (kmax + 1)^dimension() * 2 * cout * cin Lux.statelength(::FourierLayer) = 0 # Pass inputs through Fourier layer -function ((; dimension, cout, cin, kmax, σ)::FourierLayer)(x, params, state) +function ((; dimension, kmax, cout, cin, σ)::FourierLayer)(x, params, state) # TODO: Check if this is more efficient for # size(x) = (cin, nx..., nsample) or # size(x) = (nx..., cin, nsample) @@ -119,7 +119,7 @@ function ((; dimension, cout, cin, kmax, σ)::FourierLayer)(x, params, state) N = dimension() - _cin, nx..., nsample = size(x) + nx..., _cin, nsample = size(x) @assert _cin == cin "Number of input channels must be compatible with weights" @assert all(==(first(nx)), nx) "Fourier layer requires same number of grid points in each dimension" @assert kmax ≤ first(nx) "Fourier layer input must be discretized on at least `kmax` points" @@ -127,13 +127,14 @@ function ((; dimension, cout, cin, kmax, σ)::FourierLayer)(x, params, state) # Destructure params # The real and imaginary parts of R are stored in two separate channels W = params.spatial_weight + W = reshape(W, ntuple(Returns(1), N)..., cout, cin) R = params.spectral_weights - R = selectdim(R, 3 + N, 1) .+ im .* selectdim(R, 3 + N, 2) + R = selectdim(R, N + 3, 1) .+ im .* selectdim(R, N + 3, 2) # Spatial part (applied point-wise) - y = reshape(x, cin, :) - y = W * y - y = reshape(y, cout, nx..., :) + y = reshape(x, nx..., 1, cin, :) + y = sum(W .* y; dims = N + 2) + y = reshape(y, nx..., cout, :) # Spectral part (applied mode-wise) # - go to complex-valued spectral space @@ -142,13 +143,13 @@ function ((; dimension, cout, cin, kmax, σ)::FourierLayer)(x, params, state) # - pad with zeros to restore original shape # - go back to real valued spatial representation ikeep = ntuple(Returns(1:kmax+1), N) - dims = ntuple(i -> 1 + i, N) + nkeep = ntuple(Returns(kmax+1), N) + dims = ntuple(identity, N) z = fft(x, dims) - z = z[:, ikeep..., :] - z = reshape(z, 1, cin, ikeep..., :) - z = sum(R .* z; dims = 2) - z = reshape(z, cout, ikeep..., :) - # z = pad_zeros(z, ntuple(i -> isodd(i) ? 0 : first(nx) - kmax - 1, 2N); dims = 2:1+N) + z = z[ikeep..., :, :] + z = reshape(z, nkeep..., 1, cin, :) + z = sum(R .* z; dims = N + 2) + z = reshape(z, nkeep..., cout, :) z = pad_zeros(z, ntuple(i -> isodd(i) ? 0 : first(nx) - kmax - 1, 2N); dims) z = real.(ifft(z, dims)) From 2fd1e333d7214756ed245e0ae245c4d6354e3a24 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Fri, 25 Aug 2023 13:20:58 +0200 Subject: [PATCH 024/379] FNO: Add variable kmax --- scratch/train_model.jl | 21 ++++++++++++++++++--- src/closures/fno.jl | 39 +++++++++++++++++++++++++-------------- 2 files changed, 43 insertions(+), 17 deletions(-) diff --git a/scratch/train_model.jl b/scratch/train_model.jl index c3480032a..9e1c1d7c8 100644 --- a/scratch/train_model.jl +++ b/scratch/train_model.jl @@ -182,7 +182,22 @@ norm(commutator_error[:, 1, 1]) / norm(filtered.F[:, 1, 1]) # [true, true, false]; # ) -closure, θ₀ = fno(les, 8, [2, 16, 8, 8], [gelu, gelu, identity], gelu) +closure, θ₀ = fno( + # Setup + les, + + # Cut-off wavenumbers + [8, 8, 8], + + # Channel sizes + [16, 8, 8], + + # Fourier activations + [gelu, gelu, identity], + + # Dense activation + gelu, +) length(θ₀) @@ -214,7 +229,7 @@ randloss(θ) V_test = device(reshape(filtered.V[:, 1:20, 1:2], :, 40)) c_test = device(reshape(commutator_error[:, 1:20, 1:2], :, 40)) -opt = Optimisers.setup(Adam(1.0f-3), θ) +opt = Optimisers.setup(Adam(1.0f-2), θ) obs = Observable([(0, T(0))]) @@ -230,7 +245,7 @@ niter = 500 for i = 1:niter g = first(gradient(randloss, θ)) opt, θ = Optimisers.update(opt, θ, g) - if i % nplot == 0 + if i % nplot == 0 e_test = norm(closure(V_test, θ) - c_test) / norm(c_test) @info "Iteration $i\trelative test error: $e_test" _i = (j += nplot) diff --git a/src/closures/fno.jl b/src/closures/fno.jl index 74345d065..7470f69f7 100644 --- a/src/closures/fno.jl +++ b/src/closures/fno.jl @@ -1,8 +1,8 @@ """ fno(setup, kmax, c, σ, ψ; rng = Random.default_rng(), kwargs...) -Create FNO closure model. Return a tuple `(closure, θ)` where `θ` are the initial -parameters and `closure(V, θ)` predicts the commutator error. +Create FNO closure model. Return a tuple `(closure, θ)` where `θ` are the +initial parameters and `closure(V, θ)` predicts the commutator error. """ function fno(setup, kmax, c, σ, ψ; rng = Random.default_rng(), kwargs...) (; grid) = setup @@ -19,8 +19,11 @@ function fno(setup, kmax, c, σ, ψ; rng = Random.default_rng(), kwargs...) end @assert all(==(first(_nx)), _nx) + # Fourier layers + @assert length(kmax) == length(c) == length(σ) + # Make sure there are two velocity fields in input and output - @assert c[1] == 2 + c = [2; c] # Create FNO closure model NN = Chain( @@ -32,7 +35,9 @@ function fno(setup, kmax, c, σ, ψ; rng = Random.default_rng(), kwargs...) # V -> reshape(V, Nx, Ny, 4, :), # Some Fourier layers - (FourierLayer(dimension, kmax, c[i] => c[i+1]; σ = σ[i]) for i ∈ eachindex(σ))..., + ( + FourierLayer(dimension, kmax[i], c[i] => c[i+1]; σ = σ[i]) for i ∈ eachindex(σ) + )..., # Put channels in first dimension V -> permutedims(V, (N + 1, (1:N)..., N + 2)), @@ -97,10 +102,7 @@ FourierLayer( init_weight = glorot_uniform, ) = FourierLayer(dimension, kmax, first(ch), last(ch), σ, init_weight) -Lux.initialparameters( - rng::AbstractRNG, - (; dimension, kmax, cin, cout, init_weight)::FourierLayer, -) = (; +Lux.initialparameters(rng, (; dimension, kmax, cin, cout, init_weight)::FourierLayer) = (; spatial_weight = init_weight(rng, cout, cin), spectral_weights = init_weight(rng, fill(kmax + 1, dimension())..., cout, cin, 2), ) @@ -117,6 +119,7 @@ function ((; dimension, kmax, cout, cin, σ)::FourierLayer)(x, params, state) # TODO: Set FFT normalization so that layer is truly grid independent + # Spatial dimension N = dimension() nx..., _cin, nsample = size(x) @@ -132,18 +135,26 @@ function ((; dimension, kmax, cout, cin, σ)::FourierLayer)(x, params, state) R = selectdim(R, N + 3, 1) .+ im .* selectdim(R, N + 3, 2) # Spatial part (applied point-wise) + # Do matrix multiplication manually for now + # TODO: Make W*x more efficient with Tullio.jl y = reshape(x, nx..., 1, cin, :) y = sum(W .* y; dims = N + 2) y = reshape(y, nx..., cout, :) # Spectral part (applied mode-wise) + # + # Steps: + # # - go to complex-valued spectral space # - chop off high wavenumbers # - multiply with weights mode-wise # - pad with zeros to restore original shape # - go back to real valued spatial representation + # + # We do matrix multiplications manually for now + # TODO: Make R*xhat more efficient with Tullio ikeep = ntuple(Returns(1:kmax+1), N) - nkeep = ntuple(Returns(kmax+1), N) + nkeep = ntuple(Returns(kmax + 1), N) dims = ntuple(identity, N) z = fft(x, dims) z = z[ikeep..., :, :] @@ -155,11 +166,11 @@ function ((; dimension, kmax, cout, cin, σ)::FourierLayer)(x, params, state) # Outer layer: Activation over combined spatial and spectral parts # Note: Even though high wavenumbers are chopped off in `z` and may - # possibly not be present in the input at all, `σ` creates new high wavenumbers. - # High wavenumber functions may thus be represented using a sequence of - # Fourier layers. In this case, the `y`s are the only place where - # information contained in high - # input wavenumbers survive in a Fourier layer. + # possibly not be present in the input at all, `σ` creates new high + # wavenumbers. High wavenumber functions may thus be represented using a + # sequence of Fourier layers. In this case, the `y`s are the only place + # where information contained in high input wavenumbers survive in a + # Fourier layer. v = σ.(y .+ z) # Fourier layer does not modify state From e61fe92d5f2e0d8e33d36c786ffdb214fc2577b4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Fri, 25 Aug 2023 13:21:19 +0200 Subject: [PATCH 025/379] feat(solver): Do not allocate too many setups --- src/solvers/solve_unsteady.jl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/solvers/solve_unsteady.jl b/src/solvers/solve_unsteady.jl index 00ff3e75a..620bf8a5b 100644 --- a/src/solvers/solve_unsteady.jl +++ b/src/solvers/solve_unsteady.jl @@ -56,6 +56,7 @@ function solve_unsteady( inplace = false, processors = (), device = identity, + devsetup = device(setup), ) t_start, t_end = tlims isadaptive = isnothing(Δt) @@ -69,7 +70,7 @@ function solve_unsteady( bc_vectors = get_bc_vectors(setup, t_start) # Move vectors and operators to device (if any). - setup = device(setup) + setup = devsetup V₀ = device(V₀) p₀ = device(p₀) bc_vectors = device(bc_vectors) From aa968159e78157de8d7632ecb6f191c20e55b46a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Fri, 25 Aug 2023 13:22:24 +0200 Subject: [PATCH 026/379] Add fieldnames to save --- src/processors/processors.jl | 43 ++++++++++++++++++++++++++---------- 1 file changed, 31 insertions(+), 12 deletions(-) diff --git a/src/processors/processors.jl b/src/processors/processors.jl index a3c48f539..86585d6d1 100644 --- a/src/processors/processors.jl +++ b/src/processors/processors.jl @@ -56,12 +56,24 @@ step_logger(; nupdate = 1) = processor(function (step_observer) end; nupdate) """ - vtk_writer(setup; nupdate, dir = "output", filename = "solution") + vtk_writer( + setup; + nupdate = 1, + dir = "output", + filename = "solution", + fields = (:velocity, :pressure, :vorticity), + ) Create processor that writes the solution every `nupdate` time steps to a VTK file. The resulting Paraview data collection file is stored in `"\$dir/\$filename.pvd"`. """ -vtk_writer(setup; nupdate = 1, dir = "output", filename = "solution") = processor( +vtk_writer( + setup; + nupdate = 1, + dir = "output", + filename = "solution", + fields = (:velocity, :pressure, :vorticity), +) = processor( function (step_observer) ispath(dir) || mkpath(dir) pvd = paraview_collection(joinpath(dir, filename)) @@ -70,9 +82,6 @@ vtk_writer(setup; nupdate = 1, dir = "output", filename = "solution") = processo (; dimension, xp, yp) = grid (; V, p, t) = $step_observer - V = Array(V) - p = Array(p) - N = dimension() if N == 2 coords = (xp, yp) @@ -81,16 +90,26 @@ vtk_writer(setup; nupdate = 1, dir = "output", filename = "solution") = processo coords = (xp, yp, zp) end + # Move arryas to CPU before writing + V isa Array || (V = Array(V)) + p isa Array || (p = Array(p)) + tformat = replace(string(t), "." => "p") vtk_grid("$(dir)/$(filename)_t=$tformat", coords...) do vtk - vels = get_velocity(setup, V, t) - if N == 2 - # ParaView prefers 3D vectors. Add zero z-component. - wp = zeros(size(vels[1])) - vels = (vels..., wp) + if :velocity ∈ fields + vels = get_velocity(setup, V, t) + if N == 2 + # ParaView prefers 3D vectors. Add zero z-component. + wp = zeros(size(vels[1])) + vels = (vels..., wp) + end + vtk["velocity"] = vels end - vtk["velocity"] = vels - vtk["pressure"] = p + + :pressure ∈ fields && (vtk["pressure"] = p) + :vorticity ∈ fields && + (vtk["vorticity"] = reshape(get_vorticity(setup, V, t), :)) + pvd[t] = vtk end end From f53ffead0ae1d20e96a5b2d1389442b17c23fec8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Fri, 25 Aug 2023 15:38:43 +0200 Subject: [PATCH 027/379] refactor: Put stuff in functions --- Project.toml | 2 + scratch/train_model.jl | 273 +++++++++--------------------- src/IncompressibleNavierStokes.jl | 12 +- src/closures/create_les_data.jl | 129 ++++++++++++++ src/closures/loss.jl | 1 - src/closures/training.jl | 116 +++++++++++++ 6 files changed, 340 insertions(+), 193 deletions(-) create mode 100644 src/closures/create_les_data.jl delete mode 100644 src/closures/loss.jl create mode 100644 src/closures/training.jl diff --git a/Project.toml b/Project.toml index 929e3e87e..c4f778431 100644 --- a/Project.toml +++ b/Project.toml @@ -12,11 +12,13 @@ LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" Lux = "b2108857-7c20-44ae-9111-449ecde12c47" Makie = "ee78f7c6-11fb-53f2-987a-cfe4a2b5a57a" NNlib = "872c559c-99b0-510c-b3b7-b6c96a88d5cd" +Optimisers = "3bd65402-5787-11e9-1adc-39752487f4e2" Printf = "de0858da-6303-5e67-8744-51eddeeeb8d7" Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" SparseArrays = "2f01184e-e22b-5df5-ae63-d93ebab69eaf" Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2" WriteVTK = "64499a7a-5c06-52f2-abe2-ccb03c286192" +Zygote = "e88e6eb3-aa80-5325-afca-941959d7151f" [compat] Adapt = "3" diff --git a/scratch/train_model.jl b/scratch/train_model.jl index 9e1c1d7c8..11ed9d9e7 100644 --- a/scratch/train_model.jl +++ b/scratch/train_model.jl @@ -5,6 +5,11 @@ if isdefined(@__MODULE__, :LanguageServer) #src using .IncompressibleNavierStokes #src end #src +# # Train closure model +# +# Here, we consider a periodic box ``[0, 1]^2``. It is discretized with a +# uniform Cartesian grid with square cells. + using GLMakie using IncompressibleNavierStokes using JLD2 @@ -27,155 +32,61 @@ using CUDA using LuxCUDA device = cu -# Viscosity model -Re = T(2_000) -viscosity_model = LaminarModel(; Re) - -# A 2D grid is a Cartesian product of two vectors -s = 4 -n_les = 128 -n_dns = s * n_les - +# Setup +n = 128 lims = T(0), T(1) -x_dns = LinRange(lims..., n_dns + 1) -y_dns = LinRange(lims..., n_dns + 1) -x_les = x_dns[1:s:end] -y_les = y_dns[1:s:end] - -# Build setup and assemble operators -dns = Setup(x_dns, y_dns; viscosity_model); -les = Setup(x_les, y_les; viscosity_model); - -# Filter -(; KV, Kp) = operator_filter(dns.grid, dns.boundary_conditions, s); - -# Since the grid is uniform and identical for x and y, we may use a specialized -# spectral pressure solver -pressure_solver = SpectralPressureSolver(dns); -pressure_solver_les = SpectralPressureSolver(les); - -filter_saver(setup, KV, Kp, Ωbar; nupdate = 1, bc_vectors = get_bc_vectors(setup, T(0))) = - processor( - function (step_observer) - (; Ω) = setup.grid - KVmom = Ωbar * KV * Diagonal(1 ./ Ω) - T = eltype(setup.grid.x) - _V = fill(zeros(T, 0), 0) - _F = fill(zeros(T, 0), 0) - _FG = fill(zeros(T, 0), 0) - _p = fill(zeros(T, 0), 0) - _t = fill(zero(T), 0) - on(step_observer) do (; V, p, t) - F, = momentum(V, V, p, t, setup; bc_vectors, nopressure = true) - FG, = momentum(V, V, p, t, setup; bc_vectors, nopressure = false) - push!(_V, KV * Array(V)) - push!(_F, KVmom * Array(F)) - push!(_FG, KVmom * Array(FG)) - push!(_p, Kp * Array(p)) - push!(_t, t) - end - (; V = _V, F = _F, FG = _FG, p = _p, t = _t) - end; - nupdate, - ) - -# Time interval, including burn-in time -t_start, t_burn, t_end = T(0.0), T(0.1), T(0.2) - -# Number of time steps -Δt = T(2e-4) -n_t = round(Int, (t_end - t_burn) / Δt) - -# Number of random initial conditions -n_ic = 10 - -# Filtered quantities to store -filtered = (; - V = zeros(T, n_les * n_les * 2, n_t + 1, n_ic), - F = zeros(T, n_les * n_les * 2, n_t + 1, n_ic), - FG = zeros(T, n_les * n_les * 2, n_t + 1, n_ic), - p = zeros(T, n_les * n_les, n_t + 1, n_ic), +viscosity_model = LaminarModel(; Re = T(2_000)) +t_burn = T(0.1) +t_sim = T(0.1) + +# Build LES setup and assemble operators +x = LinRange(lims..., n + 1) +y = LinRange(lims..., n + 1) +setup = Setup(x, y; viscosity_model) + +# Create LES data from DNS +filtered = create_les_data( + T; + viscosity_model, + lims, + n_les = n, + compression = 4, + n_ic = 10, + t_burn, + t_sim, + Δt = T(2e-4), + device, ) +jldsave("output/filtered/filtered.jld2"; filtered) -Base.summarysize(filtered) / 1e6 - -# Iteration processors -processors = ( - filter_saver( - device(dns), - KV, - Kp, - les.grid.Ω; - bc_vectors = device(get_bc_vectors(dns, t_start)), - ), - step_logger(; nupdate = 10), -); - -for i_ic = 1:n_ic - @info "Generating data for IC $i_ic of $n_ic" - - # Initial conditions - V₀, p₀ = random_field(dns; A = T(10_000_000), σ = T(30), s = 5, pressure_solver) - - # Solve burn-in DNS - @info "Burn-in for IC $i_ic of $n_ic" - V, p, outputs = solve_unsteady( - dns, - V₀, - p₀, - (t_start, t_burn); - Δt = T(2e-4), - processors = (step_logger(; nupdate = 10),), - pressure_solver, - inplace = true, - device, - ) - - # Solve DNS and store filtered quantities - @info "Solving DNS for IC $i_ic of $n_ic" - V, p, outputs = solve_unsteady( - dns, - V, - p, - (t_burn, t_end); - Δt = T(2e-4), - processors, - pressure_solver, - inplace = true, - device, - ) - f = outputs[1] - - # Store result for current IC - filtered.V[:, :, i_ic] = stack(f.V) - filtered.F[:, :, i_ic] = stack(f.F) - filtered.FG[:, :, i_ic] = stack(f.FG) - filtered.p[:, :, i_ic] = stack(f.p) -end - -# jldsave("output/filtered/filtered.jld2"; filtered) +# Load previous LES data filtered = load("output/filtered/filtered.jld2", "filtered") size(filtered.V) -plot_vorticity(les, filtered.V[:, end, 1], t_burn) +# Inspect data +plot_vorticity(setup, filtered.V[:, end, 1], T(0)) + +# Uniform periodic grid +pressure_solver_les = SpectralPressureSolver(setup) # Compute commutator errors -bc_vectors = get_bc_vectors(les, t_burn) +_, n_t, n_ic = size(filtered.V) +bc_vectors = get_bc_vectors(setup, T(0)) commutator_error = zero(filtered.F) pbar = filtered.p[:, 1, 1] for i_t = 1:n_t, i_ic = 1:n_ic @info "Computing commutator error for time $i_t of $n_t, IC $i_ic of $n_ic" V = filtered.V[:, i_t, i_ic] F = filtered.F[:, i_t, i_ic] - Fbar, = momentum(V, V, pbar, t_burn, les; bc_vectors, nopressure = true) + Fbar, = momentum(V, V, pbar, T(0), setup; bc_vectors, nopressure = true) commutator_error[:, i_t, i_ic] .= F .- Fbar end norm(commutator_error[:, 1, 1]) / norm(filtered.F[:, 1, 1]) # closure, θ₀ = cnn( -# les, +# setup, # [5, 5, 5], # [2, 8, 8, 2], # [leakyrelu, leakyrelu, identity], @@ -184,7 +95,7 @@ norm(commutator_error[:, 1, 1]) / norm(filtered.F[:, 1, 1]) closure, θ₀ = fno( # Setup - les, + setup, # Cut-off wavenumbers [8, 8, 8], @@ -199,60 +110,44 @@ closure, θ₀ = fno( gelu, ) -length(θ₀) - -loss(x, y, θ) = sum(abs2, closure(x, θ) - y) / sum(abs2, y) - -loss(filtered.V[:, 1, 1], commutator_error[:, 1, 1], θ₀) - -function create_loss(x, y; nuse = size(x, 2), device = identity) - x = reshape(x, size(x, 1), :) - y = reshape(y, size(y, 1), :) - nsample = size(x, 2) - d = ndims(x) - function randloss(θ) - i = Zygote.@ignore sort(shuffle(1:nsample)[1:nuse]) - xuse = Zygote.@ignore device(Array(selectdim(x, d, i))) - yuse = Zygote.@ignore device(Array(selectdim(y, d, i))) - loss(xuse, yuse, θ) - end -end - -randloss = create_loss(filtered.V, commutator_error; nuse = 50, device) - -θ = 5.0f-2 * device(θ₀) - -randloss(θ) - -@time first(gradient(randloss, θ)); +@info "Closure model has $(length(θ₀)) parameters" +# Test data V_test = device(reshape(filtered.V[:, 1:20, 1:2], :, 40)) c_test = device(reshape(commutator_error[:, 1:20, 1:2], :, 40)) +# Prepare training +θ = 5.0f-2 * device(θ₀) opt = Optimisers.setup(Adam(1.0f-2), θ) +callbackstate = Point2f[] +randloss = create_randloss( + mean_squared_error, + closure, + filtered.V, + commutator_error; + nuse = 50, + device, +) -obs = Observable([(0, T(0))]) - -fig = lines(obs; axis = (; title = "Relative prediction error", xlabel = "Iteration")) -hlines!([1.0f0]) -display(fig) - -obs[] = fill((0, T(0)), 0) -j = 0 - -nplot = 10 -niter = 500 -for i = 1:niter - g = first(gradient(randloss, θ)) - opt, θ = Optimisers.update(opt, θ, g) - if i % nplot == 0 - e_test = norm(closure(V_test, θ) - c_test) / norm(c_test) - @info "Iteration $i\trelative test error: $e_test" - _i = (j += nplot) - obs[] = push!(obs[], (_i, e_test)) - autolimits!(fig.axis) - end -end +# Warm-up +randloss(θ); +@time randloss(θ); +first(gradient(randloss, θ)); +@time first(gradient(randloss, θ)); + +# Training +# Note: The states `opt`, `θ`, and `callbackstate` +# will not be overwritten until training is finished. +# This allows for cancelling with "Control-C" should errors explode. +(; opt, θ, callbackstate) = train( + randloss, + opt, + θ; + niter = 500, + ncallback = 10, + callbackstate, + callback = create_callback(closure, V_test, c_test; state = callbackstate), +) # jldsave("output/theta.jld2"; θ = Array(θ)) # θθ = load("output/theta.jld2") @@ -262,38 +157,38 @@ end relative_error(closure(V_test, θ), c_test) -size(filtered.V) - -devles = device(les); +devsetup = device(setup); V_nm, p_nm, outputs_nm = solve_unsteady( - les, + setup, filtered.V[:, 1, 1], filtered.p[:, 1, 1], - (t_burn, t_end); + (T(0), t_sim); Δt = T(2e-4), processors = ( step_logger(; nupdate = 10), - field_plotter(devles; type = heatmap, nupdate = 1), + field_plotter(devsetup; type = heatmap, nupdate = 1), ), pressure_solver = pressure_solver_les, inplace = false, device, + devsetup, ) V_fno, p_fno, outputs_fno = solve_unsteady( - (; les..., closure_model = V -> closure(V, θ)), + (; setup..., closure_model = V -> closure(V, θ)), filtered.V[:, 1, 1], filtered.p[:, 1, 1], - (t_burn, t_end); + (T(0), t_sim); Δt = T(2e-4), processors = ( step_logger(; nupdate = 10), - field_plotter(devles; type = heatmap, nupdate = 1), + field_plotter(devsetup; type = heatmap, nupdate = 1), ), pressure_solver = pressure_solver_les, inplace = false, device, + devsetup, ) V = filtered.V[:, end, 1] @@ -302,9 +197,9 @@ p = filtered.p[:, end, 1] relative_error(V_nm, V) relative_error(V_fno, V) -plot_vorticity(les, V_nm, t_end) -plot_vorticity(les, V_fno, t_end) -plot_vorticity(les, V, t_end) +plot_vorticity(setup, V_nm, t_sim) +plot_vorticity(setup, V_fno, t_sim) +plot_vorticity(setup, V, t_sim) CUDA.memory_status() GC.gc() diff --git a/src/IncompressibleNavierStokes.jl b/src/IncompressibleNavierStokes.jl index 635447880..017406325 100644 --- a/src/IncompressibleNavierStokes.jl +++ b/src/IncompressibleNavierStokes.jl @@ -11,13 +11,15 @@ using FFTW using IterativeSolvers using LinearAlgebra using Lux +using Makie using NNlib +using Optimisers using Printf using Random using SparseArrays using Statistics using WriteVTK: CollectionFile, paraview_collection, vtk_grid, vtk_save -using Makie +using Zygote # Convenience notation const ⊗ = kron @@ -127,7 +129,8 @@ include("postprocess/save_vtk.jl") # Closure models include("closures/cnn.jl") include("closures/fno.jl") -include("closures/loss.jl") +include("closures/training.jl") +include("closures/create_les_data.jl") # Force export SteadyBodyForce @@ -164,7 +167,10 @@ export plot_force, export plotmat # Closure models -export cnn, fno, relative_error +export cnn, fno +export train +export mean_squared_error, relative_error +export create_randloss, create_callback, create_les_data # ODE methods diff --git a/src/closures/create_les_data.jl b/src/closures/create_les_data.jl new file mode 100644 index 000000000..ebead617b --- /dev/null +++ b/src/closures/create_les_data.jl @@ -0,0 +1,129 @@ +_filter_saver( + setup, + KV, + Kp, + Ωbar; + nupdate = 1, + bc_vectors = get_bc_vectors(setup, zero(eltype(KV))), +) = processor( + function (step_observer) + (; Ω) = setup.grid + KVmom = Ωbar * KV * Diagonal(1 ./ Ω) + T = eltype(setup.grid.x) + _V = fill(zeros(T, 0), 0) + _F = fill(zeros(T, 0), 0) + _FG = fill(zeros(T, 0), 0) + _p = fill(zeros(T, 0), 0) + _t = fill(zero(T), 0) + on(step_observer) do (; V, p, t) + F, = momentum(V, V, p, t, setup; bc_vectors, nopressure = true) + FG, = momentum(V, V, p, t, setup; bc_vectors, nopressure = false) + push!(_V, KV * Array(V)) + push!(_F, KVmom * Array(F)) + push!(_FG, KVmom * Array(FG)) + push!(_p, Kp * Array(p)) + push!(_t, t) + end + (; V = _V, F = _F, FG = _FG, p = _p, t = _t) + end; + nupdate, +) + +function create_les_data( + T; + viscosity_model = LaminarModel(; T(2_000)), + lims = (T(0), T(1)), + n_les = 64, + compression = 4, + n_ic = 10, + t_burn = T(0.1), + t_sim = T(0.1), + Δt = T(1e-4), + device = identity, +) + n_dns = compression * n_les + x_dns = LinRange(lims..., n_dns + 1) + y_dns = LinRange(lims..., n_dns + 1) + x_les = x_dns[1:compression:end] + y_les = y_dns[1:compression:end] + + # Build setup and assemble operators + dns = Setup(x_dns, y_dns; viscosity_model) + les = Setup(x_les, y_les; viscosity_model) + + # Filter + (; KV, Kp) = operator_filter(dns.grid, dns.boundary_conditions, compression) + + # Since the grid is uniform and identical for x and y, we may use a specialized + # spectral pressure solver + pressure_solver = SpectralPressureSolver(dns) + + # Number of time steps to save + n_t = round(Int, t_sim / Δt) + + # Filtered quantities to store + filtered = (; + V = zeros(T, n_les * n_les * 2, n_t + 1, n_ic), + F = zeros(T, n_les * n_les * 2, n_t + 1, n_ic), + FG = zeros(T, n_les * n_les * 2, n_t + 1, n_ic), + p = zeros(T, n_les * n_les, n_t + 1, n_ic), + ) + + @info "Generating $(Base.summarysize(filtered) / 1e6) Mb of LES data" + + # Iteration processors + processors = ( + _filter_saver( + device(dns), + KV, + Kp, + les.grid.Ω; + bc_vectors = device(get_bc_vectors(dns, t_start)), + ), + step_logger(; nupdate = 10), + ) + + for i_ic = 1:n_ic + @info "Generating data for IC $i_ic of $n_ic" + + # Initial conditions + V₀, p₀ = random_field(dns; A = T(10_000_000), σ = T(30), s = 5, pressure_solver) + + # Solve burn-in DNS + @info "Burn-in for IC $i_ic of $n_ic" + V, p, outputs = solve_unsteady( + dns, + V₀, + p₀, + (t_start, t_burn); + Δt, + processors = (step_logger(; nupdate = 10),), + pressure_solver, + inplace = true, + device, + ) + + # Solve DNS and store filtered quantities + @info "Solving DNS for IC $i_ic of $n_ic" + V, p, outputs = solve_unsteady( + dns, + V, + p, + (t_burn, t_end); + Δt, + processors, + pressure_solver, + inplace = true, + device, + ) + f = outputs[1] + + # Store result for current IC + filtered.V[:, :, i_ic] = stack(f.V) + filtered.F[:, :, i_ic] = stack(f.F) + filtered.FG[:, :, i_ic] = stack(f.FG) + filtered.p[:, :, i_ic] = stack(f.p) + end + + filtered +end diff --git a/src/closures/loss.jl b/src/closures/loss.jl deleted file mode 100644 index 308011f20..000000000 --- a/src/closures/loss.jl +++ /dev/null @@ -1 +0,0 @@ -relative_error(x, y) = sum(norm(x - y) / norm(y) for (x, y) ∈ zip(eachcol(x), eachcol(y))) / size(x, 2) diff --git a/src/closures/training.jl b/src/closures/training.jl new file mode 100644 index 000000000..a6c26918b --- /dev/null +++ b/src/closures/training.jl @@ -0,0 +1,116 @@ +raw""" + train( + loss, + opt, + θ; + niter = 100, + ncallback = 1, + callback = (i, θ) -> println("Iteration $i of $niter"), + ) + +Update parameters `θ` to minimize `loss(θ)` using the optimiser `opt` for +`niter` iterations. + +Return the a new named tuple `(; opt, θ, callbackstate)` with updated state and +parameters. +""" +function train( + loss, + opt, + θ; + niter = 100, + ncallback = 1, + callback = (state, i, θ) -> println("Iteration $i of $niter"), + callbackstate = nothing, +) + for i = 1:niter + g = first(gradient(loss, θ)) + opt, θ = Optimisers.update(opt, θ, g) + if i % ncallback == 0 + callbackstate = callback(callbackstate, i, θ) + end + end + (; opt, θ, callbackstate) +end + +""" + create_randloss(loss, f, x, y; nuse = size(x, 2), device = identity) + +Create loss function `randloss(θ)` that uses a batch of `nuse` random samples from +`(x, y)` at each evaluation. + +The function `loss` should take inputs like `loss(f, x, y, θ)`. + +The batch is moved to `device` before the loss is evaluated. +""" +function create_randloss(loss, f, x, y; nuse = size(x, 2), device = identity) + x = reshape(x, size(x, 1), :) + y = reshape(y, size(y, 1), :) + nsample = size(x, 2) + d = ndims(x) + function randloss(θ) + i = Zygote.@ignore sort(shuffle(1:nsample)[1:nuse]) + xuse = Zygote.@ignore device(Array(selectdim(x, d, i))) + yuse = Zygote.@ignore device(Array(selectdim(y, d, i))) + loss(f, xuse, yuse, θ) + end +end + +""" + mean_squared_error(f, x, y, θ; normalize = y -> sum(abs2, y)) + +Compute MSE between `f(x, θ)` and `y`. + +The MSE is further divided by `normalize(y)`. +""" +mean_squared_error(f, x, y, θ; normalize = y -> sum(abs2, y)) = + sum(abs2, f(x, θ) - y) / normalize(y) + +""" + relative_error(x, y) + +Compute average column relative error between matrices `x` and `y`. +""" +relative_error(x, y) = + sum(norm(x - y) / norm(y) for (x, y) ∈ zip(eachcol(x), eachcol(y))) / size(x, 2) + +""" + create_callback( + f, + x, + y; + state = Point2f[], + display_each_iteration = false, + ) + +Create convergence plot for relative error between `f(x, θ)` and `y`. +At each callback, plot is updated and current error is printed. + +If `state` is nonempty, it also plots previous convergence. + +If not using interactive GLMakie window, set `display_each_iteration` to +`true`. +""" +function create_callback( + f, + x, + y; + state = Point2f[], + display_each_iteration = false, +) + istart = isempty(state) ? 0 : Int(first(state[end])) + obs = Observable(state) + fig = lines(obs; axis = (; title = "Relative prediction error", xlabel = "Iteration")) + hlines!([1.0f0]; linestyle = :dash) + display(fig) + function callback(state, i, θ) + e = norm(f(x, θ) - y) / norm(y) + @info "Iteration $i \trelative error: $e_test" + state = push(state, Point2f(istart + i, e_test)) + obs[] = state + autolimits!(fig.axis) + display_each_iteration && display(fig) + state + end +end + From 64ae7e7b95c76e7661b5465640e47f3af89cddc5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Fri, 25 Aug 2023 15:40:02 +0200 Subject: [PATCH 028/379] rename --- scratch/train_model.jl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scratch/train_model.jl b/scratch/train_model.jl index 11ed9d9e7..9b69df4ea 100644 --- a/scratch/train_model.jl +++ b/scratch/train_model.jl @@ -68,7 +68,7 @@ size(filtered.V) plot_vorticity(setup, filtered.V[:, end, 1], T(0)) # Uniform periodic grid -pressure_solver_les = SpectralPressureSolver(setup) +pressure_solver = SpectralPressureSolver(setup) # Compute commutator errors _, n_t, n_ic = size(filtered.V) @@ -169,7 +169,7 @@ V_nm, p_nm, outputs_nm = solve_unsteady( step_logger(; nupdate = 10), field_plotter(devsetup; type = heatmap, nupdate = 1), ), - pressure_solver = pressure_solver_les, + pressure_solver, inplace = false, device, devsetup, @@ -185,7 +185,7 @@ V_fno, p_fno, outputs_fno = solve_unsteady( step_logger(; nupdate = 10), field_plotter(devsetup; type = heatmap, nupdate = 1), ), - pressure_solver = pressure_solver_les, + pressure_solver, inplace = false, device, devsetup, From 28d2a0557cd8e6da21a23b6d04feead17a482c1d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Fri, 25 Aug 2023 17:01:52 +0200 Subject: [PATCH 029/379] fix(data_gen) --- src/closures/create_les_data.jl | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/src/closures/create_les_data.jl b/src/closures/create_les_data.jl index ebead617b..f32910ee8 100644 --- a/src/closures/create_les_data.jl +++ b/src/closures/create_les_data.jl @@ -8,7 +8,7 @@ _filter_saver( ) = processor( function (step_observer) (; Ω) = setup.grid - KVmom = Ωbar * KV * Diagonal(1 ./ Ω) + KVmom = Diagonal(Ωbar) * (KV * Diagonal(1 ./ Ω)) T = eltype(setup.grid.x) _V = fill(zeros(T, 0), 0) _F = fill(zeros(T, 0), 0) @@ -18,10 +18,10 @@ _filter_saver( on(step_observer) do (; V, p, t) F, = momentum(V, V, p, t, setup; bc_vectors, nopressure = true) FG, = momentum(V, V, p, t, setup; bc_vectors, nopressure = false) - push!(_V, KV * Array(V)) - push!(_F, KVmom * Array(F)) - push!(_FG, KVmom * Array(FG)) - push!(_p, Kp * Array(p)) + push!(_V, Array(KV * V)) + push!(_F, Array(KVmom * F)) + push!(_FG, Array(KVmom * FG)) + push!(_p, Array(Kp * p)) push!(_t, t) end (; V = _V, F = _F, FG = _FG, p = _p, t = _t) @@ -31,7 +31,7 @@ _filter_saver( function create_les_data( T; - viscosity_model = LaminarModel(; T(2_000)), + viscosity_model = LaminarModel(; Re = T(2_000)), lims = (T(0), T(1)), n_les = 64, compression = 4, @@ -75,10 +75,10 @@ function create_les_data( processors = ( _filter_saver( device(dns), - KV, - Kp, - les.grid.Ω; - bc_vectors = device(get_bc_vectors(dns, t_start)), + device(KV), + device(Kp), + device(les.grid.Ω); + bc_vectors = device(get_bc_vectors(dns, T(0))), ), step_logger(; nupdate = 10), ) @@ -95,7 +95,7 @@ function create_les_data( dns, V₀, p₀, - (t_start, t_burn); + (T(0), t_burn); Δt, processors = (step_logger(; nupdate = 10),), pressure_solver, @@ -109,7 +109,7 @@ function create_les_data( dns, V, p, - (t_burn, t_end); + (T(0), t_sim); Δt, processors, pressure_solver, From e61f2f2335dd3e320669fd02d92043385597e2c1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Fri, 25 Aug 2023 17:03:49 +0200 Subject: [PATCH 030/379] fix: Method ambiguity --- src/closures/fno.jl | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/closures/fno.jl b/src/closures/fno.jl index 7470f69f7..b2cf68853 100644 --- a/src/closures/fno.jl +++ b/src/closures/fno.jl @@ -102,7 +102,10 @@ FourierLayer( init_weight = glorot_uniform, ) = FourierLayer(dimension, kmax, first(ch), last(ch), σ, init_weight) -Lux.initialparameters(rng, (; dimension, kmax, cin, cout, init_weight)::FourierLayer) = (; +Lux.initialparameters( + rng::AbstractRNG, + (; dimension, kmax, cin, cout, init_weight)::FourierLayer, +) = (; spatial_weight = init_weight(rng, cout, cin), spectral_weights = init_weight(rng, fill(kmax + 1, dimension())..., cout, cin, 2), ) From 3b2dc34c5756344f58f205e30d9e095079ab00db Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Fri, 25 Aug 2023 17:33:19 +0200 Subject: [PATCH 031/379] fix: Correct push to state --- src/closures/training.jl | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/closures/training.jl b/src/closures/training.jl index a6c26918b..7fc62adba 100644 --- a/src/closures/training.jl +++ b/src/closures/training.jl @@ -99,14 +99,15 @@ function create_callback( display_each_iteration = false, ) istart = isempty(state) ? 0 : Int(first(state[end])) - obs = Observable(state) + obs = Observable([Point2f(0, 0)]) fig = lines(obs; axis = (; title = "Relative prediction error", xlabel = "Iteration")) hlines!([1.0f0]; linestyle = :dash) + obs[] = state display(fig) function callback(state, i, θ) e = norm(f(x, θ) - y) / norm(y) - @info "Iteration $i \trelative error: $e_test" - state = push(state, Point2f(istart + i, e_test)) + @info "Iteration $i \trelative error: $e" + state = push!(copy(state), Point2f(istart + i, e)) obs[] = state autolimits!(fig.axis) display_each_iteration && display(fig) From 5e981eaed9418bacd1b65eab7e6315717be3f09c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Wed, 30 Aug 2023 13:05:14 +0200 Subject: [PATCH 032/379] Rename variables --- src/closures/create_les_data.jl | 52 ++++++++++++++++----------------- 1 file changed, 26 insertions(+), 26 deletions(-) diff --git a/src/closures/create_les_data.jl b/src/closures/create_les_data.jl index f32910ee8..de3c89e94 100644 --- a/src/closures/create_les_data.jl +++ b/src/closures/create_les_data.jl @@ -33,23 +33,23 @@ function create_les_data( T; viscosity_model = LaminarModel(; Re = T(2_000)), lims = (T(0), T(1)), - n_les = 64, + nles = 64, compression = 4, - n_ic = 10, - t_burn = T(0.1), - t_sim = T(0.1), + nsim = 10, + tburn = T(0.1), + tsim = T(0.1), Δt = T(1e-4), device = identity, ) - n_dns = compression * n_les - x_dns = LinRange(lims..., n_dns + 1) - y_dns = LinRange(lims..., n_dns + 1) - x_les = x_dns[1:compression:end] - y_les = y_dns[1:compression:end] + ndns = compression * nles + xdns = LinRange(lims..., ndns + 1) + ydns = LinRange(lims..., ndns + 1) + xles = xdns[1:compression:end] + yles = ydns[1:compression:end] # Build setup and assemble operators - dns = Setup(x_dns, y_dns; viscosity_model) - les = Setup(x_les, y_les; viscosity_model) + dns = Setup(xdns, ydns; viscosity_model) + les = Setup(xles, yles; viscosity_model) # Filter (; KV, Kp) = operator_filter(dns.grid, dns.boundary_conditions, compression) @@ -59,14 +59,14 @@ function create_les_data( pressure_solver = SpectralPressureSolver(dns) # Number of time steps to save - n_t = round(Int, t_sim / Δt) + nt = round(Int, tsim / Δt) # Filtered quantities to store filtered = (; - V = zeros(T, n_les * n_les * 2, n_t + 1, n_ic), - F = zeros(T, n_les * n_les * 2, n_t + 1, n_ic), - FG = zeros(T, n_les * n_les * 2, n_t + 1, n_ic), - p = zeros(T, n_les * n_les, n_t + 1, n_ic), + V = zeros(T, nles * nles * 2, nt + 1, nsim), + F = zeros(T, nles * nles * 2, nt + 1, nsim), + FG = zeros(T, nles * nles * 2, nt + 1, nsim), + p = zeros(T, nles * nles, nt + 1, nsim), ) @info "Generating $(Base.summarysize(filtered) / 1e6) Mb of LES data" @@ -83,19 +83,19 @@ function create_les_data( step_logger(; nupdate = 10), ) - for i_ic = 1:n_ic - @info "Generating data for IC $i_ic of $n_ic" + for isim = 1:nsim + @info "Generating data for simulation $isim of $nsim" # Initial conditions V₀, p₀ = random_field(dns; A = T(10_000_000), σ = T(30), s = 5, pressure_solver) # Solve burn-in DNS - @info "Burn-in for IC $i_ic of $n_ic" + @info "Burn-in for IC $isim of $nsim" V, p, outputs = solve_unsteady( dns, V₀, p₀, - (T(0), t_burn); + (T(0), tburn); Δt, processors = (step_logger(; nupdate = 10),), pressure_solver, @@ -104,12 +104,12 @@ function create_les_data( ) # Solve DNS and store filtered quantities - @info "Solving DNS for IC $i_ic of $n_ic" + @info "Solving DNS for IC $isim of $nsim" V, p, outputs = solve_unsteady( dns, V, p, - (T(0), t_sim); + (T(0), tsim); Δt, processors, pressure_solver, @@ -119,10 +119,10 @@ function create_les_data( f = outputs[1] # Store result for current IC - filtered.V[:, :, i_ic] = stack(f.V) - filtered.F[:, :, i_ic] = stack(f.F) - filtered.FG[:, :, i_ic] = stack(f.FG) - filtered.p[:, :, i_ic] = stack(f.p) + filtered.V[:, :, isim] = stack(f.V) + filtered.F[:, :, isim] = stack(f.F) + filtered.FG[:, :, isim] = stack(f.FG) + filtered.p[:, :, isim] = stack(f.p) end filtered From ad7c664b86b3d95aa132df69d29d3204f1061cf2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Thu, 31 Aug 2023 13:59:42 +0200 Subject: [PATCH 033/379] Add commutator errors to data --- src/closures/create_les_data.jl | 93 ++++++++++++++++++++++++--------- 1 file changed, 69 insertions(+), 24 deletions(-) diff --git a/src/closures/create_les_data.jl b/src/closures/create_les_data.jl index de3c89e94..e6003e3f4 100644 --- a/src/closures/create_les_data.jl +++ b/src/closures/create_les_data.jl @@ -1,30 +1,69 @@ _filter_saver( - setup, + dns, + les, KV, - Kp, - Ωbar; + Kp; nupdate = 1, - bc_vectors = get_bc_vectors(setup, zero(eltype(KV))), + bc_vectors_dns = get_bc_vectors(dns, zero(eltype(KV))), + bc_vectors_les = get_bc_vectors(les, zero(eltype(KV))), ) = processor( - function (step_observer) - (; Ω) = setup.grid + function (state) + (; Ω, x) = dns.grid + Ωbar = les.grid.Ω + T = eltype(x) KVmom = Diagonal(Ωbar) * (KV * Diagonal(1 ./ Ω)) - T = eltype(setup.grid.x) + _t = fill(zero(T), 0) _V = fill(zeros(T, 0), 0) + _p = fill(zeros(T, 0), 0) _F = fill(zeros(T, 0), 0) _FG = fill(zeros(T, 0), 0) - _p = fill(zeros(T, 0), 0) - _t = fill(zero(T), 0) - on(step_observer) do (; V, p, t) - F, = momentum(V, V, p, t, setup; bc_vectors, nopressure = true) - FG, = momentum(V, V, p, t, setup; bc_vectors, nopressure = false) - push!(_V, Array(KV * V)) - push!(_F, Array(KVmom * F)) - push!(_FG, Array(KVmom * FG)) - push!(_p, Array(Kp * p)) + _cF = fill(zeros(T, 0), 0) + _cFG = fill(zeros(T, 0), 0) + on(state) do (; V, p, t) + Vbar = KV * V + pbar = Kp * p + F, = momentum(V, V, p, t, dns; bc_vectors = bc_vectors_dns, nopressure = true) + FG, = momentum( + V, + V, + p, + t, + dns; + bc_vectors = bc_vectors_dns, + nopressure = false, + ) + Fbar = KVmom * F + FGbar = KVmom * FG + FVbar, = momentum( + Vbar, + Vbar, + pbar, + t, + les; + bc_vectors = bc_vectors_les, + nopressure = true, + ) + FGVbar, = momentum( + Vbar, + Vbar, + pbar, + t, + les; + bc_vectors = bc_vectors_les, + nopressure = false, + ) + cF = Fbar - FVbar + cFG = FGbar - FGVbar push!(_t, t) + push!(_V, Array(Vbar)) + push!(_p, Array(pbar)) + push!(_F, Array(Fbar)) + push!(_FG, Array(FGbar)) + push!(_cF, Array(cF)) + push!(_cFG, Array(cFG)) end - (; V = _V, F = _F, FG = _FG, p = _p, t = _t) + state[] = state[] + (; t = _t, V = _V, p = _p, F = _F, FG = _FG, cF = _cF, cFG = _cFG) end; nupdate, ) @@ -60,13 +99,16 @@ function create_les_data( # Number of time steps to save nt = round(Int, tsim / Δt) + Δt = tsim / nt # Filtered quantities to store filtered = (; V = zeros(T, nles * nles * 2, nt + 1, nsim), + p = zeros(T, nles * nles, nt + 1, nsim), F = zeros(T, nles * nles * 2, nt + 1, nsim), FG = zeros(T, nles * nles * 2, nt + 1, nsim), - p = zeros(T, nles * nles, nt + 1, nsim), + cF = zeros(T, nles * nles * 2, nt + 1, nsim), + cFG = zeros(T, nles * nles * 2, nt + 1, nsim), ) @info "Generating $(Base.summarysize(filtered) / 1e6) Mb of LES data" @@ -75,10 +117,11 @@ function create_les_data( processors = ( _filter_saver( device(dns), + device(les), device(KV), - device(Kp), - device(les.grid.Ω); - bc_vectors = device(get_bc_vectors(dns, T(0))), + device(Kp); + bc_vectors_dns = device(get_bc_vectors(dns, T(0))), + bc_vectors_les = device(get_bc_vectors(les, T(0))), ), step_logger(; nupdate = 10), ) @@ -90,7 +133,7 @@ function create_les_data( V₀, p₀ = random_field(dns; A = T(10_000_000), σ = T(30), s = 5, pressure_solver) # Solve burn-in DNS - @info "Burn-in for IC $isim of $nsim" + @info "Burn-in for simulation $isim of $nsim" V, p, outputs = solve_unsteady( dns, V₀, @@ -104,7 +147,7 @@ function create_les_data( ) # Solve DNS and store filtered quantities - @info "Solving DNS for IC $isim of $nsim" + @info "Solving DNS for simulation $isim of $nsim" V, p, outputs = solve_unsteady( dns, V, @@ -120,9 +163,11 @@ function create_les_data( # Store result for current IC filtered.V[:, :, isim] = stack(f.V) + filtered.p[:, :, isim] = stack(f.p) filtered.F[:, :, isim] = stack(f.F) filtered.FG[:, :, isim] = stack(f.FG) - filtered.p[:, :, isim] = stack(f.p) + filtered.cF[:, :, isim] = stack(f.cF) + filtered.cFG[:, :, isim] = stack(f.cFG) end filtered From d4f6d6f89accc8b35757214470588767392b8de8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Thu, 31 Aug 2023 14:02:37 +0200 Subject: [PATCH 034/379] Add regularization --- src/closures/training.jl | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/src/closures/training.jl b/src/closures/training.jl index 7fc62adba..b82a3e630 100644 --- a/src/closures/training.jl +++ b/src/closures/training.jl @@ -57,14 +57,14 @@ function create_randloss(loss, f, x, y; nuse = size(x, 2), device = identity) end """ - mean_squared_error(f, x, y, θ; normalize = y -> sum(abs2, y)) + mean_squared_error(f, x, y, θ; normalize = y -> sum(abs2, y), λ = sqrt(eps(eltype(x)))) Compute MSE between `f(x, θ)` and `y`. The MSE is further divided by `normalize(y)`. """ -mean_squared_error(f, x, y, θ; normalize = y -> sum(abs2, y)) = - sum(abs2, f(x, θ) - y) / normalize(y) +mean_squared_error(f, x, y, θ; normalize = y -> sum(abs2, y), λ = sqrt(eps(eltype(x)))) = + sum(abs2, f(x, θ) - y) / normalize(y) + λ * sum(abs2, θ) / length(θ) """ relative_error(x, y) @@ -91,13 +91,7 @@ If `state` is nonempty, it also plots previous convergence. If not using interactive GLMakie window, set `display_each_iteration` to `true`. """ -function create_callback( - f, - x, - y; - state = Point2f[], - display_each_iteration = false, -) +function create_callback(f, x, y; state = Point2f[], display_each_iteration = false) istart = isempty(state) ? 0 : Int(first(state[end])) obs = Observable([Point2f(0, 0)]) fig = lines(obs; axis = (; title = "Relative prediction error", xlabel = "Iteration")) @@ -114,4 +108,3 @@ function create_callback( state end end - From e6a68d6783fbf97b3d2f053ec1a405d05e61c3b3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Thu, 31 Aug 2023 14:10:27 +0200 Subject: [PATCH 035/379] Add missing paramter --- src/closures/fno.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/closures/fno.jl b/src/closures/fno.jl index b2cf68853..4f3cb6425 100644 --- a/src/closures/fno.jl +++ b/src/closures/fno.jl @@ -110,7 +110,7 @@ Lux.initialparameters( spectral_weights = init_weight(rng, fill(kmax + 1, dimension())..., cout, cin, 2), ) Lux.initialstates(::AbstractRNG, ::FourierLayer) = (;) -Lux.parameterlength((; kmax, cin, cout)::FourierLayer) = +Lux.parameterlength((; dimension, kmax, cin, cout)::FourierLayer) = cout * cin + (kmax + 1)^dimension() * 2 * cout * cin Lux.statelength(::FourierLayer) = 0 From 247c12c7ad725c669e129123192f6ba10f1f674f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Wed, 6 Sep 2023 10:51:26 +0200 Subject: [PATCH 036/379] Add force --- scratch/train_model.jl | 228 ++++++++++++++++++++++++-------- src/closures/create_les_data.jl | 77 ++++++++--- 2 files changed, 231 insertions(+), 74 deletions(-) diff --git a/scratch/train_model.jl b/scratch/train_model.jl index 9b69df4ea..8c5b50028 100644 --- a/scratch/train_model.jl +++ b/scratch/train_model.jl @@ -35,96 +35,130 @@ device = cu # Setup n = 128 lims = T(0), T(1) -viscosity_model = LaminarModel(; Re = T(2_000)) -t_burn = T(0.1) -t_sim = T(0.1) +viscosity_model = LaminarModel(; Re = T(6_000)) +tburn = T(0.05) +tsim = T(0.05) # Build LES setup and assemble operators x = LinRange(lims..., n + 1) y = LinRange(lims..., n + 1) -setup = Setup(x, y; viscosity_model) +setup = Setup(x, y; viscosity_model); + +# Number of simulations +ntrain = 10 +nvalid = 2 +ntest = 5 # Create LES data from DNS -filtered = create_les_data( - T; +params = (; viscosity_model, lims, - n_les = n, + nles = n, compression = 4, - n_ic = 10, - t_burn, - t_sim, - Δt = T(2e-4), + tburn, + tsim, + Δt = T(1e-4), device, ) -jldsave("output/filtered/filtered.jld2"; filtered) +data_train = create_les_data(T; params..., nsim = ntrain) +data_valid = create_les_data(T; params..., nsim = nvalid) +data_test = create_les_data(T; params..., nsim = ntest) + +# jldsave("output/filtered/data.jld2"; data_train, data_valid, data_test) # Load previous LES data -filtered = load("output/filtered/filtered.jld2", "filtered") +data_train, data_valid, data_test = load("output/filtered/data.jld2", "data_train", "data_valid", "data_test") + +nt = size(data_train.V, 2) - 1 -size(filtered.V) +size(data_train.V) +size(data_valid.V) +size(data_test.V) # Inspect data -plot_vorticity(setup, filtered.V[:, end, 1], T(0)) +plot_vorticity(setup, data_valid.V[:, 1, 1], T(0)) +plot_vorticity(setup, data_valid.V[:, end, 1], T(0)) +norm(data_valid.cF[:, 1, 1]) / norm(data_valid.F[:, 1, 1]) +norm(data_valid.cF[:, end, 1]) / norm(data_valid.F[:, end, 1]) # Uniform periodic grid -pressure_solver = SpectralPressureSolver(setup) - -# Compute commutator errors -_, n_t, n_ic = size(filtered.V) -bc_vectors = get_bc_vectors(setup, T(0)) -commutator_error = zero(filtered.F) -pbar = filtered.p[:, 1, 1] -for i_t = 1:n_t, i_ic = 1:n_ic - @info "Computing commutator error for time $i_t of $n_t, IC $i_ic of $n_ic" - V = filtered.V[:, i_t, i_ic] - F = filtered.F[:, i_t, i_ic] - Fbar, = momentum(V, V, pbar, T(0), setup; bc_vectors, nopressure = true) - commutator_error[:, i_t, i_ic] .= F .- Fbar +pressure_solver = SpectralPressureSolver(setup); + +q = data_valid.V +# q = data_train.FG +q = q[:, :, 1] +q = selectdim(reshape(q, n, n, 2, :), 3, 1) + +qc = reshape(selectdim(data_valid.cF, 3, 2), n, n, :) + +obs = Observable(randn(T, 1, 1)) +# obs = Observable(selectdim(q, 3, 1)) +# obs = Observable([selectdim(q, 3, 1) selectdim(qc, 3, 1)]) +fig = heatmap(obs) +fig + +# for snap in eachslice(q; dims = 3) +for (s1, s2) in zip(eachslice(q; dims = 3), eachslice(qc; dims = 3)) + obs[] = s1 + # obs[] = [s1 s2] + autolimits!(fig.axis) + sleep(0.005) end -norm(commutator_error[:, 1, 1]) / norm(filtered.F[:, 1, 1]) +heatmap(selectdim(reshape(data_valid.force[:, 1], n, n, 2), 3, 1)) + +fx, fy = eachslice(reshape(data_valid.force[:, 1], n, n, 2); dims = 3) +heatmap(fx) +arrows(x, y, fx, fy; lengthscale = 1.0f0) # closure, θ₀ = cnn( # setup, +# +# # Radius # [5, 5, 5], -# [2, 8, 8, 2], +# +# # Channels +# [2, 64, 64, 2], +# +# # Activations # [leakyrelu, leakyrelu, identity], +# +# # Bias # [true, true, false]; # ) closure, θ₀ = fno( - # Setup setup, # Cut-off wavenumbers - [8, 8, 8], + [32, 32, 32, 32], # Channel sizes - [16, 8, 8], + [24, 12, 8, 8], - # Fourier activations - [gelu, gelu, identity], + # Fourier layer activations + [gelu, gelu, gelu, identity], # Dense activation gelu, -) +); @info "Closure model has $(length(θ₀)) parameters" # Test data -V_test = device(reshape(filtered.V[:, 1:20, 1:2], :, 40)) -c_test = device(reshape(commutator_error[:, 1:20, 1:2], :, 40)) +V_test = device(reshape(data_test.V[:, 1:20, 1:2], :, 40)) +c_test = device(reshape(data_test.cF[:, 1:20, 1:2], :, 40)) # Prepare training -θ = 5.0f-2 * device(θ₀) -opt = Optimisers.setup(Adam(1.0f-2), θ) +θ = 1.0f-1 * device(θ₀) +# θ = device(θ₀) +opt = Optimisers.setup(Adam(1.0f-3), θ) callbackstate = Point2f[] randloss = create_randloss( mean_squared_error, closure, - filtered.V, - commutator_error; + data_train.V, + data_train.cF; nuse = 50, device, ) @@ -143,63 +177,141 @@ first(gradient(randloss, θ)); randloss, opt, θ; - niter = 500, + niter = 1000, ncallback = 10, callbackstate, callback = create_callback(closure, V_test, c_test; state = callbackstate), ) +GC.gc() +CUDA.reclaim() +Array(θ) + +# # Save trained parameters # jldsave("output/theta.jld2"; θ = Array(θ)) + +# # Load trained parameters # θθ = load("output/theta.jld2") # θθ = θθ["θ"] # θθ = cu(θθ) # θ .= θθ +relative_error(closure(device(data_train.V[:, 1, :]), θ), device(data_train.cF[:, 1, :])) +relative_error(closure(device(data_train.V[:, end, :]), θ), device(data_train.cF[:, end, :])) relative_error(closure(V_test, θ), c_test) -devsetup = device(setup); +function energy_history(setup, state) + (; Ωp) = setup.grid + points = Point2f[] + on(state) do (; V, p, t) + V = Array(V) + vels = get_velocity(setup, V, t) + vels = reshape.(vels, :) + E = sum(vel -> sum(@. Ωp * vel^2), vels) + push!(points, Point2f(t, E)) + end + points +end +energy_history_writer(setup; nupdate = 1, kwargs...) = + processor(state -> energy_history(setup, state; kwargs...); nupdate) + +devsetup = device(setup); V_nm, p_nm, outputs_nm = solve_unsteady( setup, - filtered.V[:, 1, 1], - filtered.p[:, 1, 1], - (T(0), t_sim); + data_train.V[:, 1, 1], + data_train.p[:, 1, 1], + (T(0), tsim); Δt = T(2e-4), processors = ( - step_logger(; nupdate = 10), field_plotter(devsetup; type = heatmap, nupdate = 1), + energy_history_writer(setup), + step_logger(; nupdate = 10), ), pressure_solver, inplace = false, device, devsetup, ) +ehist_nm = outputs_nm[2] +setup_fno = (; setup..., closure_model = V -> closure(V, θ)) +devsetup = device(setup_fno); V_fno, p_fno, outputs_fno = solve_unsteady( - (; setup..., closure_model = V -> closure(V, θ)), - filtered.V[:, 1, 1], - filtered.p[:, 1, 1], - (T(0), t_sim); + setup_fno, + data_train.V[:, 1, 1], + data_train.p[:, 1, 1], + (T(0), tsim); Δt = T(2e-4), processors = ( - step_logger(; nupdate = 10), field_plotter(devsetup; type = heatmap, nupdate = 1), + energy_history_writer(setup), + step_logger(; nupdate = 10), ), pressure_solver, inplace = false, device, devsetup, ) +ehist_fno = outputs_fno[2] + +state = Observable((; V = data_train.V[:, 1, 1], p = data_train.p[:, 1, 1], t = T(0))) +ehist = energy_history(setup, state) +for i = 2:nt+1 + t = (i - 1) / T(nt - 1) * tsim + V = data_train.V[:, i, 1] + p = data_train.p[:, i, 1] + state[] = (; V, p, t) +end +ehist + +fig = Figure() +ax = Axis(fig[1, 1]; xlabel = "t", ylabel = "Kinetic energy") +lines!(ax, ehist; label = "Reference") +lines!(ax, ehist_nm; label = "No closure") +lines!(ax, ehist_fno; label = "FNO") +axislegend(ax) +fig -V = filtered.V[:, end, 1] -p = filtered.p[:, end, 1] +save("output/train/energy.png", fig) + +V = data_train.V[:, end, 1] +p = data_train.p[:, end, 1] relative_error(V_nm, V) relative_error(V_fno, V) -plot_vorticity(setup, V_nm, t_sim) -plot_vorticity(setup, V_fno, t_sim) -plot_vorticity(setup, V, t_sim) +box = [ + Point2f(0.72, 0.42), + Point2f(0.81, 0.42), + Point2f(0.81, 0.51), + Point2f(0.72, 0.51), + Point2f(0.72, 0.42), +] + +plot_vorticity(setup, V, tsim) +lines!(box; color = :red) +current_figure() + +save("output/train/vorticity.png", current_figure()) + +plot_vorticity(setup, V_nm, tsim) +lines!(box; color = :red) +current_figure() + +save("output/train/vorticity_nm.png", current_figure()) + +plot_vorticity(setup, V_fno, tsim) +lines!(box; color = :red) +current_figure() + +save("output/train/vorticity_fno.png", current_figure()) + +heatmap(vcat( + selectdim(reshape(V_nm, n, n, 2), 3, 1), + # selectdim(reshape(V_fno, n, n, 2), 3, 1), + selectdim(reshape(V, n, n, 2), 3, 1), +)) CUDA.memory_status() GC.gc() diff --git a/src/closures/create_les_data.jl b/src/closures/create_les_data.jl index e6003e3f4..17a714f0a 100644 --- a/src/closures/create_les_data.jl +++ b/src/closures/create_les_data.jl @@ -1,3 +1,37 @@ +# Body force +function gaussian_force( + x, + y; + σ = eltype(x)(0.05), + A = eltype(x)(0.002), + rng = Random.default_rng(), +) + T = eltype(x) + Lx = x[end] - x[1] + Ly = y[end] - y[1] + xc = x[1] + rand(rng, T) * Lx + yc = y[1] + rand(rng, T) * Ly + σx = σ * Lx + σy = σ * Ly + ϕ = T(2π) * rand(rng, T) + f = sum( + [ + A * exp(-(x - xc - lx)^2 / 2σx^2 - (y - yc - ly)^2 / 2σy^2) for + y ∈ y[2:end], x ∈ x[2:end], ly ∈ (-Ly, T(0), Ly), lx ∈ (-Lx, T(0), Lx) + ]; + dims = (3, 4), + )[ + :, + :, + 1, + 1, + ] + force = cat(sin(ϕ) * f, cos(ϕ) * f; dims = 3) + force = reshape(force, :) + force = force .- sum(force) / length(force) + force +end + _filter_saver( dns, les, @@ -92,6 +126,7 @@ function create_les_data( # Filter (; KV, Kp) = operator_filter(dns.grid, dns.boundary_conditions, compression) + KVmom = Diagonal(les.grid.Ω) * (KV * Diagonal(1 ./ dns.grid.Ω)) # Since the grid is uniform and identical for x and y, we may use a specialized # spectral pressure solver @@ -109,33 +144,32 @@ function create_les_data( FG = zeros(T, nles * nles * 2, nt + 1, nsim), cF = zeros(T, nles * nles * 2, nt + 1, nsim), cFG = zeros(T, nles * nles * 2, nt + 1, nsim), + force = zeros(T, nles * nles * 2, nsim), ) @info "Generating $(Base.summarysize(filtered) / 1e6) Mb of LES data" - # Iteration processors - processors = ( - _filter_saver( - device(dns), - device(les), - device(KV), - device(Kp); - bc_vectors_dns = device(get_bc_vectors(dns, T(0))), - bc_vectors_les = device(get_bc_vectors(les, T(0))), - ), - step_logger(; nupdate = 10), - ) - for isim = 1:nsim @info "Generating data for simulation $isim of $nsim" # Initial conditions V₀, p₀ = random_field(dns; A = T(10_000_000), σ = T(30), s = 5, pressure_solver) + # Random body force + force_dns = gaussian_force(xdns, ydns) + + gaussian_force(xdns, ydns) + + # gaussian_force(xdns, ydns) + + # gaussian_force(xdns, ydns) + + gaussian_force(xdns, ydns) + force_les = KVmom * force_dns + + _dns = (; dns..., force = force_dns) + _les = (; les..., force = force_les) + # Solve burn-in DNS @info "Burn-in for simulation $isim of $nsim" V, p, outputs = solve_unsteady( - dns, + _dns, V₀, p₀, (T(0), tburn); @@ -149,12 +183,22 @@ function create_les_data( # Solve DNS and store filtered quantities @info "Solving DNS for simulation $isim of $nsim" V, p, outputs = solve_unsteady( - dns, + _dns, V, p, (T(0), tsim); Δt, - processors, + processors = ( + _filter_saver( + device(_dns), + device(_les), + device(KV), + device(Kp); + bc_vectors_dns = device(get_bc_vectors(_dns, T(0))), + bc_vectors_les = device(get_bc_vectors(_les, T(0))), + ), + step_logger(; nupdate = 10), + ), pressure_solver, inplace = true, device, @@ -168,6 +212,7 @@ function create_les_data( filtered.FG[:, :, isim] = stack(f.FG) filtered.cF[:, :, isim] = stack(f.cF) filtered.cFG[:, :, isim] = stack(f.cFG) + filtered.force[:, isim] = force_les end filtered From d57404565b92431f1f107c05053d8bea627d8717 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Wed, 6 Sep 2023 13:36:20 +0200 Subject: [PATCH 037/379] refactor: Simplify Re --- README.md | 6 +++--- examples/Actuator2D.jl | 4 ++-- examples/Actuator3D.jl | 4 ++-- examples/BackwardFacingStep2D.jl | 4 ++-- examples/BackwardFacingStep3D.jl | 4 ++-- examples/DecayingTurbulence2D.jl | 4 ++-- examples/DecayingTurbulence3D.jl | 4 ++-- examples/LidDrivenCavity2D.jl | 14 +++--------- examples/LidDrivenCavity3D.jl | 4 ++-- examples/PlanarMixing2D.jl | 4 ++-- examples/PlaneJets2D.jl | 6 +++--- examples/ShearLayer2D.jl | 4 ++-- examples/TaylorGreenVortex2D.jl | 6 +++--- examples/TaylorGreenVortex3D.jl | 4 ++-- scratch/filtered.jl | 10 ++++----- scratch/train_model.jl | 6 +++--- src/boundary_conditions/get_bc_vectors.jl | 8 ++----- src/closures/create_les_data.jl | 6 +++--- src/models/viscosity_models.jl | 26 ++++++++++------------- src/momentum/diffusion.jl | 14 ++++++------ src/setup.jl | 14 ++++++++---- src/time_steppers/step_ab_cn.jl | 6 ++---- test/models.jl | 12 +++++------ test/solvers.jl | 3 +-- 24 files changed, 82 insertions(+), 95 deletions(-) diff --git a/README.md b/README.md index 4e358b3dc..b36f42b8e 100644 --- a/README.md +++ b/README.md @@ -53,8 +53,8 @@ wind conditions. using GLMakie using IncompressibleNavierStokes -# Viscosity model -viscosity_model = LaminarModel(; Re = 100.0) +# Reynolds number +Re = 100.0 # Boundary conditions: Unsteady BC requires time derivatives u_bc(x, y, t) = x ≈ 0.0 ? cos(π / 6 * sin(π / 6 * t)) : 0.0 @@ -84,7 +84,7 @@ bodyforce_v(x, y) = 0.0 setup = Setup( x, y; - viscosity_model, + Re, u_bc, v_bc, dudt_bc, diff --git a/examples/Actuator2D.jl b/examples/Actuator2D.jl index ca3434f88..f1afaaf49 100644 --- a/examples/Actuator2D.jl +++ b/examples/Actuator2D.jl @@ -25,7 +25,7 @@ using IncompressibleNavierStokes name = "Actuator2D" # Viscosity model -viscosity_model = LaminarModel(; Re = 100.0) +Re = 100.0 # Boundary conditions: Unsteady BC requires time derivatives u_bc(x, y, t) = x ≈ 0.0 ? cos(π / 6 * sin(π / 6 * t)) : 0.0 @@ -57,7 +57,7 @@ bodyforce_v(x, y) = 0.0 setup = Setup( x, y; - viscosity_model, + Re, u_bc, v_bc, dudt_bc, diff --git a/examples/Actuator3D.jl b/examples/Actuator3D.jl index 9aed8102e..c87af57dd 100644 --- a/examples/Actuator3D.jl +++ b/examples/Actuator3D.jl @@ -25,7 +25,7 @@ using IncompressibleNavierStokes name = "Actuator3D" # Viscosity model -viscosity_model = LaminarModel(; Re = 100.0) +Re = 100.0 # Boundary conditions: Unsteady BC requires time derivatives u_bc(x, y, z, t) = x ≈ 0.0 ? cos(π / 6 * sin(π / 6 * t)) : 0.0 @@ -76,7 +76,7 @@ setup = Setup( x, y, z; - viscosity_model, + Re, u_bc, v_bc, w_bc, diff --git a/examples/BackwardFacingStep2D.jl b/examples/BackwardFacingStep2D.jl index fe539ecad..135db693a 100644 --- a/examples/BackwardFacingStep2D.jl +++ b/examples/BackwardFacingStep2D.jl @@ -26,7 +26,7 @@ using IncompressibleNavierStokes name = "BackwardFacingStep2D" # Viscosity model -viscosity_model = LaminarModel(; Re = 3000.0) +Re = 3000.0 # Boundary conditions: steady inflow on the top half u_bc(x, y, t) = x ≈ 0 && y ≥ 0 ? 24y * (1 / 2 - y) : 0.0 @@ -43,7 +43,7 @@ y = cosine_grid(-0.5, 0.5, 50) plot_grid(x, y) # Build setup and assemble operators -setup = Setup(x, y; viscosity_model, u_bc, v_bc, bc_type); +setup = Setup(x, y; Re, u_bc, v_bc, bc_type); # Time interval t_start, t_end = tlims = (0.0, 7.0) diff --git a/examples/BackwardFacingStep3D.jl b/examples/BackwardFacingStep3D.jl index af5701e31..29392e15c 100644 --- a/examples/BackwardFacingStep3D.jl +++ b/examples/BackwardFacingStep3D.jl @@ -26,7 +26,7 @@ using IncompressibleNavierStokes name = "BackwardFacingStep3D" # Viscosity model -viscosity_model = LaminarModel(; Re = 3000.0) +Re = 3000.0 # Boundary conditions: steady inflow on the top half u_bc(x, y, z, t) = x ≈ 0 && y ≥ 0 ? 24y * (1 / 2 - y) : 0.0 @@ -57,7 +57,7 @@ z = LinRange(-0.25, 0.25, 8) plot_grid(x, y, z) # Build setup and assemble operators -setup = Setup(x, y, z; viscosity_model, u_bc, v_bc, w_bc, bc_type); +setup = Setup(x, y, z; Re, u_bc, v_bc, w_bc, bc_type); # Time interval t_start, t_end = tlims = (0.0, 7.0) diff --git a/examples/DecayingTurbulence2D.jl b/examples/DecayingTurbulence2D.jl index 3c33ec9a6..99e7a541c 100644 --- a/examples/DecayingTurbulence2D.jl +++ b/examples/DecayingTurbulence2D.jl @@ -38,7 +38,7 @@ device = identity ## device = cu # Viscosity model -viscosity_model = LaminarModel(; Re = T(10_000)) +Re = T(10_000) # A 2D grid is a Cartesian product of two vectors n = 256 @@ -48,7 +48,7 @@ y = LinRange(lims..., n + 1) # plot_grid(x, y) # Build setup and assemble operators -setup = Setup(x, y; viscosity_model); +setup = Setup(x, y; Re); # Since the grid is uniform and identical for x and y, we may use a specialized # spectral pressure solver diff --git a/examples/DecayingTurbulence3D.jl b/examples/DecayingTurbulence3D.jl index f7856f8ff..602c6b045 100644 --- a/examples/DecayingTurbulence3D.jl +++ b/examples/DecayingTurbulence3D.jl @@ -38,7 +38,7 @@ device = identity ## device = cu # Viscosity model -viscosity_model = LaminarModel(; Re = T(10_000)) +Re = T(10_000) # A 3D grid is a Cartesian product of three vectors n = 32 @@ -49,7 +49,7 @@ z = LinRange(lims..., n + 1) # plot_grid(x, y, z) # Build setup and assemble operators -setup = Setup(x, y, z; viscosity_model); +setup = Setup(x, y, z; Re); # Since the grid is uniform and identical for x, y, and z, we may use a # specialized spectral pressure solver diff --git a/examples/LidDrivenCavity2D.jl b/examples/LidDrivenCavity2D.jl index 5676328b1..f29771f18 100644 --- a/examples/LidDrivenCavity2D.jl +++ b/examples/LidDrivenCavity2D.jl @@ -53,16 +53,8 @@ device = identity ## using CUDA ## device = cu -# Available viscosity models are: -# -# - [`LaminarModel`](@ref), -# - [`MixingLengthModel`](@ref), -# - [`SmagorinskyModel`](@ref), and -# - [`QRModel`](@ref). -# -# They all take a Reynolds number as a parameter. Here we choose a moderate -# Reynolds number. Note how we pass the floating point type. -viscosity_model = LaminarModel(; Re = T(1_000)) +# Here we choose a moderate Reynolds number. Note how we pass the floating point type. +Re = T(1_000) # Dirichlet boundary conditions are specified as plain Julia functions. They # are marked by the `:dirichlet` symbol. Other possible BC types are @@ -85,7 +77,7 @@ plot_grid(x, y) # We can now build the setup and assemble operators. # A 3D setup is built if we also provide a vector of z-coordinates. -setup = Setup(x, y; viscosity_model, u_bc, v_bc, bc_type); +setup = Setup(x, y; Re, u_bc, v_bc, bc_type); # The pressure solver is used to solve the pressure Poisson equation. # Available solvers are diff --git a/examples/LidDrivenCavity3D.jl b/examples/LidDrivenCavity3D.jl index 52401891a..7a465facd 100644 --- a/examples/LidDrivenCavity3D.jl +++ b/examples/LidDrivenCavity3D.jl @@ -25,7 +25,7 @@ using IncompressibleNavierStokes name = "LidDrivenCavity3D" # Viscosity model -viscosity_model = LaminarModel(; Re = 1000.0) +Re = 1000.0 # Boundary conditions: horizontal movement of the top lid u_bc(x, y, z, t) = y ≈ 1.0 ? 1.0 : 0.0 @@ -57,7 +57,7 @@ z = LinRange(-0.2, 0.2, 10) plot_grid(x, y, z) # Build setup and assemble operators -setup = Setup(x, y, z; viscosity_model, u_bc, v_bc, w_bc, bc_type); +setup = Setup(x, y, z; Re, u_bc, v_bc, w_bc, bc_type); # Time interval t_start, t_end = tlims = (0.0, 0.2) diff --git a/examples/PlanarMixing2D.jl b/examples/PlanarMixing2D.jl index 1c1508eb4..90367cb35 100644 --- a/examples/PlanarMixing2D.jl +++ b/examples/PlanarMixing2D.jl @@ -23,7 +23,7 @@ using IncompressibleNavierStokes name = "PlanarMixing2D" # Viscosity model -viscosity_model = LaminarModel(; Re = 500.0) +Re = 500.0 # Boundary conditions: Unsteady BC requires time derivatives ΔU = 1.0 @@ -52,7 +52,7 @@ y = LinRange(-32.0, 32.0, n) plot_grid(x, y) # Build setup and assemble operators -setup = Setup(x, y; viscosity_model, u_bc, v_bc, dudt_bc, dvdt_bc, bc_type); +setup = Setup(x, y; Re, u_bc, v_bc, dudt_bc, dvdt_bc, bc_type); # Time interval t_start, t_end = tlims = (0.0, 100.0) diff --git a/examples/PlaneJets2D.jl b/examples/PlaneJets2D.jl index 87373b041..cde44faf1 100644 --- a/examples/PlaneJets2D.jl +++ b/examples/PlaneJets2D.jl @@ -26,7 +26,7 @@ using LaTeXStrings name = "PlaneJets2D" # Viscosity model -viscosity_model = LaminarModel(; Re = 6000.0) +Re = 6000.0 # Test cases (A, B, C, D; in order) # U() = sqrt(467.4) @@ -71,8 +71,8 @@ y = LinRange(-10.0, 10.0, 5n) plot_grid(x, y) # Build setup and assemble operators -setup = Setup(x, y; viscosity_model); -# setup = Setup(x, y; viscosity_model, u_bc, v_bc, bc_type); +setup = Setup(x, y; Re); +# setup = Setup(x, y; Re, u_bc, v_bc, bc_type); # Since the grid is uniform and identical for x and y, we may use a specialized # spectral pressure solver diff --git a/examples/ShearLayer2D.jl b/examples/ShearLayer2D.jl index dd4cc1de4..782c4ba18 100644 --- a/examples/ShearLayer2D.jl +++ b/examples/ShearLayer2D.jl @@ -23,7 +23,7 @@ using IncompressibleNavierStokes name = "ShearLayer2D" # Viscosity model -viscosity_model = LaminarModel(; Re = Inf) +Re = Inf # A 2D grid is a Cartesian product of two vectors n = 100 @@ -32,7 +32,7 @@ y = LinRange(0, 2π, n + 1) plot_grid(x, y) # Build setup and assemble operators -setup = Setup(x, y; viscosity_model); +setup = Setup(x, y; Re); # Time interval t_start, t_end = tlims = (0.0, 8.0) diff --git a/examples/TaylorGreenVortex2D.jl b/examples/TaylorGreenVortex2D.jl index 31f28bb36..6f7601eb3 100644 --- a/examples/TaylorGreenVortex2D.jl +++ b/examples/TaylorGreenVortex2D.jl @@ -25,8 +25,8 @@ name = "TaylorGreenVortex2D" # Floating point type T = Float32 -# Viscosity model -viscosity_model = LaminarModel(; Re = T(2_000)) +# Reynolds number +Re = T(2_000) # A 2D grid is a Cartesian product of two vectors n = 128 @@ -36,7 +36,7 @@ y = LinRange(lims..., n + 1) plot_grid(x, y) # Build setup and assemble operators -setup = Setup(x, y; viscosity_model); +setup = Setup(x, y; Re); # Since the grid is uniform and identical for x and y, we may use a specialized # spectral pressure solver diff --git a/examples/TaylorGreenVortex3D.jl b/examples/TaylorGreenVortex3D.jl index cb7f2b07d..5c82a8d0e 100644 --- a/examples/TaylorGreenVortex3D.jl +++ b/examples/TaylorGreenVortex3D.jl @@ -33,7 +33,7 @@ device = identity ## device = cu # Viscosity model -viscosity_model = LaminarModel(; Re = T(2_000)) +Re = T(2_000) # A 3D grid is a Cartesian product of three vectors n = 32 @@ -44,7 +44,7 @@ z = LinRange(lims..., n + 1) plot_grid(x, y, z) # Build setup and assemble operators -setup = Setup(x, y, z; viscosity_model); +setup = Setup(x, y, z; Re); # Since the grid is uniform and identical for x, y, and z, we may use a # specialized spectral pressure solver diff --git a/scratch/filtered.jl b/scratch/filtered.jl index 0f2fa2e91..ae6965e00 100644 --- a/scratch/filtered.jl +++ b/scratch/filtered.jl @@ -22,10 +22,8 @@ using CUDA using LuxCUDA device = cu -# Viscosity model +# Reynolds number Re = T(2_000) -laminar = LaminarModel(; Re) -smagorinsky = SmagorinskyModel(; Re, C_s = T(0.173)) # A 2D grid is a Cartesian product of two vectors s = 2 @@ -42,9 +40,9 @@ y_coarse = y[1:s:end] plot_grid(x_coarse, y_coarse) # Build setup and assemble operators -setup = Setup(x, y; viscosity_model = laminar); +setup = Setup(x, y; Re); devsetup = device(setup); -setup_coarse = Setup(x_coarse, y_coarse; viscosity_model = laminar); +setup_coarse = Setup(x_coarse, y_coarse; Re); # Filter (; KV, Kp) = operator_filter(setup.grid, setup.boundary_conditions, s); @@ -134,7 +132,7 @@ Vbar_nomodel, pbar_nomodel, outputs_lam = solve_unsteady( ); Vbar_smag, pbar_smag, outputs_smag = solve_unsteady( - (; setup_coarse..., viscosity_model = smagorinsky), + (; setup_coarse..., viscosity_model = SmagorinskyModel(; C_s = T(0.173))), KV * V₀, Kp * p₀, tlims; diff --git a/scratch/train_model.jl b/scratch/train_model.jl index 8c5b50028..686bba163 100644 --- a/scratch/train_model.jl +++ b/scratch/train_model.jl @@ -35,14 +35,14 @@ device = cu # Setup n = 128 lims = T(0), T(1) -viscosity_model = LaminarModel(; Re = T(6_000)) +Re = T(6_000) tburn = T(0.05) tsim = T(0.05) # Build LES setup and assemble operators x = LinRange(lims..., n + 1) y = LinRange(lims..., n + 1) -setup = Setup(x, y; viscosity_model); +setup = Setup(x, y; Re); # Number of simulations ntrain = 10 @@ -51,7 +51,7 @@ ntest = 5 # Create LES data from DNS params = (; - viscosity_model, + Re, lims, nles = n, compression = 4, diff --git a/src/boundary_conditions/get_bc_vectors.jl b/src/boundary_conditions/get_bc_vectors.jl index e718a7841..f2c790523 100644 --- a/src/boundary_conditions/get_bc_vectors.jl +++ b/src/boundary_conditions/get_bc_vectors.jl @@ -9,7 +9,7 @@ get_bc_vectors(setup, t) = get_bc_vectors(setup.grid.dimension, setup, t) # 2D version function get_bc_vectors(::Dimension{2}, setup, t) - (; grid, operators, boundary_conditions, viscosity_model) = setup + (; grid, operators, boundary_conditions, viscosity_model, Re) = setup (; Nux_in, Nvy_in, Np, Npx, Npy) = grid (; xin, yin, x, y, hx, hy, xp, yp) = grid @@ -27,8 +27,6 @@ function get_bc_vectors(::Dimension{2}, setup, t) (; u_bc, v_bc, dudt_bc, dvdt_bc) = boundary_conditions (; p_bc, bc_unsteady) = boundary_conditions - (; Re) = viscosity_model - if order4 (; Au_ux_bc3, Au_uy_bc3, Av_vx_bc3, Av_vy_bc3) = operators (; Iu_ux_bc3, Iv_uy_bc_lu3, Iv_uy_bc_lr3) = operators @@ -479,9 +477,7 @@ end # 3D version function get_bc_vectors(::Dimension{3}, setup, t) - (; grid, operators, boundary_conditions, viscosity_model) = setup - - (; Re) = viscosity_model + (; grid, operators, boundary_conditions, viscosity_model, Re) = setup (; u_bc, v_bc, w_bc, dudt_bc, dvdt_bc, dwdt_bc) = boundary_conditions (; p_bc, bc_unsteady) = boundary_conditions diff --git a/src/closures/create_les_data.jl b/src/closures/create_les_data.jl index 17a714f0a..d2bf55526 100644 --- a/src/closures/create_les_data.jl +++ b/src/closures/create_les_data.jl @@ -104,7 +104,7 @@ _filter_saver( function create_les_data( T; - viscosity_model = LaminarModel(; Re = T(2_000)), + Re = T(2_000), lims = (T(0), T(1)), nles = 64, compression = 4, @@ -121,8 +121,8 @@ function create_les_data( yles = ydns[1:compression:end] # Build setup and assemble operators - dns = Setup(xdns, ydns; viscosity_model) - les = Setup(xles, yles; viscosity_model) + dns = Setup(xdns, ydns; Re) + les = Setup(xles, yles; Re) # Filter (; KV, Kp) = operator_filter(dns.grid, dns.boundary_conditions, compression) diff --git a/src/models/viscosity_models.jl b/src/models/viscosity_models.jl index f5dd24282..a76d0490b 100644 --- a/src/models/viscosity_models.jl +++ b/src/models/viscosity_models.jl @@ -3,42 +3,38 @@ Abstract viscosity model. """ -abstract type AbstractViscosityModel{T} end +abstract type AbstractViscosityModel end """ - LaminarModel(Re) + LaminarModel() -Laminar model with Reynolds number `Re`. +Laminar model. """ -Base.@kwdef struct LaminarModel{T} <: AbstractViscosityModel{T} - Re::T # Reynolds number +Base.@kwdef struct LaminarModel <: AbstractViscosityModel end """ - MixingLengthModel(Re) + MixingLengthModel() -Mixing-length model with Reynolds number `Re` and mixing length `lm`. +Mixing-length model with mixing length `lm`. """ -Base.@kwdef struct MixingLengthModel{T} <: AbstractViscosityModel{T} - Re::T # Reynolds number +Base.@kwdef struct MixingLengthModel{T} <: AbstractViscosityModel lm::T = 1 # Mixing length end """ SmagorinskyModel(Re, C_s = 0.17) -Smagorinsky-Lilly model with Reynolds number `Re` and constant `C_s`. +Smagorinsky-Lilly model with constant `C_s`. """ -Base.@kwdef struct SmagorinskyModel{T} <: AbstractViscosityModel{T} - Re::T # Reynolds number +Base.@kwdef struct SmagorinskyModel{T} <: AbstractViscosityModel C_s::T = 0.17 # Smagorinsky constant end """ QR(Re) -QR-model with Reynolds number `Re`. +QR-model. """ -Base.@kwdef struct QRModel{T} <: AbstractViscosityModel{T} - Re::T # Reynolds number +struct QRModel{T} <: AbstractViscosityModel end diff --git a/src/momentum/diffusion.jl b/src/momentum/diffusion.jl index 3b7a66421..afbf95adc 100644 --- a/src/momentum/diffusion.jl +++ b/src/momentum/diffusion.jl @@ -11,8 +11,8 @@ function diffusion end diffusion(m, V, setup; kwargs...) = diffusion(setup.grid.dimension, m, V, setup; kwargs...) -function diffusion(m::LaminarModel, V, setup; bc_vectors, get_jacobian = false) - (; Re) = m +function diffusion(::LaminarModel, V, setup; bc_vectors, get_jacobian = false) + (; Re) = setup (; Diff) = setup.operators (; yDiff) = bc_vectors @@ -36,6 +36,7 @@ function diffusion( bc_vectors, get_jacobian = false, ) + (; Re) = setup (; indu, indv) = setup.grid (; Dux, Duy, Dvx, Dvy) = setup.operators (; Su_ux, Su_uy, Su_vx, Sv_vx, Sv_vy, Sv_uy) = setup.operators @@ -63,7 +64,7 @@ function diffusion( # Tau = 2*(ν+ν_t)*S(u), with S(u) = 1/2*(∇u + (∇u)^T) # Molecular viscosity - ν = 1 / model.Re + ν = 1 / Re du = Dux * (2 .* (ν .+ ν_t_ux) .* S11[:]) .+ Duy * (2 .* (ν .+ ν_t_uy) .* S12[:]) dv = Dvx * (2 .* (ν .+ ν_t_vx) .* S21[:]) .+ Dvy * (2 .* (ν .+ ν_t_vy) .* S22[:]) @@ -126,8 +127,8 @@ function diffusion! end diffusion!(m, d, ∇d, V, setup; kwargs...) = diffusion!(setup.grid.dimension, m, d, ∇d, V, setup; kwargs...) -function diffusion!(m::LaminarModel, d, ∇d, V, setup; bc_vectors, get_jacobian = false) - (; Re) = m +function diffusion!(::LaminarModel, d, ∇d, V, setup; bc_vectors, get_jacobian = false) + (; Re) = setup (; Diff) = setup.operators (; yDiff) = bc_vectors @@ -151,6 +152,7 @@ function diffusion!( bc_vectors, get_jacobian = false, ) + (; Re) = setup (; indu, indv) = setup.grid (; Dux, Duy, Dvx, Dvy) = setup.operators (; Su_ux, Su_uy, Su_vx, Sv_vx, Sv_vy, Sv_uy) = setup.operators @@ -181,7 +183,7 @@ function diffusion!( # Tau = 2*(ν+ν_t)*S(u), with S(u) = 1/2*(∇u + (∇u)^T) # Molecular viscosity - ν = 1 / model.Re + ν = 1 / Re du .= Dux * (2 .* (ν .+ ν_t_ux) .* S11) .+ Duy * (2 .* (ν .+ ν_t_uy) .* S12) dv .= Dvx * (2 .* (ν .+ ν_t_vx) .* S21) .+ Dvy * (2 .* (ν .+ ν_t_vy) .* S22) diff --git a/src/setup.jl b/src/setup.jl index f32708f44..043fe24ff 100644 --- a/src/setup.jl +++ b/src/setup.jl @@ -1,7 +1,8 @@ """ Setup( x, y; - viscosity_model = LaminarModel(; Re = 1000), + Re = 1000, + viscosity_model = LaminarModel(), convection_model = NoRegConvectionModel(), u_bc = (x, y, t) -> 0, v_bc = (x, y, t) -> 0, @@ -22,7 +23,8 @@ Create 2D setup. function Setup( x, y; - viscosity_model = LaminarModel(; Re = convert(eltype(x), 1000)), + Re = convert(eltype(x), 1000), + viscosity_model = LaminarModel(), convection_model = NoRegConvectionModel(), u_bc = (x, y, t) -> 0, v_bc = (x, y, t) -> 0, @@ -47,6 +49,7 @@ function Setup( (; grid, boundary_conditions, + Re, viscosity_model, convection_model, force, @@ -58,7 +61,8 @@ end """ Setup( x, y, z; - viscosity_model = LaminarModel(; Re = 1000), + Re = convert(eltype(x), 1000), + viscosity_model = LaminarModel(), convection_model = NoRegConvectionModel(), u_bc = (x, y, w, t) -> 0.0, v_bc = (x, y, w, t) -> 0.0, @@ -101,7 +105,8 @@ function Setup( x, y, z; - viscosity_model = LaminarModel(; Re = convert(eltype(x), 1000)), + Re = convert(eltype(x), 1000), + viscosity_model = LaminarModel(), convection_model = NoRegConvectionModel(), u_bc = (x, y, w, t) -> 0, v_bc = (x, y, w, t) -> 0, @@ -153,6 +158,7 @@ function Setup( (; grid, boundary_conditions, + Re, viscosity_model, convection_model, force, diff --git a/src/time_steppers/step_ab_cn.jl b/src/time_steppers/step_ab_cn.jl index 2266257ae..fa88a59a7 100644 --- a/src/time_steppers/step_ab_cn.jl +++ b/src/time_steppers/step_ab_cn.jl @@ -18,14 +18,13 @@ create_stepper( function step(method::AdamsBashforthCrankNicolsonMethod, stepper, Δt) (; setup, pressure_solver, bc_vectors, V, p, t, n, Vₙ, pₙ, cₙ, tₙ, Diff_fact) = stepper - (; convection_model, viscosity_model, force, grid, operators, boundary_conditions) = + (; convection_model, viscosity_model, Re, force, grid, operators, boundary_conditions) = setup (; bc_unsteady) = boundary_conditions (; NV, Ω) = grid (; G, M) = operators (; Diff) = operators (; p_add_solve, α₁, α₂, θ, method_startup) = method - (; Re) = viscosity_model T = typeof(Δt) @@ -169,7 +168,7 @@ function step!( momentum_cache, ) (; setup, pressure_solver, bc_vectors, V, p, t, n, Vₙ, pₙ, cₙ, tₙ, Diff_fact) = stepper - (; convection_model, viscosity_model, force, grid, operators, boundary_conditions) = + (; convection_model, viscosity_model, Re, force, grid, operators, boundary_conditions) = setup (; bc_unsteady) = boundary_conditions (; NV, Ω) = grid @@ -178,7 +177,6 @@ function step!( (; p_add_solve, α₁, α₂, θ, method_startup) = method (; cₙ₋₁, F, f, Δp, Rr, b, bₙ, bₙ₊₁, yDiffₙ, yDiffₙ₊₁, Gpₙ) = cache (; d, ∇d) = momentum_cache - (; Re) = viscosity_model T = typeof(Δt) diff --git a/test/models.jl b/test/models.jl index 5f23e7473..a122f8be4 100644 --- a/test/models.jl +++ b/test/models.jl @@ -23,10 +23,10 @@ # Viscosity models T = Float64 Re = 1000.0 - lam = LaminarModel(; Re) - ml = MixingLengthModel(; Re, lm = Δ) - smag = SmagorinskyModel(; Re) - qr = QRModel(; Re) + lam = LaminarModel() + ml = MixingLengthModel(; lm = Δ) + smag = SmagorinskyModel() + qr = QRModel() # Convection models noreg = NoRegConvectionModel() @@ -46,7 +46,7 @@ for (viscosity_model, convection_model) in models @testset "$(typeof(viscosity_model)) $(typeof(convection_model))" begin @info "Testing $(typeof(viscosity_model)) and $(typeof(convection_model))" - setup = Setup(x, y; viscosity_model, convection_model, u_bc, v_bc, bc_type) + setup = Setup(x, y; Re, viscosity_model, convection_model, u_bc, v_bc, bc_type) V₀, p₀ = create_initial_conditions( setup, @@ -73,7 +73,7 @@ for (viscosity_model, convection_model) in models @testset "$(typeof(viscosity_model)) $(typeof(convection_model))" begin @info "Testing $(typeof(viscosity_model)) and $(typeof(convection_model))" - setup = Setup(x, y; viscosity_model, convection_model, u_bc, v_bc, bc_type) + setup = Setup(x, y; Re, viscosity_model, convection_model, u_bc, v_bc, bc_type) end end end diff --git a/test/solvers.jl b/test/solvers.jl index d5cbd37b5..b4fdb4515 100644 --- a/test/solvers.jl +++ b/test/solvers.jl @@ -1,12 +1,11 @@ @testset "Solvers" begin T = Float64 Re = 500.0 - viscosity_model = LaminarModel(; Re) n = 50 x = LinRange(0, 2π, n + 1) y = LinRange(0, 2π, n + 1) - setup = Setup(x, y; viscosity_model) + setup = Setup(x, y; Re) pressure_solver = SpectralPressureSolver(setup) From c3b5d3290165ff9b264eebc08998c46fdfc40b89 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Fri, 8 Sep 2023 14:01:34 +0200 Subject: [PATCH 038/379] docs: Reformulate discretization --- docs/src/equations/spatial.md | 370 ++++++++++++++++------------------ 1 file changed, 169 insertions(+), 201 deletions(-) diff --git a/docs/src/equations/spatial.md b/docs/src/equations/spatial.md index 557cb45f5..4c0d3b55d 100644 --- a/docs/src/equations/spatial.md +++ b/docs/src/equations/spatial.md @@ -3,65 +3,85 @@ To discretize the incompressible Navier-Stokes equations, we will use finite volumes on a staggered Cartesian grid, as proposed by Harlow and Welsh [Harlow1965](@cite). We will use the notation of Sanderse [Sanderse2012](@cite) -[Sanderse2013](@cite) [Sanderse2014](@cite). For simplicity, we will illustrate -everything in 2D. The 3D discretization is very similar, but more verbose. +[Sanderse2013](@cite) [Sanderse2014](@cite). + +Let ``d \in \{2, 3\}`` denote the spatial dimension (2D or 3D). We will make +use of the "Cartesian" index ``I = (i, j)`` in 2D or ``I = (i, j, k)`` in 3D, +with ``I(1) = i``, ``I(2) = j``, and ``I(3) = k``. Here, the indices ``I``, +``i``, ``j``, and ``k``, represent discrete degrees of freedom. To specify a +spatial dimension, we will use the symbols ``(\alpha, \beta, \gamma) \in \{1, +\dots, d\}^3``. We will use the symbol ``\delta(\alpha) = (\delta_{\alpha +\beta})_{\beta = 1}^d \in \{0, 1\}^d`` to indicate a perturbation in the +direction ``\alpha``, where ``\delta_{\alpha \beta}`` is the Kronecker symbol. +The spatial variable is ``x = (x^1, \dots, x^d) \in \Omega \subset +\mathbb{R}^d``. Note that ``\Omega = \prod_{\alpha = 1}^d [0, L^\alpha]`` is +assumed to have the shape of a box with side lengths ``L^\alpha > 0``. ## Finite volumes -The finite volumes are given by +The finite volumes are defined as ```math -\Omega_{i, j} = [x_{i - -\frac{1}{2}}, x_{i + \frac{1}{2}}] \times [y_{j - \frac{1}{2}}, y_{j + -\frac{1}{2}}]. +\Omega_I = \prod_{\alpha = 1}^d \left[ x^\alpha_{I(\alpha) - \frac{1}{2}}, +x^\alpha_{I(\alpha) + \frac{1}{2}} \right], \quad I \in \mathcal{I}. ``` -They are fully defined by the vectors of volume faces ``x = -(x_{i + \frac{1}{2}})_i`` and ``y = (y_{j + \frac{1}{2}})_j``. Note that the -components are not assumed to be uniformly spaced. But we do assume that they -are strictly increasing. +They represent rectangles in 2D and prisms in 3D. They are fully defined by the +vectors of volume faces ``x^\alpha = \left( x^\alpha_{i + \frac{1}{2}} +\right)_{i = 0}^{N(\alpha)}``, where ``N = (N(1), \dots, N(d)) \in +\mathbb{N}^d`` are the numbers of volumes in each dimension and ``\mathcal{I} = +\prod_{\alpha = 1}^d \{1, \dots, N(\alpha)\}`` the set of finite volume +indices. Note that the components ``x^\alpha_{i}`` are not assumed to be +uniformly spaced. But we do assume that they are strictly increasing with +``i``. -The volume center coordinates are determined from the volume boundary -boundaries by ``x_i = \frac{1}{2} (x_{i - \frac{1}{2}} + x_{i + \frac{1}{2}})`` -and ``y_j = \frac{1}{2} (y_{j - \frac{1}{2}} + y_{j + \frac{1}{2}})``. This -allows for defining the shifted volumes ``\Omega_{i + \frac{1}{2}, j}``, -``\Omega_{i, j + \frac{1}{2}}``, and ``\Omega_{i + \frac{1}{2}, j + -\frac{1}{2}}``. +The volume center coordinates are determined from the volume boundaries by +``x^\alpha_{i} = \frac{1}{2} (x^\alpha_{i - \frac{1}{2}} + x_{i + +\frac{1}{2}})``. This allows for defining the shifted volumes ``\Omega_{I + +\delta(\alpha) / 2}``. -We also define the volume widths ``\Delta x_i = x_{i + \frac{1}{2}} - x_{i - -\frac{1}{2}}`` and volume heights ``\Delta y_j = y_{j + \frac{1}{2}} - y_{j - -\frac{1}{2}}``. The volume sizes are thus ``| \Omega_{i, j} | = \Delta x_i -\Delta y_j``. +We also define the volume widths/depths/heights ``\Delta x^\alpha_i = +x^\alpha_{i + \frac{1}{2}} - x^\alpha_{i - \frac{1}{2}}``, where ``i`` can take +half values. The volume sizes are thus ``| \Omega_{I} | = \prod_{\alpha = 1}^d +\Delta x^\alpha_{I(\alpha)}``. -In each finite volume ``\Omega_{i, j}``, there are four different positions in -which quantities of interest can be defined: +In addition to the finite volumes and their half-indexed shifted variants, we +define the surface -- The ``p``-point ``(x_i, y_j)``, -- The ``u``-point ``(x_{i + \frac{1}{2}}, y_j)``, -- The ``v``-point ``(x_i, y_{j + \frac{1}{2}})``, -- The ``\omega``-point ``(x_{i + \frac{1}{2}}, y_{j + \frac{1}{2}})``. +```math +\Gamma^\alpha_I = \prod_{\beta = 1}^d \begin{cases} + \left\{ x^\beta_{I(\beta)} \right\}, & \quad \alpha = \beta \\ + \left[ x^\beta_{I(\beta) - 1 / 2}, x^\beta_{I(\beta) + 1 / 2} \right], & \quad + \text{otherwise}, +\end{cases} +``` +where ``I`` can take half-values. It is the interface between ``\Omega_{I - +\delta(\alpha) / 2}`` and ``\Omega_{I + \delta(\alpha) / 2}``, and has surface +normal ``\delta(\alpha)``. -They form the corners of the top-right quadrant of ``\Omega_{i, j}``. -The vectors of unknowns will not contain all the half-index values, only those -of their own point: +In each finite volume ``\Omega_{I}`` (integer ``I``), there are three different +types of positions in which quantities of interest can be defined: -- The vector ``u_h`` will only consist of the components ``u_{i + \frac{1}{2}, - j}``. -- The vector ``v_h`` will only consist of the components ``v_{i, j + - \frac{1}{2}}``. -- The vector ``p_h`` will only consist of the components ``p_{i, j}``. +- The volume center ``x_I = (x_{I(1)}, \dots, x_{I(d)})``, where the discrete + pressure ``p_I`` is defined; +- The right/rear/top volume face centers ``x_{I + \delta(\alpha) / 2}``, where + the discrete ``\alpha``-velocity component ``u^\alpha_{I + \delta(\alpha) / 2}`` is defined; +- The right-rear-top volume corner ``x_{I + \sum_{\alpha} \delta(\alpha) / + 2}``, where the discrete vorticity ``\omega_{I + \sum_{\alpha} \delta(\alpha) / + 2}`` is defined. -In addtion, we define the full velocity vector -``V_h = \begin{pmatrix} u_h \\ v_h \end{pmatrix}``. +The vectors of unknowns ``u^\alpha_h`` and ``p_h`` will not contain all the +half-index components, only those from their own canonical position. -!!! note "Storage convection" +!!! note "Storage convention" We use the column-major convention (Julia, MATLAB, Fortran), and not the - row-major convention (Python, C). Thus the ``x``-index ``i`` will vary for - one whole cycle in the vectors ``u_h``, ``v_h``, ``p_h`` before the - ``y``-index ``j`` is incremented. + row-major convention (Python, C). Thus the ``x^1``-index ``i`` will vary for + one whole cycle in the vectors before the + ``x^2``-index ``j`` is incremented, e.g. ``p_h = (p_{(1, 1, 1)}, + p_{(2, 1, 1)}, \dots p_{(N(1), N(2), N(3))})`` in 3D. -This finite volume configuration is illustrated as follows: +In 2D, this finite volume configuration is illustrated as follows: ![Grid](../assets/grid.png) @@ -69,40 +89,39 @@ This finite volume configuration is illustrated as follows: When a quantity is required *outside* of its native point, we will use interpolation. Examples: -- To compute ``u`` at the pressure points: +- To compute ``u^\alpha`` at the pressure point ``x_I``: ```math \begin{split} - u_{i, j} & = - \frac{x_{i + \frac{1}{2}} - x_i}{x_{i + \frac{1}{2}} - x_{i - \frac{1}{2}}} - u_{i - \frac{1}{2}, j} - + \frac{x_i - x_{i - \frac{1}{2}}}{x_{i + \frac{1}{2}} - x_{i - \frac{1}{2}}} - u_{i + \frac{1}{2}, j} \\ - & = - \frac{1}{2} (u_{i - \frac{1}{2}, j} + u_{i + \frac{1}{2}, j}) + u^\alpha_I & = \frac{x^\alpha_{I(\alpha) + 1 / 2} - + x^\alpha_{I(\alpha)}}{x^\alpha_{I(\alpha) + 1 / 2} - x^\alpha_{I(\alpha) - 1 / 2}} + u_{I - \delta(\alpha) / 2} + + \frac{x^\alpha_{I(\alpha)} - x^\alpha_{I(\alpha) - 1 / 2}}{x^\alpha_{I(\alpha) + 1 / 2} - x^\alpha_{I(\alpha) - 1 / 2}} + u_{I + \delta(\alpha) / 2} \\ + & = \frac{1}{2} \left( u_{I - \delta(\alpha) / 2} + u_{I + \delta(\alpha) / 2} \right). \end{split} ``` Interpolation weights from volume faces to volume centers are always ``\frac{1}{2}``. -- To compute ``v`` at vorticity points: +- To compute ``u^\alpha`` at center of edge between ``\alpha``-face and + ``\beta``-face ``x_{I + \delta(\alpha) / 2 + \delta(\beta) / 2}``: ```math - v_{i + \frac{1}{2}, j + \frac{1}{2}} = - \frac{x_{i + 1} - x_{i + \frac{1}{2}}}{x_{i + 1} - x_i} - v_{i, j + \frac{1}{2}} - + \frac{x_{i + \frac{1}{2}} - x_i}{x_{i + 1} - x_i} - v_{i + 1, j + \frac{1}{2}} + u^\alpha_{I + \delta(\alpha) / 2 + \delta(\beta) / 2} = + \frac{x^\beta_{I(\beta) + 1} - x^\beta_{I(\beta) + 1 / 2}}{x^\beta_{I(\beta) + 1} - x^\beta_{I(\beta)}} + u^\alpha_{I + \delta(\alpha) / 2} + + \frac{x^\beta_{I(\beta) + 1 / 2} - x^\beta_{I(\beta)}}{x^\beta_{I(\beta) + 1} - x^\beta_{I(\beta)}} + u^\alpha_{I + \delta(\alpha) / 2 + \delta(\beta)}. ``` -- To compute ``p`` at ``v``-points: + Note that the grid is allowed to be non-uniform, so the interpolation weights + may unequal and different from ``\frac{1}{2}``. +- To compute ``p`` at ``u^\alpha``-points: ```math - p_{i, j + \frac{1}{2}} = - \frac{y_{j + 1} - y_{j + \frac{1}{2}}}{y_{j + 1} - y_j} - p_{i, j} - + \frac{y_{j + \frac{1}{2}} - y_j}{y_{j + 1} - y_j} - p_{i, j + 1} + p_{I + \delta(\alpha) / 2} = + \frac{x^\alpha_{I(\alpha) + 1} - x^\alpha_{I(\alpha) + 1 / 2}}{x^\alpha_{I(\alpha) + 1} - x^\alpha_{I(\alpha)}} + p_{I} + + \frac{x^\alpha_{I(\alpha) + 1 / 2} - x^\alpha_{I(\alpha)}}{x^\alpha_{I(\alpha) + 1} - x^\alpha_{I(\alpha)}} + p_{I + \delta(\alpha)} ``` -Note that the grid is allowed to be non-uniform, so the weights of interpolation from -volume centers to volume faces may unequal and different from ``\frac{1}{2}``. - ## Finite volume discretization of the Navier-Stokes equations We will consider the integral form of the Navier-Stokes equations. This has the @@ -114,55 +133,41 @@ of finite difference approximations we need to perform. The mass equation takes the form ```math -\int_{\partial \mathcal{O}} V \cdot n \, \mathrm{d} \Gamma = 0, \quad \forall +\int_{\partial \mathcal{O}} u \cdot n \, \mathrm{d} \Gamma = 0, \quad \forall \mathcal{O} \subset \Omega. ``` -Using the pressure volume ``\mathcal{O} = \Omega_{i, j}``, we get +Using the pressure volume ``\mathcal{O} = \Omega_{I}``, we get ```math -\int_{y_{j - \frac{1}{2}}}^{y_{j + \frac{1}{2}}} u(x_{i + \frac{1}{2}}, y) \, \mathrm{d} y -- \int_{y_{j - \frac{1}{2}}}^{y_{j + \frac{1}{2}}} u(x_{i - \frac{1}{2}}, y) \, \mathrm{d} y -+ \int_{x_{i - \frac{1}{2}}}^{x_{i + \frac{1}{2}}} v(x, y_{j + \frac{1}{2}}) \, \mathrm{d} x -- \int_{x_{i - \frac{1}{2}}}^{x_{i + \frac{1}{2}}} v(x, y_{j - \frac{1}{2}}) \, \mathrm{d} x -= 0. +\sum_{\alpha = 1}^d \left( \int_{\Gamma^\alpha_{I + \delta(\alpha) / 2}} u^\alpha \, \mathrm{d} \Gamma - +\int_{\Gamma_{I - \delta(\alpha) / 2}^\alpha} u^\alpha \, \mathrm{d} \Gamma +\right) = 0. ``` -Assuming that the flow is fully resolved, meaning that ``\Omega_{i, j}`` is is -sufficiently small such that ``u`` and ``v`` are locally linear, we can perform the +Assuming that the flow is fully resolved, meaning that ``\Omega_{I}`` is is +sufficiently small such that ``u`` is locally linear, we can perform the local approximation (quadrature) ```math -\begin{split} - \int_{y_{j - \frac{1}{2}}}^{y_{j + \frac{1}{2}}} u(x_{i - \frac{1}{2}}, y) \, \mathrm{d} y - & \approx \Delta y_j u_{i - \frac{1}{2}, j}, \\ - \int_{y_{j - \frac{1}{2}}}^{y_{j + \frac{1}{2}}} u(x_{i + \frac{1}{2}}, y) \, \mathrm{d} y - & \approx \Delta y_j u_{i + \frac{1}{2}, j}, \\ - \int_{x_{i - \frac{1}{2}}}^{x_{i + \frac{1}{2}}} v(x, y_{j - \frac{1}{2}}) \, \mathrm{d} x - & \approx \Delta x_i v_{i, j - \frac{1}{2}}, \\ - \int_{x_{i - \frac{1}{2}}}^{x_{i + \frac{1}{2}}} v(x, y_{j + \frac{1}{2}}) \, \mathrm{d} x - & \approx \Delta x_i v_{i, j + \frac{1}{2}}. -\end{split} +\int_{\Gamma^\alpha_I} u^\alpha \, \mathrm{d} \Gamma \approx | \Gamma^\alpha_I | u^\alpha_{I}. ``` This yields the discrete mass equation ```math -\Delta y_j (u_{i + \frac{1}{2}, j} - u_{i - -\frac{1}{2}, j}) -+ -\Delta x_i (v_{i, j + \frac{1}{2}} - v_{i, j - -\frac{1}{2}}) -= 0 +\sum_{\alpha = 1}^d +\left| \Gamma^\alpha_I \right| \left( u^\alpha_{I + \delta(\alpha) / 2} - +u^\alpha_{I - \delta(\alpha) / 2} \right) = 0 ``` which can also be written in the matrix form ```math -M V_h = M_x u_h + M_y v_h = 0, +M V = \sum_{\alpha = 1}^d M^\alpha u^\alpha = 0, ``` -where ``M = \begin{pmatrix} M_x & M_y \end{pmatrix}`` is the discrete +where ``M = \begin{pmatrix} M_1 & \dots & M_d \end{pmatrix}`` is the discrete divergence operator. !!! note "Approximation error" @@ -171,147 +176,110 @@ divergence operator. ### Momentum equations -We will consider the two momentum equations separately. Grouping the -convection, pressure gradient, diffusion, and body force terms in each of their -own integrals, we get, for all ``\mathcal{O} \subset \Omega``: +Grouping the convection, pressure gradient, diffusion, and body force terms in +each of their own integrals, we get, for all ``\mathcal{O} \subset \Omega``: ```math -\begin{split} - \frac{\partial }{\partial t} \int_\mathcal{O} u \, \mathrm{d} \Omega - & = - - \int_{\partial \mathcal{O}} \left( u u n_x + u v n_y \right) \, \mathrm{d} \Gamma - - \int_{\partial \mathcal{O}} p n_x \, \mathrm{d} \Gamma - + \nu \int_{\partial \mathcal{O}} \left( \frac{\partial u}{\partial x} n_x + \frac{\partial u}{\partial y} n_y \right) \, \mathrm{d} \Gamma - + \int_\mathcal{O} f_u \mathrm{d} \Omega, \\ - \frac{\partial }{\partial t} \int_\mathcal{O} v \, \mathrm{d} \Omega - & = - - \int_{\partial \mathcal{O}} \left( v u n_x + v v n_y \right) \, \mathrm{d} \Gamma - - \int_{\partial \mathcal{O}} p n_y \, \mathrm{d} \Gamma - + \nu \int_{\partial \mathcal{O}} \left( \frac{\partial v}{\partial x} n_x + \frac{\partial v}{\partial y} n_y \right) \, \mathrm{d} \Gamma - + \int_\mathcal{O} f_v \mathrm{d} \Omega, -\end{split} +\frac{\partial }{\partial t} \int_\mathcal{O} u^\alpha \, \mathrm{d} \Omega += +- \sum_{\beta = 1}^d \int_{\partial \mathcal{O}} u^\alpha u^\beta n^\beta \, \mathrm{d} \Gamma +- \int_{\partial \mathcal{O}} p n^\alpha \, \mathrm{d} \Gamma ++ \nu \sum_{\beta = 1}^d \int_{\partial \mathcal{O}} \frac{\partial u^\alpha}{\partial x^\beta} n^\beta \, \mathrm{d} \Gamma ++ \int_\mathcal{O} f^\alpha \mathrm{d} \Omega, ``` -where ``n = (n_x, n_y)``. +where ``n = (n^1, \dots, n^d)`` is the surface normal vector to ``\partial +\Omega``. This time, we will not let ``\mathcal{O}`` be the reference finite volume -``\Omega_{i, j}`` (the ``p``-volume), but rather the shifted ``u``- and -``v``-volumes. Setting ``\mathcal{O} = \Omega_{i + \frac{1}{2}, j}`` gives +``\Omega_{I}`` (the ``p``-volume), but rather the shifted ``u^\alpha``-volume. +Setting ``\mathcal{O} = \Omega_{I + \delta(\alpha) / 2}`` (with right/rear/top +``\beta``-faces ``\Gamma^\beta_{I + \delta(\alpha) / 2 + \delta(\beta) / 2}``) +gives ```math \begin{split} \frac{\partial }{\partial t} - \int_{y_{j - \frac{1}{2}}}^{y_{j + \frac{1}{2}}} - \int_{x_i}^{x_{i + 1}} - u(x, y) \, \mathrm{d} x \mathrm{d} y + \int_{\Omega_{I + \delta(\alpha) / 2}} + \! \! \! + \! \! \! + \! \! \! + \! \! \! + u^\alpha \, \mathrm{d} \Omega = - & - - \int_{y_{j - \frac{1}{2}}}^{y_{j + \frac{1}{2}}} - \left( (u u)(x_{i + 1}, y) - (u u)(x_i, y) \right) - \, \mathrm{d} y \\ - & - - \int_{x_i}^{x_{i + 1}} - \left( (u v)(x, y_{j + \frac{1}{2}}) - (u v)(x, y_{j - \frac{1}{2}}) - \right) - \, \mathrm{d} x \\ - + \sum_{\beta = 1}^d \left( + \int_{\Gamma^{\beta}_{I + \delta(\alpha) / 2 + \delta(\beta) / 2}} + \! \! \! + \! \! \! + \! \! \! + \! \! \! + u^\alpha u^\beta \, \mathrm{d} \Gamma + - \int_{\Gamma^{\beta}_{I + \delta(\alpha) / 2 - \delta(\beta) / 2}} + \! \! \! + \! \! \! + \! \! \! + \! \! \! + \! \! \! + \! \! \! + u^\alpha u^\beta \, \mathrm{d} \Gamma + \right) \\ & - - \int_{y_{j - \frac{1}{2}}}^{y_{j + \frac{1}{2}}} - \left( p(x_{i + 1}, y) - p(x_i, y) \right) - \, \mathrm{d} y \\ - - & + \nu - \int_{y_{j - \frac{1}{2}}}^{y_{j + \frac{1}{2}}} - \left( \frac{\partial u}{\partial x}(x_{i + 1}, y) - \frac{\partial - u}{\partial x}(x_i, y) \right) - \, \mathrm{d} y \\ - - & + \nu \int_{x_i}^{x_{i + 1}} - \left( \frac{\partial u}{\partial y}(x, y_{j + \frac{1}{2}}) - - \frac{\partial u}{\partial y}(x, y_{j - \frac{1}{2}}) \right) - \, \mathrm{d} x \\ - + \left( + \int_{\Gamma^{\alpha}_{I + \delta(\alpha)}} p \, \mathrm{d} \Gamma + - \int_{\Gamma^{\alpha}_{I}} p \, \mathrm{d} \Gamma + \right) \\ + & + \nu \sum_{\beta = 1}^d + \left( + \int_{\Gamma^{\beta}_{I + \delta(\alpha) / 2 + \delta(\beta) / 2}} + \frac{\partial u^\alpha}{\partial x^\beta} \, \mathrm{d} \Gamma + - \int_{\Gamma^{\beta}_{I + \delta(\alpha) / 2 - \delta(\beta) / 2}} + \frac{\partial u^\alpha}{\partial x^\beta} \, \mathrm{d} \Gamma + \right) \\ & + - \int_{y_{j - \frac{1}{2}}}^{y_{j + \frac{1}{2}}} - \int_{x_i}^{x_{i + 1}} - f_u(x, y) \, \mathrm{d} x \mathrm{d} y \\ + \int_{\Omega_{I + \delta(\alpha) / 2}} + f^\alpha \, \mathrm{d} \Omega \end{split} ``` This equation is still exact. We now introduce some approximations on -``\Omega_{i + \frac{1}{2}, j}`` and its boundaries to remove all unknown +``\Omega_{I + \delta(\alpha) / 2}`` and its boundaries to remove all unknown continuous quantities. 1. We replace the integrals with a mid-point quadrature rule. -1. The the mid-point values of derivatives are - approximated using a central finite difference: +1. The mid-point values of derivatives are approximated using a central-like finite difference: ```math - \frac{\partial u}{\partial x}(x_i, y_j) \approx \frac{u_{i + \frac{1}{2}, j} - - u_{i - \frac{1}{2}, j}}{x_{i + \frac{1}{2}} - x_{i - \frac{1}{2}}}, + \frac{\partial u^\alpha}{\partial x^\beta}(x_I) \approx + \frac{u^\alpha_{I + \delta(\beta) / 2} + - u^\alpha_{I - \delta(\beta) / 2}}{x^\beta_{I(\beta) + 1 / 2} - x^\beta_{I(\beta) - 1 / 2}}. ``` - and similarly for ``\frac{\partial u}{\partial x}(x_{i + 1}, y_j)``. If the - grid is non-uniform, we also translate the derivative value to the midpoint: - ```math - \frac{\partial u}{\partial y}(x_{i + \frac{1}{2}}, y_{j + \frac{1}{2}}) - \approx \frac{\partial u}{\partial y} \left( x_{i + \frac{1}{2}}, \frac{y_i + y_{i + 1}}{2} \right) - \approx \frac{u_{i + \frac{1}{2}, j + 1} - u_{i + \frac{1}{2}, j}}{y_{j + 1} - y_j}, - ``` - and similarly for - ``\frac{\partial u}{\partial y}(x_{i + \frac{1}{2}}, y_{j - \frac{1}{2}})``. 1. Quantities outside their canonical positions are obtained through interpolation. -Finally, the discrete ``u``-momentum equations are given by +Finally, the discrete ``\alpha``-momentum equations are given by ```math \begin{split} - \Delta x_{i + \frac{1}{2}} - \Delta y_j - \frac{\mathrm{d} }{\mathrm{d} t} u_{i + \frac{1}{2}, j} = - & - \Delta y_j \left( (u u)_{i + 1, j} - (u u)_{i, j} \right) \\ - & - \Delta x_{i + \frac{1}{2}} \left( (u v)_{i + \frac{1}{2}, j + - \frac{1}{2}} - (u v)_{i + \frac{1}{2}, j - \frac{1}{2}} \right) \\ - & - \Delta y_j \left( p_{i + 1, j} - p_{i, j} \right) \\ - & + \nu \Delta y_j \left( - \frac{u_{i + \frac{3}{2}, j} - u_{i + \frac{1}{2}, j}}{\Delta x_{i + 1}} - - \frac{u_{i + \frac{1}{2}, j} - u_{i - \frac{1}{2}, j}}{\Delta x_i} - \right) \\ - & + \nu \Delta x_{i + \frac{1}{2}} \left( - \frac{u_{i + \frac{1}{2}, j + 1} - u_{i + \frac{1}{2}, j}}{\Delta y_{j + \frac{1}{2}}} - - \frac{u_{i + \frac{1}{2}, j} - u_{i + \frac{1}{2}, j - 1}}{\Delta y_{j - \frac{1}{2}}} + \left| \Omega_{I + \delta(\alpha) / 2} \right| + \frac{\mathrm{d} }{\mathrm{d} t} u^\alpha_{I + \delta(\alpha) / 2} = + & - \sum_{\beta = 1}^d \left| \Gamma^\beta_{I + \delta(\alpha) / 2} \right| + \left( + (u^\alpha + u^\beta)_{I + \delta(\alpha) / 2 + \delta(\beta) / 2} + - + (u^\alpha + u^\beta )_{I + \delta(\alpha) / 2 - \delta(\beta) / 2} \right) \\ - & + - \Delta x_{i + \frac{1}{2}} - \Delta y_j - f_u \left( x_{i + \frac{1}{2}}, y_j \right). -\end{split} -``` - -A similar derivation over the ``v``-volume ``\Omega_{i, j + \frac{1}{2}}`` -yields the discrete ``v``-momentum equations - -```math -\begin{split} - \Delta x_i - \Delta y_{j + \frac{1}{2}} - \frac{\mathrm{d} }{\mathrm{d} t} v_{i, j + \frac{1}{2}} = - & - \Delta y_{j + \frac{1}{2}} \left( (v u)_{i + \frac{1}{2}, j + \frac{1}{2}} - - (v u)_{i - \frac{1}{2}, j + \frac{1}{2}} \right) \\ - & - \Delta x_i \left( (v v)_{i, j + 1} - (v v)_{i, j} \right) \\ - & - \Delta x_i \left( p_{i, j + 1} - p_{i, j} \right) \\ - & + \nu \Delta x_{i + \frac{1}{2}} \left( - \frac{v_{i + 1, j + \frac{1}{2}} - v_{i, j + \frac{1}{2}}}{\Delta x_{i + \frac{1}{2}}} - - \frac{v_{i, j + \frac{1}{2}} - u_{i - 1, j + \frac{1}{2}}}{\Delta x_{i - - \frac{1}{2}}} \right) \\ - & + \nu \Delta x_i \left( - \frac{v_{i, j + \frac{3}{2}} - u_{i, j + \frac{1}{2}}}{\Delta y_{j + 1}} - - \frac{v_{i, j + \frac{1}{2}} - v_{i, j - \frac{1}{2}}}{\Delta y_j} + & - \left| \Gamma^\alpha_{I + \delta(\alpha) / 2} \right| + \left( p_{I + \delta(\alpha)} - p_{I} \right) \\ + & + \nu \sum_{\beta = 1}^d \left| \Gamma^\beta_{I + \delta(\alpha) / 2} \right| + \left( + \frac{u^\alpha_{I + \delta(\alpha) / 2 + \delta(\beta)} - u^\alpha_{I + + \delta(\alpha) / 2}}{x^\beta_{I(\beta) + 1} - x^\beta_{I(\beta)}} + - \frac{u^\alpha_{I + \delta(\alpha) / 2} - u^\alpha_{I + \delta(\alpha) + / 2 - \delta(\beta)}}{x^\beta_{I(\beta)} - x^\beta_{I(\beta) - 1}} \right) \\ - & + - \Delta x_i - \Delta y_{j + \frac{1}{2}} - f_v \left( x_i, y_{j + \frac{1}{2}} \right). + & + \left| \Omega_{I + \delta(\alpha) / 2} \right| f^\alpha(x_{I + \delta(\alpha) / 2}). \end{split} ``` From c3e3ad3f95d4dab355fb4c5624b089acfa7a7268 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Fri, 8 Sep 2023 21:06:11 +0200 Subject: [PATCH 039/379] docs: Update --- docs/src/equations/spatial.md | 268 +++++++++++++++++++++++----------- 1 file changed, 179 insertions(+), 89 deletions(-) diff --git a/docs/src/equations/spatial.md b/docs/src/equations/spatial.md index 4c0d3b55d..ba14eaf5f 100644 --- a/docs/src/equations/spatial.md +++ b/docs/src/equations/spatial.md @@ -8,44 +8,54 @@ volumes on a staggered Cartesian grid, as proposed by Harlow and Welsh Let ``d \in \{2, 3\}`` denote the spatial dimension (2D or 3D). We will make use of the "Cartesian" index ``I = (i, j)`` in 2D or ``I = (i, j, k)`` in 3D, with ``I(1) = i``, ``I(2) = j``, and ``I(3) = k``. Here, the indices ``I``, -``i``, ``j``, and ``k``, represent discrete degrees of freedom. To specify a +``i``, ``j``, and ``k``, represent discrete degrees of freedom, and can take +integer or half values (e.g. ``3`` or ``5/2``). To specify a spatial dimension, we will use the symbols ``(\alpha, \beta, \gamma) \in \{1, \dots, d\}^3``. We will use the symbol ``\delta(\alpha) = (\delta_{\alpha \beta})_{\beta = 1}^d \in \{0, 1\}^d`` to indicate a perturbation in the direction ``\alpha``, where ``\delta_{\alpha \beta}`` is the Kronecker symbol. The spatial variable is ``x = (x^1, \dots, x^d) \in \Omega \subset -\mathbb{R}^d``. Note that ``\Omega = \prod_{\alpha = 1}^d [0, L^\alpha]`` is -assumed to have the shape of a box with side lengths ``L^\alpha > 0``. +\mathbb{R}^d``. + ## Finite volumes -The finite volumes are defined as +We here assume that ``\Omega = \prod_{\alpha = 1}^d [0, L^\alpha]`` has the +shape of a box with side lengths ``L^\alpha > 0``. This allows for partitioning +``\Omega`` into the finite volumes ```math \Omega_I = \prod_{\alpha = 1}^d \left[ x^\alpha_{I(\alpha) - \frac{1}{2}}, x^\alpha_{I(\alpha) + \frac{1}{2}} \right], \quad I \in \mathcal{I}. ``` -They represent rectangles in 2D and prisms in 3D. They are fully defined by the -vectors of volume faces ``x^\alpha = \left( x^\alpha_{i + \frac{1}{2}} -\right)_{i = 0}^{N(\alpha)}``, where ``N = (N(1), \dots, N(d)) \in -\mathbb{N}^d`` are the numbers of volumes in each dimension and ``\mathcal{I} = -\prod_{\alpha = 1}^d \{1, \dots, N(\alpha)\}`` the set of finite volume -indices. Note that the components ``x^\alpha_{i}`` are not assumed to be +Just like ``\Omega`` itself, they represent rectangles in 2D and prisms in 3D. +They are fully defined by the vectors of volume face coordinates ``x^\alpha = +\left( x^\alpha_{i} \right)_{i = 0}^{N(\alpha)} \in \mathbb{R}^{N(\alpha) + +1}``, where ``N = (N(1), \dots, N(d)) \in \mathbb{N}^d`` are the numbers of +finite volumes in each dimension and ``\mathcal{I} = \prod_{\alpha = 1}^d +\left\{ \frac{1}{2}, 2 - \frac{1}{2}, \dots, N(\alpha) - \frac{1}{2} \right\}`` +the set of finite volume indices (note that the reference volumes are indexed +by half indices only). The components ``x^\alpha_{i}`` are not assumed to be uniformly spaced. But we do assume that they are strictly increasing with -``i``. - -The volume center coordinates are determined from the volume boundaries by -``x^\alpha_{i} = \frac{1}{2} (x^\alpha_{i - \frac{1}{2}} + x_{i + -\frac{1}{2}})``. This allows for defining the shifted volumes ``\Omega_{I + -\delta(\alpha) / 2}``. +``i``, with ``x^\alpha_0 = 0`` and ``x^\alpha_{N(\alpha)} = L^\alpha``. + +The coordinates of the volume centers are determined from the those of the +volume boundaries by +``x^\alpha_{I(\alpha)} = \frac{1}{2} (x^\alpha_{I(\alpha) - \frac{1}{2}} + +x_{I(\alpha) + \frac{1}{2}})`` +for ``I \in \mathcal{I}``. This allows for defining shifted volumes such as +``\Omega_{I + \delta(\alpha) / 2}`` and +``\Omega_{I + \delta(\alpha) / 2 + \delta(\beta) / 2}``. The original volumes +(with indices in ``\mathcal{I}``) will be referred to as the reference finite +volumes. We also define the volume widths/depths/heights ``\Delta x^\alpha_i = x^\alpha_{i + \frac{1}{2}} - x^\alpha_{i - \frac{1}{2}}``, where ``i`` can take half values. The volume sizes are thus ``| \Omega_{I} | = \prod_{\alpha = 1}^d \Delta x^\alpha_{I(\alpha)}``. -In addition to the finite volumes and their half-indexed shifted variants, we +In addition to the finite volumes and their shifted variants, we define the surface ```math @@ -55,13 +65,13 @@ define the surface \text{otherwise}, \end{cases} ``` -where ``I`` can take half-values. It is the interface between ``\Omega_{I - -\delta(\alpha) / 2}`` and ``\Omega_{I + \delta(\alpha) / 2}``, and has surface -normal ``\delta(\alpha)``. - +where ``I`` can take integer or half-values. It is the interface between +``\Omega_{I - \delta(\alpha) / 2}`` and ``\Omega_{I + \delta(\alpha) / 2}``, +and has surface normal ``\delta(\alpha)``. -In each finite volume ``\Omega_{I}`` (integer ``I``), there are three different -types of positions in which quantities of interest can be defined: +In each reference finite volume ``\Omega_{I}`` (``I \in \mathcal{I}``), there +are three different types of positions in which quantities of interest can be +defined: - The volume center ``x_I = (x_{I(1)}, \dots, x_{I(d)})``, where the discrete pressure ``p_I`` is defined; @@ -72,24 +82,21 @@ types of positions in which quantities of interest can be defined: 2}`` is defined. The vectors of unknowns ``u^\alpha_h`` and ``p_h`` will not contain all the -half-index components, only those from their own canonical position. - -!!! note "Storage convention" - We use the column-major convention (Julia, MATLAB, Fortran), and not the - row-major convention (Python, C). Thus the ``x^1``-index ``i`` will vary for - one whole cycle in the vectors before the - ``x^2``-index ``j`` is incremented, e.g. ``p_h = (p_{(1, 1, 1)}, - p_{(2, 1, 1)}, \dots p_{(N(1), N(2), N(3))})`` in 3D. +half-index components, only those from their own canonical position. The +unknown discrete pressure represents the average pressure in each reference +volume, and the unknown discrete velocity components represent exchange of mass +between neighboring volumes. In 2D, this finite volume configuration is illustrated as follows: ![Grid](../assets/grid.png) + ## Interpolation When a quantity is required *outside* of its native point, we will use interpolation. Examples: -- To compute ``u^\alpha`` at the pressure point ``x_I``: +- To compute ``u^\alpha`` at the pressure point ``x_I``, ``I \in \mathcal{I}``: ```math \begin{split} u^\alpha_I & = \frac{x^\alpha_{I(\alpha) + 1 / 2} - @@ -122,12 +129,14 @@ When a quantity is required *outside* of its native point, we will use interpola p_{I + \delta(\alpha)} ``` + ## Finite volume discretization of the Navier-Stokes equations We will consider the integral form of the Navier-Stokes equations. This has the advantage that some of the spatial derivatives dissapear, reducing the amount of finite difference approximations we need to perform. + ### Mass equation The mass equation takes the form @@ -140,8 +149,9 @@ The mass equation takes the form Using the pressure volume ``\mathcal{O} = \Omega_{I}``, we get ```math -\sum_{\alpha = 1}^d \left( \int_{\Gamma^\alpha_{I + \delta(\alpha) / 2}} u^\alpha \, \mathrm{d} \Gamma - -\int_{\Gamma_{I - \delta(\alpha) / 2}^\alpha} u^\alpha \, \mathrm{d} \Gamma +\sum_{\alpha = 1}^d \left( \int_{\Gamma^\alpha_{I + \delta(\alpha) / 2}} +u^\alpha \, \mathrm{d} \Gamma - \int_{\Gamma_{I - \delta(\alpha) / 2}^\alpha} +u^\alpha \, \mathrm{d} \Gamma \right) = 0. ``` @@ -150,7 +160,8 @@ sufficiently small such that ``u`` is locally linear, we can perform the local approximation (quadrature) ```math -\int_{\Gamma^\alpha_I} u^\alpha \, \mathrm{d} \Gamma \approx | \Gamma^\alpha_I | u^\alpha_{I}. +\int_{\Gamma^\alpha_I} u^\alpha \, \mathrm{d} \Gamma \approx | \Gamma^\alpha_I +| u^\alpha_{I}. ``` This yields the discrete mass equation @@ -158,22 +169,14 @@ This yields the discrete mass equation ```math \sum_{\alpha = 1}^d \left| \Gamma^\alpha_I \right| \left( u^\alpha_{I + \delta(\alpha) / 2} - -u^\alpha_{I - \delta(\alpha) / 2} \right) = 0 -``` - -which can also be written in the matrix form - -```math -M V = \sum_{\alpha = 1}^d M^\alpha u^\alpha = 0, +u^\alpha_{I - \delta(\alpha) / 2} \right) = 0. ``` -where ``M = \begin{pmatrix} M_1 & \dots & M_d \end{pmatrix}`` is the discrete -divergence operator. - !!! note "Approximation error" For the mass equation, the only approximation we have performed is quadrature. No interpolation or finite difference error is present. + ### Momentum equations Grouping the convection, pressure gradient, diffusion, and body force terms in @@ -182,9 +185,11 @@ each of their own integrals, we get, for all ``\mathcal{O} \subset \Omega``: ```math \frac{\partial }{\partial t} \int_\mathcal{O} u^\alpha \, \mathrm{d} \Omega = -- \sum_{\beta = 1}^d \int_{\partial \mathcal{O}} u^\alpha u^\beta n^\beta \, \mathrm{d} \Gamma +- \sum_{\beta = 1}^d \int_{\partial \mathcal{O}} u^\alpha u^\beta n^\beta \, +\mathrm{d} \Gamma - \int_{\partial \mathcal{O}} p n^\alpha \, \mathrm{d} \Gamma -+ \nu \sum_{\beta = 1}^d \int_{\partial \mathcal{O}} \frac{\partial u^\alpha}{\partial x^\beta} n^\beta \, \mathrm{d} \Gamma ++ \nu \sum_{\beta = 1}^d \int_{\partial \mathcal{O}} \frac{\partial +u^\alpha}{\partial x^\beta} n^\beta \, \mathrm{d} \Gamma + \int_\mathcal{O} f^\alpha \mathrm{d} \Omega, ``` @@ -247,11 +252,13 @@ This equation is still exact. We now introduce some approximations on continuous quantities. 1. We replace the integrals with a mid-point quadrature rule. -1. The mid-point values of derivatives are approximated using a central-like finite difference: +1. The mid-point values of derivatives are approximated using a central-like + finite difference: ```math \frac{\partial u^\alpha}{\partial x^\beta}(x_I) \approx \frac{u^\alpha_{I + \delta(\beta) / 2} - - u^\alpha_{I - \delta(\beta) / 2}}{x^\beta_{I(\beta) + 1 / 2} - x^\beta_{I(\beta) - 1 / 2}}. + - u^\alpha_{I - \delta(\beta) / 2}}{x^\beta_{I(\beta) + 1 / 2} - + x^\beta_{I(\beta) - 1 / 2}}. ``` 1. Quantities outside their canonical positions are obtained through interpolation. @@ -272,68 +279,133 @@ Finally, the discrete ``\alpha``-momentum equations are given by \right) \\ & - \left| \Gamma^\alpha_{I + \delta(\alpha) / 2} \right| \left( p_{I + \delta(\alpha)} - p_{I} \right) \\ - & + \nu \sum_{\beta = 1}^d \left| \Gamma^\beta_{I + \delta(\alpha) / 2} \right| + & + \nu \sum_{\beta = 1}^d \left| \Gamma^\beta_{I + \delta(\alpha) / 2} + \right| \left( \frac{u^\alpha_{I + \delta(\alpha) / 2 + \delta(\beta)} - u^\alpha_{I + \delta(\alpha) / 2}}{x^\beta_{I(\beta) + 1} - x^\beta_{I(\beta)}} - - \frac{u^\alpha_{I + \delta(\alpha) / 2} - u^\alpha_{I + \delta(\alpha) - / 2 - \delta(\beta)}}{x^\beta_{I(\beta)} - x^\beta_{I(\beta) - 1}} + - \frac{u^\alpha_{I + \delta(\alpha) / 2} - u^\alpha_{I + + \delta(\alpha) / 2 - \delta(\beta)}}{x^\beta_{I(\beta)} + - x^\beta_{I(\beta) - 1}} \right) \\ - & + \left| \Omega_{I + \delta(\alpha) / 2} \right| f^\alpha(x_{I + \delta(\alpha) / 2}). + & + \left| \Omega_{I + \delta(\alpha) / 2} \right| f^\alpha(x_{I + + \delta(\alpha) / 2}). \end{split} ``` -In matrix form, we will denote this as - -```math -\Omega_h \frac{\mathrm{d} V_h}{\mathrm{d} t} = - C(V_h) + \nu D V_h + \Omega_h f_h - G p_h. -``` - -Note the important property ``G = M^\mathsf{T}``. - ## Boundary conditions -If a domain boundary is not periodic, the boundary values of certain quantities -are prescribed. Consider the left boundary defined by ``i = \frac{1}{2}``. - -- For Dirichlet boundary conditions, we prescribe the value +Depending on the type of boundary conditions, certain indices used in the left- +and right hand sides of the equations may not be part of the solution vectors, +even if they are indeed at their canonical positions. Consider for example the +``\alpha``-left/front/bottom boundary ``\{ x \in \Omega | x^{\alpha} = +x^\alpha_{0} \}``. Let ``\Omega_I`` be one of the reference finite volumes +touching this boundary, i.e. ``I(\alpha) = \frac{1}{2}``. + +- For periodic boundary conditions, we also consider the opposite boundary. + We add two "ghost" reference volumes, one at each side of ``\Omega``: + 1. The ghost volume ``\Omega_{I - \delta(\alpha)}`` has the same shape as + ``\Omega_{I + (N(\alpha) - 1) \delta(\alpha)}`` and contains the same + components: + - ``u^\alpha_{I - \delta(\alpha) / 2} = u^\alpha_{I + (N(\alpha) - 1 / 2) + \delta(\alpha)}``, + - ``u^\beta_{I + \delta(\beta) / 2 - \delta(\alpha)} = + u^\beta_{I + \delta(\beta) / 2 + N(\alpha) \delta(\alpha)}`` for ``\beta + \neq \alpha``, + - ``p_{I - \delta(\alpha)} = p_{I + (N(\alpha) - 1) \delta(\alpha)}``. + 2. The ghost volume ``\Omega_{I + N(\alpha) \delta(\alpha)}`` has the same + shape as ``\Omega_I`` and contains the same pressure and velocity + components. +- For Dirichlet boundary conditions, all the veloctity components are + prescribed. For the normal velocity component, this is straightforward: ```math - u_{\frac{1}{2}, j} := u(x_{\frac{1}{2}}, y_j) + u^\alpha_{I - \delta(\alpha) / 2} = u^\alpha(x_{I - \delta(\alpha) / 2}). ``` - For ``v``, we also prescribe - values, but only at the boundary, thus replacing otherwise interpolated - ``v``-fluxes: + The parallel (``\beta \neq \alpha``) velocity components + ``u^\beta_{I + \delta(\beta) / 2 - \delta(\alpha)}`` + appear in some of the right hand side expressions. Their ``\alpha`` + position ``x^\alpha_{- 1 / 2}`` has actually never been defined, + so we simply define it to be on the boundary itself: + ``x^\alpha_{- 1 / 2} = x^\alpha_0``. The value can then be + prescribed: ```math - v_{\frac{1}{2}, j - \frac{1}{2}} := v(x_{\frac{1}{2}}, y_{j - \frac{1}{2}}). + u^\beta_{I + \delta(\beta) / 2 - \delta(\alpha)} = u^\beta \left( x_{I + + \delta(\beta) / 2 - \delta(\alpha)} \right) = u^\beta \left( x_{I + + \delta(\beta) / 2 - \delta(\alpha) / 2} \right). ``` -- For a symmetric left boundary, only ``u_{\frac{1}{2}, j}`` is prescribed. -- For a pressure left boundary, we prescribe ``p``: + The pressure does not require any boundary modifications. +- For a symmetric boundary, the normal component has zero Dirichlet + conditions, while the parallel components have zero Neumann conditions. For + this, we add a ghost volume ``\Omega_{I - \delta(\alpha)}`` which has the + same shape as ``\Omega_{I}``, meaning that ``x^\alpha_{-1} = x^\alpha_0 - + (x^\alpha_1 - x^\alpha_0) = - x^\alpha_1``. The pressure in this volume is + never used, but we set ``u^\alpha_{I - \delta(\alpha) / 2} = 0`` and + ``u^\beta_{I + \delta(\beta) / 2 - \delta(\alpha)} = u^\beta_{I + + \delta(\beta) / 2}`` for ``\beta \neq \alpha``. +- For a pressure boundary, the value of the pressure is prescribed, while the + velocity has zero Neumann boundary conditions. For this, we add an + infinitely thin ghost volume ``\Omega_{I - \delta(\alpha)}``, by setting + ``x^\alpha_{-1} = x^\alpha_0 - \epsilon`` for some epsilon. We then let this + thickness go to zero *after* substituting the boundary conditions in + the momentum equations. The pressure is prescribed by ```math - p_{0, j} = p(x_0, y_j). + p_{I - \delta(\alpha)} = p(x_{I - \delta(\alpha)}) \underset{\epsilon \to 0}{\to} p(x_{I - \delta(\alpha) / 2}). ``` + The Neumann boundary conditions are obtained by setting + ``u^\alpha_{I - 3 / 2 \delta(\alpha)} = u^\alpha_{I - \delta(\alpha) / 2}`` + and + ``u^\beta_{I - \delta(\alpha) + \delta(\beta) / 2} = u^\beta_{I + + \delta(\beta) / 2}`` for ``\beta \neq \alpha``. + Note that the normal velocity component at the boundary ``u^\alpha_{I - + \delta(\alpha) / 2}`` is now a degree of freedom, and we need to + observe that the first normal derivative in the diffusion term of its + corresponding momentum equation is zero before taking the limit as ``\epsilon + \to 0``, to avoid dividing by zero. + +It should now be clear from the above cases which components of the discrete +velocity and pressure are unknown degrees of freedom, and which components are +prescribed or obtained otherwise. The *unknown* degrees of freedom are stored +in the vectors ``u_h = (u^1_h, \dots, u^d_h)`` and ``p_h`` using the +column-major convention. Note that the ``d`` discrete velocity fields ``u^1_h, +\dots u^d_h`` may have different numbers of elements. + +!!! note "Storage convention" + We use the column-major convention (Julia, MATLAB, Fortran), and not the + row-major convention (Python, C). Thus the ``x^1``-index ``i`` will vary for + one whole cycle in the vectors before the + ``x^2``-index ``j``, ``x^3`` index ``k``, and component-index ``\alpha`` + are incremented, e.g. ``u_h = (u^1_{(1, 1, 1)}, + u^1_{(2, 1, 1)}, \dots u^3_{(N_{u^3}(1), N_{u^3}(2), N_{u^3}(3))})`` in 3D. -The boundary components are removed from the two vectors ``V_h`` and ``p_h``. -Instead, these components are prescribed as constants. -The discrete mass equation then becomes +## Matrix representation + +We can write the mass and momentum equations in matrix form. The discrete mass +equation then becomes ```math -M V_h = y_M, +M u_h = y_M, ``` -where ``y_M`` are the boundary conditions for ``M``. +where ``M`` is the discrete divergence operator and ``y_M`` contains the +boundary value contributions of the velocity to the divergence field. The discrete momentum equations become ```math \begin{split} - \Omega_h \frac{\mathrm{d} V_h}{\mathrm{d} t} & = -C(V_h) V_h + \nu (D V_h + + \Omega_h \frac{\mathrm{d} u_h}{\mathrm{d} t} & = -C(u_h) + \nu (D u_h + y_D) + f_h - (G p_h + y_G) \\ & = F(V_h) - (G p_h + y_G), \end{split} ``` -where ``y_D`` is diffusion boundary vector and ``y_G`` is the -pressure boundary vector. +where ``\Omega_h`` is a diagonal matrix containing the velocity volumes, ``C`` +is the convection operator (including boundary contributions), ``D`` is the +diffusion operator, ``y_D`` is boundary contribution to the diffusion term, ``G += M^\mathsf{T}`` is the pressure gradient operator, and ``y_G`` contains the +boundary contribution of the pressure to the pressure gradient (only non-zero +for pressure boundary conditions). The term ``F`` refers to all the forces +except for the pressure gradient. ## Discrete pressure Poisson equation @@ -345,28 +417,46 @@ discrete divergence operator ``M`` to the discrete momentum equations yields the discrete pressure Poisson equation ```math -- L p_h = - M \Omega_h^{-1} (F(V_h) - y_G) + \frac{\mathrm{d} y_M}{\mathrm{d} t} +L p_h = M \Omega_h^{-1} (F(V_h) - y_G) - \frac{\mathrm{d} y_M}{\mathrm{d} t}, ``` where ``L = M \Omega_h^{-1} G`` is a discrete Laplace operator. It is positive symmetric since ``G = M^\mathsf{T}``. +!!! note "Unsteady Dirichlet boundary conditions" + + If the equations are prescribed with unsteady Dirichlet boundary + conditions, for example an inflow that varies with time, the term + ``\frac{\mathrm{d} y_M}{\mathrm{d} t}`` will be non-zero. If this term is + not known exactly, for example if the next value of the inflow is unknown at + the time of the current value, it must be computed using past values of + of the velocity inflow only, for example ``\frac{\mathrm{d} y_M}{\mathrm{d} + t} \approx (y_M(t) - y_M(t - \Delta t) / \Delta t``. + +!!! note "Uniqueness of pressure field" + + Unless pressure boundary conditions are present, the pressure is only + determined up to a constant, as ``L`` will have an eigenvalue of zero. + Since only the gradient of the pressure appears in the equations, we can + set the unknown constant to zero without affecting the velocity field. + !!! note "Pressure projection" - The pressure vector ``p_h`` can be seen as a Lagrange multiplier enforcing - the constraint of divergence freeness. It is possible to write a the - momentum equations without the pressure by explicitly solving the discrete - Poisson equation: + + The pressure field ``p_h`` can be seen as a Lagrange multiplier enforcing + the constraint of discrete divergence freeness. It is also possible to + write the momentum equations without the pressure by explicitly solving the + discrete Poisson equation: ```math - p_h = L^{-1} M \Omega_h^{-1} (F(V_h) - y_G) - L^{-1} \frac{\mathrm{d} y_M}{\mathrm{d} t}. + p_h = L^{-1} M \Omega_h^{-1} (F(u_h) - y_G) - L^{-1} \frac{\mathrm{d} y_M}{\mathrm{d} t}. ``` The momentum equations then become ```math - \Omega_h \frac{\mathrm{d} V_h}{\mathrm{d} t} = (I - G L^{-1} M \Omega_h^{-1}) - (F(V_h) - y_G) + G L^{-1} \frac{\mathrm{d} y_M}{\mathrm{d} t}. + \Omega_h \frac{\mathrm{d} u_h}{\mathrm{d} t} = (I - G L^{-1} M \Omega_h^{-1}) + (F(u_h) - y_G) + G L^{-1} \frac{\mathrm{d} y_M}{\mathrm{d} t}. ``` The matrix ``(I - G L^{-1} M \Omega^{-1})`` is a projector onto the space From b2e47e5d398bde86033c55567bb48086e3a602ad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Fri, 8 Sep 2023 21:36:33 +0200 Subject: [PATCH 040/379] docs: Update equations --- docs/src/equations/ns.md | 117 ++++++++++++++++----------------------- 1 file changed, 49 insertions(+), 68 deletions(-) diff --git a/docs/src/equations/ns.md b/docs/src/equations/ns.md index ea9b299a9..1904172e5 100644 --- a/docs/src/equations/ns.md +++ b/docs/src/equations/ns.md @@ -1,57 +1,31 @@ # Incompressible Navier-Stokes equations -The incompressible Navier-Stokes equations are comprised of a mass equation and -two or three momentum equations. In conservative form, they are given by +Let ``d \in \{2, 3\}`` denote the spatial dimension, and ``\Omega \subset +\mathbb{R}^d`` some spatial domain. The incompressible Navier-Stokes equations +on ``\Omega`` are comprised of a mass equation and ``d`` momentum equations. In +conservative form, they are given by ```math \begin{align*} -\nabla \cdot V & = 0, \\ -\frac{\mathrm{d} V}{\mathrm{d} t} + \nabla \cdot (V V^\mathsf{T}) & = -\nabla p + -\nu \nabla^2 V + f. +\nabla \cdot u & = 0, \\ +\frac{\mathrm{d} u}{\mathrm{d} t} + \nabla \cdot (u u^\mathsf{T}) & = -\nabla p + +\nu \nabla^2 u + f. \end{align*} ``` -where ``V = (u, v)`` or ``V = (u, v, w)`` is the velocity field, ``p`` is the -pressure, ``\nu`` is the kinematic viscosity, and ``f = (f_u, f_v)`` or ``f = -(f_u, f_v, f_w)`` is the body force per unit of volume. In 2D, the equations -become +where ``u = (u^1, \dots, u^d)`` is the velocity field, ``p`` is the +pressure, ``\nu`` is the kinematic viscosity, and ``f = (f^1, \dots, f^d)`` is +the body force per unit of volume. In scalar notation, the equations can be +written as ```math -\begin{split} - \frac{\partial u}{\partial x} + \frac{\partial v}{\partial y} & = 0, \\ - \frac{\partial u}{\partial t} + \frac{\partial (u u)}{\partial x} + - \frac{\partial (v u)}{\partial y} & = - \frac{\partial p}{\partial x} + - \nu \left( \frac{\partial^2 u}{\partial x^2} + \frac{\partial^2 - u}{\partial y^2} \right) + f_u, \\ - \frac{\partial v}{\partial t} + \frac{\partial (u v)}{\partial x} + - \frac{\partial (v v)}{\partial y} & = - \frac{\partial p}{\partial y} + - \nu \left( \frac{\partial^2 v}{\partial x^2} + \frac{\partial^2 - v}{\partial y^2} \right) + f_v. -\end{split} -``` - -In 3D, the equations become - -```math -\begin{split} - \frac{\partial u}{\partial x} + \frac{\partial v}{\partial y} + - \frac{\partial w}{\partial z} & = 0, \\ - \frac{\partial u}{\partial t} + \frac{\partial (u u)}{\partial x} + - \frac{\partial (v u)}{\partial y} + \frac{\partial (w u)}{\partial z} & = - - \frac{\partial p}{\partial x} + \nu \left( \frac{\partial^2 u}{\partial - x^2} + \frac{\partial^2 u}{\partial y^2} + \frac{\partial^2 u}{\partial - z^2} \right) + f_u, \\ - \frac{\partial v}{\partial t} + \frac{\partial (u v)}{\partial x} + - \frac{\partial (v v)}{\partial y} + \frac{\partial (w v)}{\partial z} & = - - \frac{\partial p}{\partial y} + \nu \left( \frac{\partial^2 v}{\partial - x^2} + \frac{\partial^2 v}{\partial y^2} + \frac{\partial^2 v}{\partial - z^2} \right) + f_v, \\ - \frac{\partial w}{\partial t} + \frac{\partial (u w)}{\partial x} + - \frac{\partial (v w)}{\partial y} + \frac{\partial (w w)}{\partial z} & = - - \frac{\partial p}{\partial z} + \nu \left( \frac{\partial^2 w}{\partial - x^2} + \frac{\partial^2 w}{\partial y^2} + \frac{\partial^2 w}{\partial - z^2} \right) + f_w. \\ -\end{split} +\begin{align*} +\sum_{\beta = 1}^d \frac{\partial u^\beta}{\partial x^\beta} & = 0, \\ +\frac{\partial u^\alpha}{\partial t} + \sum_{\beta = 1}^d +\frac{\partial}{\partial x^\beta} (u^\alpha u^\beta) & = -\frac{\partial +p}{\partial x^\alpha} + \nu \sum_{\beta = 1}^d \frac{\partial^2 u}{\partial +(x^\beta)^2} + f^\alpha. +\end{align*} ``` @@ -65,55 +39,62 @@ be denoted ``\partial \mathcal{O}``, with normal ``n`` and surface element The mass equation in integral form is given by ```math -\int_{\partial \mathcal{O}} V \cdot n \, \mathrm{d} \Gamma = 0. +\int_{\partial \mathcal{O}} u \cdot n \, \mathrm{d} \Gamma = 0. ``` where we have used the divergence theorem to convert the volume integral to a surface integral. Similarly, the momentum equations take the form ```math -\frac{\partial }{\partial t} \int_\mathcal{O} V \, \mathrm{d} \Omega -= \int_{\partial \mathcal{O}} \left( - V V^\mathsf{T} - P + \nu S \right) \cdot n \, +\frac{\partial }{\partial t} \int_\mathcal{O} u \, \mathrm{d} \Omega += \int_{\partial \mathcal{O}} \left( - u u^\mathsf{T} - P + \nu S \right) \cdot n \, \mathrm{d} \Gamma + \int_\mathcal{O} f \mathrm{d} \Omega ``` -where ``P = p \mathrm{I} = \operatorname{diag}(p, p)`` is the hydrostatic stress tensor -and ``S = \nabla V + (\nabla V)^\mathsf{T}`` is the strain-rate tensor. Here we +where ``P = p \mathrm{I}`` is the hydrostatic stress tensor +and ``S = \nabla u + (\nabla u)^\mathsf{T}`` is the strain-rate tensor. Here we have once again used the divergence theorem. !!! note "Strain-rate tensor" The second term in the strain rate tensor is optional, as ```math - \int_{\partial \mathcal{O}} (\nabla V)^\mathsf{T} \cdot n \, \mathrm{d} \Gamma = 0 + \int_{\partial \mathcal{O}} (\nabla u)^\mathsf{T} \cdot n \, \mathrm{d} \Gamma = 0 ``` - due to the divergence freeness of ``V`` (mass equation). + due to the divergence freeness of ``u`` (mass equation). ## Boundary conditions -Because we will use a Cartesian grid for discretization, the fields ``V``, -``p``, and ``f`` are defined over the rectangular or prismatic domain ``\Omega -= [x_1, x_2] \times [y_1, y_2]`` or ``\Omega = [x_1, x_2] \times [y_1, y_2] -\times [z_1, z_2]``. Along each of the two or three dimensions, we allow for -the following boundary conditions (here illustrated on the first boundary of -the first dimension) - -- Periodic: ``V(x = x_1) = V(x = x_2)`` and ``p(x = x_1) = p(x = x_2)`` -- Dirichlet: ``V(x = x_1) = V_1`` -- Pressure: ``p(x = x_1) = p_1`` -- Symmetric (no movement through boundary ``x = x_1``, but free movement along - it): ``u(x = x_1) = 0`` +Consider a part ``\Gamma`` of the total boundary ``\partial \Omega``, with +normal ``n``. We allow for the following types of boundary conditions on +``\Gamma``: + +- Periodic boundary conditions: ``u(x) = u(x + \tau)`` and ``p(x) = p(x + \tau)`` for ``x \in + \Gamma``, where ``\tau`` is a constant translation to another part of the + boundary ``\partial \Omega``. This usually requires ``\Gamma`` and its + periodic counterpart ``\Gamma + \tau`` to be flat (including in this software + suite). +- Dirichlet boundary conditions: ``u = u_\text{BC}`` on ``\Gamma``. A common + situation here is that of a sticky wall, with "no slip boundary conditions. + Then ``u_\text{BC} = 0``. +- Symmetric boundary conditions: Same velocity field at each side. This implies + zero Dirichlet conditions for the normal component (``n \cdot u = 0``), and + zero Neumann conditions for the parallel component: ``n \cdot \nabla (u - (n + \cdot u) n) = 0``. +- Pressure boundary conditions: The pressure is prescribed (``p = + p_\text{BC}``), while the velocity has zero Neumann conditions: + ``n \cdot \nabla u = 0``. ## Pressure equation -Taking the divergence of the two or tree momemtum equations yields a Poisson +Taking the divergence of the momemtum equations yields a Poisson equation for the pressure: ```math -- \nabla^2 p = \nabla \cdot \left( \nabla \cdot (V V^\mathsf{T}) \right) - +- \nabla^2 p = \nabla \cdot \left( \nabla \cdot (u u^\mathsf{T}) \right) - \nabla \cdot f ``` @@ -157,7 +138,7 @@ a constant. We set this constant to ``1``. ### Vorticity -The vorticity is defined as ``\omega = \nabla \times V``. +The vorticity is defined as ``\omega = \nabla \times u``. In 2D, it is a scalar field given by @@ -189,7 +170,7 @@ u = \frac{\partial \psi}{\partial y}, \quad v = \frac{\partial \psi}{\partial x} In 3D, the stream function ``\psi`` is a vector field defined such that ```math -V = \nabla \times \psi +u = \nabla \times \psi ``` It is related to the 3D vorticity as @@ -200,7 +181,7 @@ It is related to the 3D vorticity as ### Kinetic energy -The kinetic energy is defined by ``e = \frac{1}{2} \| V \|^2``. +The kinetic energy is defined by ``e = \frac{1}{2} \| u \|^2``. ### Reynolds number From d24163ddf58ae2178e2988eb7709d6ef78d09a79 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Sun, 10 Sep 2023 17:04:48 +0200 Subject: [PATCH 041/379] docs: Update time discretization --- docs/src/equations/spatial.md | 10 +++--- docs/src/equations/time.md | 67 ++++++++++++++++++++++++++++++++--- 2 files changed, 67 insertions(+), 10 deletions(-) diff --git a/docs/src/equations/spatial.md b/docs/src/equations/spatial.md index ba14eaf5f..884c5986f 100644 --- a/docs/src/equations/spatial.md +++ b/docs/src/equations/spatial.md @@ -384,7 +384,7 @@ column-major convention. Note that the ``d`` discrete velocity fields ``u^1_h, We can write the mass and momentum equations in matrix form. The discrete mass equation then becomes ```math -M u_h = y_M, +M u_h + y_M = 0, ``` where ``M`` is the discrete divergence operator and ``y_M`` contains the boundary value contributions of the velocity to the divergence field. @@ -417,7 +417,7 @@ discrete divergence operator ``M`` to the discrete momentum equations yields the discrete pressure Poisson equation ```math -L p_h = M \Omega_h^{-1} (F(V_h) - y_G) - \frac{\mathrm{d} y_M}{\mathrm{d} t}, +L p_h = M \Omega_h^{-1} (F(V_h) - y_G) + \frac{\mathrm{d} y_M}{\mathrm{d} t}, ``` where ``L = M \Omega_h^{-1} G`` is a discrete Laplace operator. It is positive @@ -431,7 +431,7 @@ symmetric since ``G = M^\mathsf{T}``. not known exactly, for example if the next value of the inflow is unknown at the time of the current value, it must be computed using past values of of the velocity inflow only, for example ``\frac{\mathrm{d} y_M}{\mathrm{d} - t} \approx (y_M(t) - y_M(t - \Delta t) / \Delta t``. + t} \approx (y_M(t) - y_M(t - \Delta t)) / \Delta t`` for some ``\Delta t``. !!! note "Uniqueness of pressure field" @@ -449,14 +449,14 @@ symmetric since ``G = M^\mathsf{T}``. discrete Poisson equation: ```math - p_h = L^{-1} M \Omega_h^{-1} (F(u_h) - y_G) - L^{-1} \frac{\mathrm{d} y_M}{\mathrm{d} t}. + p_h = L^{-1} M \Omega_h^{-1} (F(u_h) - y_G) + L^{-1} \frac{\mathrm{d} y_M}{\mathrm{d} t}. ``` The momentum equations then become ```math \Omega_h \frac{\mathrm{d} u_h}{\mathrm{d} t} = (I - G L^{-1} M \Omega_h^{-1}) - (F(u_h) - y_G) + G L^{-1} \frac{\mathrm{d} y_M}{\mathrm{d} t}. + (F(u_h) - y_G) - G L^{-1} \frac{\mathrm{d} y_M}{\mathrm{d} t}. ``` The matrix ``(I - G L^{-1} M \Omega^{-1})`` is a projector onto the space diff --git a/docs/src/equations/time.md b/docs/src/equations/time.md index 62db92c62..a914590e3 100644 --- a/docs/src/equations/time.md +++ b/docs/src/equations/time.md @@ -4,15 +4,72 @@ The spatially discretized Navier-Stokes equations form a differential-algebraic system, with an ODE for the velocity ```math -\Omega \frac{\mathrm{d} V_h}{\mathrm{d} t} = F(V_h) - (G p_h + y_G) +\Omega \frac{\mathrm{d} u_h}{\mathrm{d} t} = F(u_h) - (G p_h + y_G) ``` -and an algebraic equation for the pressure +subject to the algebraic constraint formed by the mass equation ```math -A p_h = g. +M u_h + y_M = 0. ``` -## Runge-Kutta methods +In the end of the [previous section](spatial.md), we differentiated the mass +equation in time to obtain a discrete pressure Poisson equation. This equation +includes the term ``\frac{\mathrm{d} y_M}{\mathrm{d} t}``, which is non-zero if +an unsteady flow of mass is added to the domain (Dirichlet boundary +conditions). This term ensures that the time-continuous discrete velocity field +``u_h(t)`` stays divergence free (conserves mass). However, if we directly +discretize this system in time, the mass preservation may actually not be +respected. For this, we will change the definition of the pressure such that +the time-discretized velocity field is divergence free at each time step and +each time sub-step (to be defined in the following). + +Consider the interval ``[0, T]``. for some simulation time ``T``. We will +divide it into ``N`` sub-intervals ``[t^n, t^{n + 1}]`` for ``n = 0, \dots, N - +1``, with ``t^0 = 0``, ``t^N = T``, and increment ``\Delta t^n = t^{n + 1} - +t^n``. We define ``U^n \approx u_h(t_n)`` as an approximation to the exact +discrete velocity field ``u_h(t_n)``, with ``U^0 = u_h(0)`` starting from the +exact initial conditions. + +## Explicit Runge-Kutta methods + +See Sanderse [Sanderse2012](@cite). + +For explicit Runge-Kutta methods, we divide each time step into ``s`` sub-steps. +For ``i = 1, \dots, s``, we define the following quantities: + +```math +\begin{split} +U_0^n & = U^n \\ +\Delta t_i^n & = c_i \Delta t^n \\ +t_i^n & = t^n + \Delta t_i^n \\ +F_i & = F(U_{i - 1}^n, t_{i - 1}^n) \\ +V_i^n & = U^n + \Delta t^n \sum_{j = 1}^i a_{i j} F_j \\ +L P_i^n & = \frac{(M V_i^n + y_M(t_i^n)) - (M U^n + y_M(t^n))}{\Delta t_i^n} \\ +& = \frac{1}{c_i} \sum_{j = 1}^i a_{i j} F_j + +\frac{y_M(t_i^n) - y_M(t^n)}{\Delta t_i^n} \\ +U_i^n & = V_i^n - \Delta t_i^n G P_i^n, +\end{split} +``` + +where ``(a_ij)_{i j}`` are the Butcher tableau coefficients of the RK-method, +with the convention ``c_i = \sum_{j = 1}^i a_{i j}`` and ``c_0 = 0``. + +Finally, we set ``U^{n + 1} = U_s^n``. The corresponding pressure ``P^{n + 1}`` +can be calculated to the same accuracy as ``U^{n + 1}`` by doing an additional +pressure projection (if we know ``\frac{\mathrm{d} y_M}{\mathrm{d} t}(t^{n + +1})``), or to first order accuracy by simply taking ``P^{n + 1} = P_s^n``. + +Note that each of the sub-step velocities ``U_i^n`` is divergence free, after +projecting the tentative velocities ``V_i^s``. This is ensured due to the +judiciously chosen replacement of ``\frac{\mathrm{d} y_M}{\mathrm{d} t}(t_i^n)`` with +``(y_M(t_i^n) - y_M(t^n)) / \Delta t_i^n``. + +## Implicit Runge-Kutta methods + +See Sanderse [Sanderse2013](@cite). + +## Adams-Bashforth Crank-Nicolson method + +## One-leg beta method -See Sanderse [Sanderse2012](@cite) [Sanderse2013](@cite). From 185ca63d74ab46fe67182550359c6d69c40a419e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Sun, 10 Sep 2023 17:05:05 +0200 Subject: [PATCH 042/379] Add check in CFL --- src/solvers/get_timestep.jl | 120 ++++++++++++++++++------------------ 1 file changed, 60 insertions(+), 60 deletions(-) diff --git a/src/solvers/get_timestep.jl b/src/solvers/get_timestep.jl index 6097db59a..c17ae5ef6 100644 --- a/src/solvers/get_timestep.jl +++ b/src/solvers/get_timestep.jl @@ -24,33 +24,33 @@ function get_timestep(::Dimension{2}, stepper, cfl) vₕ = @view V[indv] # For explicit methods only - if isexplicit(method) - # Convective part - Cu = - Cux * spdiagm(Iu_ux * uₕ + yIu_ux) * Au_ux + - Cuy * spdiagm(Iv_uy * vₕ + yIv_uy) * Au_uy - Cv = - Cvx * spdiagm(Iu_vx * uₕ + yIu_vx) * Av_vx + - Cvy * spdiagm(Iv_vy * vₕ + yIv_vy) * Av_vy - C = blockdiag(Cu, Cv) - test = spdiagm(1 ./ Ω) * C - sum_conv = abs.(test) * ones(T, NV) - diag(abs.(test)) - diag(test) - λ_conv = maximum(sum_conv) - - # Based on max. value of stability region (not a very good indication - # For the methods that do not include the imaginary axis) - Δt_conv = lambda_conv_max(method) / λ_conv - - ## Diffusive part - test = Diagonal(1 ./ Ω) * Diff - sum_diff = abs.(test) * ones(T, NV) - diag(abs.(test)) - diag(test) - λ_diff = maximum(sum_diff) - - # Based on max. value of stability region - Δt_diff = lambda_diff_max(method) / λ_diff - - Δt = cfl * min(Δt_conv, Δt_diff) - end + @assert isexplicit(method) "Adaptive timestep requires explicit method" + + # Convective part + Cu = + Cux * spdiagm(Iu_ux * uₕ + yIu_ux) * Au_ux + + Cuy * spdiagm(Iv_uy * vₕ + yIv_uy) * Au_uy + Cv = + Cvx * spdiagm(Iu_vx * uₕ + yIu_vx) * Av_vx + + Cvy * spdiagm(Iv_vy * vₕ + yIv_vy) * Av_vy + C = blockdiag(Cu, Cv) + test = spdiagm(1 ./ Ω) * C + sum_conv = abs.(test) * ones(T, NV) - diag(abs.(test)) - diag(test) + λ_conv = maximum(sum_conv) + + # Based on max. value of stability region (not a very good indication + # For the methods that do not include the imaginary axis) + Δt_conv = lambda_conv_max(method) / λ_conv + + ## Diffusive part + test = Diagonal(1 ./ Ω) * Diff + sum_diff = abs.(test) * ones(T, NV) - diag(abs.(test)) - diag(test) + λ_diff = maximum(sum_diff) + + # Based on max. value of stability region + Δt_diff = lambda_diff_max(method) / λ_diff + + Δt = cfl * min(Δt_conv, Δt_diff) Δt end @@ -75,39 +75,39 @@ function get_timestep(::Dimension{3}, stepper, cfl) wₕ = @view V[indw] # For explicit methods only - if isexplicit(method) - # Convective part - Cu = - Cux * spdiagm(Iu_ux * uₕ + yIu_ux) * Au_ux + - Cuy * spdiagm(Iv_uy * vₕ + yIv_uy) * Au_uy + - Cuz * spdiagm(Iw_uz * wₕ + yIw_uz) * Au_uz - Cv = - Cvx * spdiagm(Iu_vx * uₕ + yIu_vx) * Av_vx + - Cvy * spdiagm(Iv_vy * vₕ + yIv_vy) * Av_vy + - Cvz * spdiagm(Iw_vz * wₕ + yIw_vz) * Av_vz - Cw = - Cwx * spdiagm(Iu_wx * uₕ + yIu_wx) * Aw_wx + - Cwy * spdiagm(Iv_wy * vₕ + yIv_wy) * Aw_wy + - Cwz * spdiagm(Iw_wz * wₕ + yIw_wz) * Aw_wz - C = blockdiag(Cu, Cv, Cw) - test = spdiagm(1 ./ Ω) * C - sum_conv = abs.(test) * ones(T, NV) - diag(abs.(test)) - diag(test) - λ_conv = maximum(sum_conv) - - # Based on max. value of stability region (not a very good indication - # For the methods that do not include the imaginary axis) - Δt_conv = lambda_conv_max(method) / λ_conv - - ## Diffusive part - test = Diagonal(1 ./ Ω) * Diff - sum_diff = abs.(test) * ones(T, NV) - diag(abs.(test)) - diag(test) - λ_diff = maximum(sum_diff) - - # Based on max. value of stability region - Δt_diff = lambda_diff_max(method) / λ_diff - - Δt = cfl * min(Δt_conv, Δt_diff) - end + @assert isexplicit(method) "Adaptive timestep requires explicit method" + + # Convective part + Cu = + Cux * spdiagm(Iu_ux * uₕ + yIu_ux) * Au_ux + + Cuy * spdiagm(Iv_uy * vₕ + yIv_uy) * Au_uy + + Cuz * spdiagm(Iw_uz * wₕ + yIw_uz) * Au_uz + Cv = + Cvx * spdiagm(Iu_vx * uₕ + yIu_vx) * Av_vx + + Cvy * spdiagm(Iv_vy * vₕ + yIv_vy) * Av_vy + + Cvz * spdiagm(Iw_vz * wₕ + yIw_vz) * Av_vz + Cw = + Cwx * spdiagm(Iu_wx * uₕ + yIu_wx) * Aw_wx + + Cwy * spdiagm(Iv_wy * vₕ + yIv_wy) * Aw_wy + + Cwz * spdiagm(Iw_wz * wₕ + yIw_wz) * Aw_wz + C = blockdiag(Cu, Cv, Cw) + test = spdiagm(1 ./ Ω) * C + sum_conv = abs.(test) * ones(T, NV) - diag(abs.(test)) - diag(test) + λ_conv = maximum(sum_conv) + + # Based on max. value of stability region (not a very good indication + # For the methods that do not include the imaginary axis) + Δt_conv = lambda_conv_max(method) / λ_conv + + ## Diffusive part + test = Diagonal(1 ./ Ω) * Diff + sum_diff = abs.(test) * ones(T, NV) - diag(abs.(test)) - diag(test) + λ_diff = maximum(sum_diff) + + # Based on max. value of stability region + Δt_diff = lambda_diff_max(method) / λ_diff + + Δt = cfl * min(Δt_conv, Δt_diff) Δt end From aa7fe3c4d24996e54e988b364e10733f1a1b430f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Sun, 10 Sep 2023 17:05:27 +0200 Subject: [PATCH 043/379] Rename variable --- src/time_steppers/step_explicit_runge_kutta.jl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/time_steppers/step_explicit_runge_kutta.jl b/src/time_steppers/step_explicit_runge_kutta.jl index bb25f37ac..62619d75e 100644 --- a/src/time_steppers/step_explicit_runge_kutta.jl +++ b/src/time_steppers/step_explicit_runge_kutta.jl @@ -58,7 +58,7 @@ function step(method::ExplicitRungeKuttaMethod, stepper, Δt) # Update velocity current stage by sum of Fᵢ's until this stage, weighted # with Butcher tableau coefficients. This gives uᵢ₊₁, and for i=s gives uᵢ₊₁ # V = dot(kV * A[i, 1:i] - V = sum(A[i, j] * kV[j] for j = 1:i) + k = sum(A[i, j] * kV[j] for j = 1:i) # Boundary conditions at tᵢ₊₁ tᵢ = tₙ + c[i] * Δtₙ @@ -68,7 +68,7 @@ function step(method::ExplicitRungeKuttaMethod, stepper, Δt) (; yM) = bc_vectors # Divergence of intermediate velocity field - f = (M * (Vₙ / Δtₙ + V) + yM / Δtₙ) / c[i] + f = (M * (Vₙ / Δtₙ + k) + yM / Δtₙ) / c[i] # Solve the Poisson equation, but not for the first step if the boundary conditions are steady if boundary_conditions.bc_unsteady || i > 1 @@ -81,7 +81,7 @@ function step(method::ExplicitRungeKuttaMethod, stepper, Δt) Gp = G * p # Update velocity current stage, which is now divergence free - V = @. Vₙ + Δtₙ * (V - c[i] / Ω * Gp) + V = @. Vₙ + Δtₙ * (k - c[i] / Ω * Gp) end # For steady bc we do an additional pressure solve From 206d48b076f523f6a587d4abd2d0cf7fde8dcd89 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Sun, 10 Sep 2023 19:48:22 +0200 Subject: [PATCH 044/379] docs: Update time integration --- docs/references.bib | 30 +++++++ docs/src/equations/time.md | 155 +++++++++++++++++++++++++++++------ src/time_steppers/methods.jl | 55 +++---------- 3 files changed, 170 insertions(+), 70 deletions(-) diff --git a/docs/references.bib b/docs/references.bib index 30ea361f5..82b05adab 100644 --- a/docs/references.bib +++ b/docs/references.bib @@ -153,3 +153,33 @@ @article{Sanderse2020 volume = {421}, year = {2020} } +@article{Verstappen1997, + doi = {10.1023/a:1004255329158}, + url = {https://doi.org/10.1023%2Fa%3A1004255329158}, + year = 1997, + publisher = {Springer Science and Business Media {LLC}}, + volume = {32}, + number = {2/3}, + pages = {143--159}, + author = {R.W.C.P. Verstappen and A.E.P. Veldman}, + journal = {Journal of Engineering Mathematics}, + title = {Direct Numerical Simulation of Turbulence at Lower Costs} +} +@article{Verstappen2003, + author = {Verstappen, R. W. C. P. and Veldman, A. E. P.}, + title = {Symmetry-Preserving Discretization of Turbulent Flow}, + year = {2003}, + issue_date = {May 2003}, + publisher = {Academic Press Professional, Inc.}, + address = {USA}, + volume = {187}, + number = {1}, + issn = {0021-9991}, + url = {https://doi.org/10.1016/S0021-9991(03)00126-8}, + doi = {10.1016/S0021-9991(03)00126-8}, + journal = {J. Comput. Phys.}, + month = {may}, + pages = {343–368}, + numpages = {26}, + keywords = {stability, nonuniform grid, conservation, higher-order discretization, turbulent channel flow, direct numerical simulation} +} diff --git a/docs/src/equations/time.md b/docs/src/equations/time.md index a914590e3..3c0692007 100644 --- a/docs/src/equations/time.md +++ b/docs/src/equations/time.md @@ -24,52 +24,155 @@ respected. For this, we will change the definition of the pressure such that the time-discretized velocity field is divergence free at each time step and each time sub-step (to be defined in the following). -Consider the interval ``[0, T]``. for some simulation time ``T``. We will -divide it into ``N`` sub-intervals ``[t^n, t^{n + 1}]`` for ``n = 0, \dots, N - -1``, with ``t^0 = 0``, ``t^N = T``, and increment ``\Delta t^n = t^{n + 1} - -t^n``. We define ``U^n \approx u_h(t_n)`` as an approximation to the exact -discrete velocity field ``u_h(t_n)``, with ``U^0 = u_h(0)`` starting from the -exact initial conditions. +Consider the interval ``[0, T]`` for some simulation time ``T``. We will divide +it into ``N`` sub-intervals ``[t^n, t^{n + 1}]`` for ``n = 0, \dots, N - 1``, +with ``t^0 = 0``, ``t^N = T``, and increment ``\Delta t^n = t^{n + 1} - t^n``. +We define ``U^n \approx u_h(t^n)`` as an approximation to the exact discrete +velocity field ``u_h(t^n)``, with ``U^0 = u_h(0)`` starting from the exact +initial conditions. + ## Explicit Runge-Kutta methods See Sanderse [Sanderse2012](@cite). -For explicit Runge-Kutta methods, we divide each time step into ``s`` sub-steps. -For ``i = 1, \dots, s``, we define the following quantities: +Consider the velocity field ``U_0`` at a certain time ``t_0``. We will now +perform one time step to ``t = t_0 + \Delta t``. For explicit Runge-Kutta +methods, this time step is divided into ``s`` sub-steps ``t_i = t_0 + \Delta +t_i`` with increment ``\Delta t_i = c_i \Delta t``. The final substep performs +the full time step ``\Delta t_s = \Delta t`` such that ``t_s = t``. + +For ``i = 1, \dots, s``, the intermediate velocity ``U_i`` and pressure ``P_i`` +are computed as follows: ```math \begin{split} -U_0^n & = U^n \\ -\Delta t_i^n & = c_i \Delta t^n \\ -t_i^n & = t^n + \Delta t_i^n \\ -F_i & = F(U_{i - 1}^n, t_{i - 1}^n) \\ -V_i^n & = U^n + \Delta t^n \sum_{j = 1}^i a_{i j} F_j \\ -L P_i^n & = \frac{(M V_i^n + y_M(t_i^n)) - (M U^n + y_M(t^n))}{\Delta t_i^n} \\ +F_i & = \Omega_h^{-1} F(U_{i - 1}, t_{i - 1}) \\ +V_i & = U_0 + \Delta t \sum_{j = 1}^i a_{i j} F_j \\ +L P_i & = \frac{(M V_i + y_M(t_i)) - (M U_0 + y_M(t_0))}{\Delta t_i^n} \\ & = \frac{1}{c_i} \sum_{j = 1}^i a_{i j} F_j + -\frac{y_M(t_i^n) - y_M(t^n)}{\Delta t_i^n} \\ -U_i^n & = V_i^n - \Delta t_i^n G P_i^n, +\frac{y_M(t_i) - y_M(t_0)}{\Delta t_i} \\ +U_i & = V_i - \Delta t_i \Omega_h^{-1} (G P_i + y_G(t_i)), \end{split} ``` -where ``(a_ij)_{i j}`` are the Butcher tableau coefficients of the RK-method, -with the convention ``c_i = \sum_{j = 1}^i a_{i j}`` and ``c_0 = 0``. +where ``(a_{i j})_{i j}`` are the Butcher tableau coefficients of the RK-method, +with the convention ``c_i = \sum_{j = 1}^i a_{i j}``. + +Finally, we set ``U = U_s``. If ``U_0 = u_h(t_0)``, we get the accuracy ``U = +u_h(t) + \mathcal{O}(\Delta t^{r + 1})``, where ``r`` is the order of the +RK-method. If we perform ``n`` RK time steps instead of one, starting at exact +initial conditions ``U^0 = u_h(0)``, then ``U^n = u_h(t^n) + +\mathcal{O}(\Delta t^r)`` for all ``n \in \{1, \dots, N\}``. Note that for a +given ``U``, the corresponding pressure ``P`` can be calculated to the same +accuracy as ``U`` by doing an additional pressure projection after each outer +time step ``\Delta t`` (if we know ``\frac{\mathrm{d} y_M}{\mathrm{d} t}(t)``), +or to first order accuracy by simply taking ``P = P_s``. -Finally, we set ``U^{n + 1} = U_s^n``. The corresponding pressure ``P^{n + 1}`` -can be calculated to the same accuracy as ``U^{n + 1}`` by doing an additional -pressure projection (if we know ``\frac{\mathrm{d} y_M}{\mathrm{d} t}(t^{n + -1})``), or to first order accuracy by simply taking ``P^{n + 1} = P_s^n``. +Note that each of the sub-step velocities ``U_i`` is divergence free, after +projecting the tentative velocities ``V_i``. This is ensured due to the +judiciously chosen replacement of ``\frac{\mathrm{d} y_M}{\mathrm{d} t}(t_i)`` +with ``(y_M(t_i) - y_M(t_0)) / \Delta t_i``. The space-discrete +divergence-freeness is thus perfectly preserved, even though the time +discretization introduces other errors. -Note that each of the sub-step velocities ``U_i^n`` is divergence free, after -projecting the tentative velocities ``V_i^s``. This is ensured due to the -judiciously chosen replacement of ``\frac{\mathrm{d} y_M}{\mathrm{d} t}(t_i^n)`` with -``(y_M(t_i^n) - y_M(t^n)) / \Delta t_i^n``. ## Implicit Runge-Kutta methods See Sanderse [Sanderse2013](@cite). + ## Adams-Bashforth Crank-Nicolson method +We here require that the time step ``\Delta t`` is constant. This methods uses +Adams-Bashforth for the convective terms and Crank-Nicolson stepping for the +diffusion and body force terms. Given the velocity field ``U_0 = u_h(t_0)`` at +a time ``t_0`` and its previous value ``U_{-1} = u_h(t_0 - \Delta t)`` at the +previous time ``t_{-1} = t_0 - \Delta t``, the predicted velocity field ``U`` +at the time ``t = t_0 + \Delta t`` is defined by first computing a tentative +velocity: + +```math +\begin{split} +\frac{V - U_0}{\Delta t} +& = - (\alpha_0 C(U_0, t_0) + \alpha_{-1} C(U_{-1}, t_{-1})) \\ +& + \theta (D U_0 + y_D(t_0)) + (1 - \theta) (D V + y_D(t)) \\ +& + \theta f(t_0) + (1 - \theta) f(t) \\ +& - (G p_0 + y_G(t_0)), +\end{split} +``` + +where ``\theta \in [0, 1]`` is the Crank-Nicolson parameter (``\theta = +\frac{1}{2}`` for second order convergence), ``(\alpha_0, \alpha_{-1}) = \left( +\frac{3}{2}, -\frac{1}{2} \right)`` are the Adams-Bashforth coefficients, and +``V`` is a tentative velocity yet to be made divergence free. We can group the +terms containing ``V`` on the left hand side, to obtain + +```math +\begin{split} +\left( \frac{1}{\Delta t} I - \theta D \right) V +& = \left(\frac{1}{\Delta t} I - (1 - \theta) D \right) U_0 \\ +& - (\alpha_0 C(U_0, t_0) + \alpha_{-1} C(U_{-1}, t_{-1})) \\ +& + \theta y_D(t_0) + (1 - \theta) y_D(t) \\ +& + \theta f(t_0) + (1 - \theta) f(t) \\ +& - (G P_0 + y_G(t_0)). +\end{split} +``` + +We can compute ``V`` by inverting the positive definite matrix ``\left( +\frac{1}{\Delta t} I - \theta D \right)`` for the given right hand side using a +suitable linear solver. Assuming ``\Delta t`` is constant, we can precompute a +Cholesky factorization of this matrix before starting time stepping. + +We then compute the pressure difference ``\Delta P`` by solving + +```math +L \Delta P = \frac{M V + y_M(t)}{\Delta t} - M (y_G(t) - y_G(t_0)), +``` + +after which a divergence free velocity ``U`` can be enforced: + +```math +U = V - \Delta t \Omega_h^{-1} (G \Delta P + y_G(t) - y_G(t_0)). +``` + +A first order accurate prediction of the corresponding pressure is ``P = P_0 + +\Delta P``. However, since this pressure is reused in the next time step, we +perform an additional pressure solve to avoid accumulating first order errors. +The resulting pressure ``P`` is then accurate to the same order as ``U``. + + ## One-leg beta method +See Verstappen and Veldman [Verstappen2003](@cite) [Verstappen1997](@cite). + +We here require that the time step ``\Delta t`` is constant. Given the velocity +``U_0`` and pressure ``P_0`` at the current time ``t_0`` and their and their +previous values ``U_{-1}`` and ``P_{-1}`` at the time ``t_{-1} = t_0 - \Delta +t``, we start by computing the "offstep" values ``V = (1 + \beta) V_0 - \beta +V_{-1}`` and ``Q = (1 + \beta) P_0 - \beta P_{-1}``. + +A tentative velocity field ``W`` is then computed as follows: + +```math +W = \frac{1}{\beta + \frac{1}{2}} \left( 2 \beta U_0 - \left( \beta - +\frac{1}{2} \right) U_{-1} + \Delta t \Omega_h^{-1} F(V, t) - \Delta t +\Omega_h^{-1} (G Q + y_G(t)) \right). +``` + +A pressure correction ``\Delta P `` is obtained by solving the Poisson equation +```math +L \Delta P = \frac{\beta + \frac{1}{2}}{\Delta t} (M W + y_M(t)). +``` + +Finally, the divergence free velocity field is given by + +```math +U = W - \frac{\delta t}{\beta + \frac{1}{2}} \Omega_h^{-1} G \Delta P, +``` + +while the second order accurate pressure is given by + +```math +P = 2 P_0 - P_{-1} + \frac{4}{3} \Delta P. +``` diff --git a/src/time_steppers/methods.jl b/src/time_steppers/methods.jl index 15b5eeb83..e8020bf54 100644 --- a/src/time_steppers/methods.jl +++ b/src/time_steppers/methods.jl @@ -15,41 +15,15 @@ abstract type AbstractODEMethod{T} end method_startup = RK44(; T), ) -IMEX AB-CN: Adams-Bashforth for explicit convection (parameters `α₁` and `α₂`) and -Crank-Nicolson for implicit diffusion (implicitness `θ`). -The method is second order for `θ = 1/2`. - -Adams-Bashforth for convection and Crank-Nicolson for diffusion -formulation: - -```math -\\begin{align*} -(\\mathbf{u}^{n+1} - \\mathbf{u}^n) / Δt & = - -(\\alpha_1 \\mathbf{c}^n + \\alpha_2 \\mathbf{c}^{n-1}) \\\\ - & + \\theta \\mathbf{d}^{n+1} + (1-\\theta) \\mathbf{d}^n \\\\ - & + \\theta \\mathbf{F}^{n+1} + (1-\\theta) \\mathbf{F}^n \\\\ - & + \\theta \\mathbf{BC}^{n+1} + (1-\\theta) \\mathbf{BC}^n \\\\ - & - \\mathbf{G} \\mathbf{p} + \\mathbf{y}_p -\\end{align*} -``` - -where BC are boundary conditions of diffusion. This is rewritten as: - -```math -\\begin{align*} -(\\frac{1}{\\Delta t} \\mathbf{I} - \\theta \\mathbf{D}) \\mathbf{u}^{n+1} & = - (\\frac{1}{\\Delta t} \\mathbf{I} - (1 - \\theta) \\mathbf{D}) \\mathbf{u}^{n} \\\\ - & - (\\alpha_1 \\mathbf{c}^n + \\alpha_2 \\mathbf{c}^{n-1}) \\\\ - & + \\theta \\mathbf{F}^{n+1} + (1-\\theta) \\mathbf{F}^{n} \\\\ - & + \\theta \\mathbf{BC}^{n+1} + (1-\\theta) \\mathbf{BC}^{n} \\\\ - & - \\mathbf{G} \\mathbf{p} + \\mathbf{y}_p -\\end{align*} -``` - -The LU decomposition of the LHS matrix is computed every time the time step changes. - -Note that, in constrast to explicit methods, the pressure from previous time steps has an -influence on the accuracy of the velocity. +IMEX AB-CN: Adams-Bashforth for explicit convection (parameters `α₁` and `α₂`) +and Crank-Nicolson for implicit diffusion (implicitness `θ`). The method is +second order for `θ = 1/2`. + +The LU decomposition of the LHS matrix is computed every time the time step +changes. + +Note that, in constrast to explicit methods, the pressure from previous time +steps has an influence on the accuracy of the velocity. """ struct AdamsBashforthCrankNicolsonMethod{T,M} <: AbstractODEMethod{T} α₁::T @@ -76,15 +50,8 @@ end ) Explicit one-leg β-method following symmetry-preserving discretization of -turbulent flow. See [Verstappen and Veldman (JCP 2003)] for details, or [Direct numerical -simulation of turbulence at lower costs (Journal of Engineering Mathematics 1997)]. - -Formulation: - -```math -\\frac{(\\beta + 1/2) u^{n+1} - 2 \\beta u^{n} + (\\beta - 1/2) u^{n-1}}{\\Delta t} = F((1 + -\\beta) u^n - \\beta u^{n-1}). -``` +turbulent flow. See Verstappen and Veldman [Verstappen2003](@cite) +[Verstappen1997](@cite) for details. """ struct OneLegMethod{T,M} <: AbstractODEMethod{T} β::T From 9d35fc91e4aa69c55c5840e3331ec7dfa2efc4f6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Sun, 10 Sep 2023 19:48:40 +0200 Subject: [PATCH 045/379] fix: Add missing fields --- src/setup.jl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/setup.jl b/src/setup.jl index 043fe24ff..ea779be04 100644 --- a/src/setup.jl +++ b/src/setup.jl @@ -11,6 +11,7 @@ bc_type = (; u = (; x = (:periodic, :periodic), y = (:periodic, :periodic)), v = (; x = (:periodic, :periodic), y = (:periodic, :periodic)), + ν = (; x = (:periodic, :periodic), y = (:periodic, :periodic)), ), order4 = false, bodyforce_u = (x, y) -> 0, @@ -33,7 +34,7 @@ function Setup( bc_type = (; u = (; x = (:periodic, :periodic), y = (:periodic, :periodic)), v = (; x = (:periodic, :periodic), y = (:periodic, :periodic)), - ν = (; x = (:dirichlet, :dirichlet), y = (:dirichlet, :dirichlet)), + ν = (; x = (:periodic, :periodic), y = (:periodic, :periodic)), ), order4 = false, bodyforce_u = (x, y) -> 0, From 3be50fb8e18c2e0521736f96a099062280cdcb00 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Sun, 10 Sep 2023 19:54:05 +0200 Subject: [PATCH 046/379] fix(closure): Only compute if non-zero --- src/momentum/momentum.jl | 18 ++++++++++-------- src/setup.jl | 4 ++-- 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/src/momentum/momentum.jl b/src/momentum/momentum.jl index f982edda5..7305c3f22 100644 --- a/src/momentum/momentum.jl +++ b/src/momentum/momentum.jl @@ -58,12 +58,12 @@ function momentum( # Body force b = force - # Closure model - cm = closure_model(V) - - # Residual in Finite Volume form, including the pressure contribution + # Residual in Finite Volume form, excluding the pressure contribution F = @. -c + d + b + cm + # Closure model + isnothing(closure_model) || (F += closure_model(V)) + # Nopressure = false is the most common situation, in which we return the entire # right-hand side vector if !nopressure @@ -74,6 +74,7 @@ function momentum( # Jacobian requested # We return only the Jacobian with respect to V (not p) ∇F = @. -∇c + ∇d + @assert isnothing(closure_model) "Jacobian of closure model currently not implemented" else ∇F = nothing end @@ -151,11 +152,11 @@ function momentum!( # Body force b = force - # Closure model - cm = closure_model(V) + # Residual in Finite Volume form, excluding the pressure contribution + @. F = -c + d + b - # Residual in Finite Volume form, including the pressure contribution - @. F = -c + d + b + cm + # Closure model + isnothing(closure_model) || (F .+= closure_model(V)) # Nopressure = false is the most common situation, in which we return the entire # right-hand side vector @@ -169,6 +170,7 @@ function momentum!( # Jacobian requested # We return only the Jacobian with respect to V (not p) @. ∇F = -∇c + ∇d + @assert isnothing(closure_model) "Jacobian of closure model currently not implemented" end F, ∇F diff --git a/src/setup.jl b/src/setup.jl index ea779be04..8db7e3c29 100644 --- a/src/setup.jl +++ b/src/setup.jl @@ -16,7 +16,7 @@ order4 = false, bodyforce_u = (x, y) -> 0, bodyforce_v = (x, y) -> 0, - closure_model = V -> zero(V), + closure_model = nothing, ) Create 2D setup. @@ -97,7 +97,7 @@ end bodyforce_u = (x, y, z) -> 0, bodyforce_v = (x, y, z) -> 0, bodyforce_w = (x, y, z) -> 0, - closure_model = V -> zero(V), + closure_model = nothing, ) Create 3D setup. From c308cded9d6e3656098e990170ea2aa4b60f41d2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Sun, 10 Sep 2023 20:04:02 +0200 Subject: [PATCH 047/379] docs(time): Minor corrections --- docs/src/equations/time.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/src/equations/time.md b/docs/src/equations/time.md index 3c0692007..a1efe44b0 100644 --- a/docs/src/equations/time.md +++ b/docs/src/equations/time.md @@ -4,7 +4,7 @@ The spatially discretized Navier-Stokes equations form a differential-algebraic system, with an ODE for the velocity ```math -\Omega \frac{\mathrm{d} u_h}{\mathrm{d} t} = F(u_h) - (G p_h + y_G) +\Omega_h \frac{\mathrm{d} u_h}{\mathrm{d} t} = F(u_h, t) - (G p_h + y_G) ``` subject to the algebraic constraint formed by the mass equation @@ -147,10 +147,10 @@ The resulting pressure ``P`` is then accurate to the same order as ``U``. See Verstappen and Veldman [Verstappen2003](@cite) [Verstappen1997](@cite). We here require that the time step ``\Delta t`` is constant. Given the velocity -``U_0`` and pressure ``P_0`` at the current time ``t_0`` and their and their -previous values ``U_{-1}`` and ``P_{-1}`` at the time ``t_{-1} = t_0 - \Delta -t``, we start by computing the "offstep" values ``V = (1 + \beta) V_0 - \beta -V_{-1}`` and ``Q = (1 + \beta) P_0 - \beta P_{-1}``. +``U_0`` and pressure ``P_0`` at the current time ``t_0`` and their previous +values ``U_{-1}`` and ``P_{-1}`` at the time ``t_{-1} = t_0 - \Delta t``, we +start by computing the "offstep" values ``V = (1 + \beta) V_0 - \beta V_{-1}`` +and ``Q = (1 + \beta) P_0 - \beta P_{-1}``. A tentative velocity field ``W`` is then computed as follows: @@ -168,7 +168,7 @@ L \Delta P = \frac{\beta + \frac{1}{2}}{\Delta t} (M W + y_M(t)). Finally, the divergence free velocity field is given by ```math -U = W - \frac{\delta t}{\beta + \frac{1}{2}} \Omega_h^{-1} G \Delta P, +U = W - \frac{\Delta t}{\beta + \frac{1}{2}} \Omega_h^{-1} G \Delta P, ``` while the second order accurate pressure is given by From 994a9926f82f942ebe5f1138da8a469ac13f436e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Mon, 11 Sep 2023 10:13:51 +0200 Subject: [PATCH 048/379] fix(momentum): Remove non-existent term --- src/momentum/momentum.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/momentum/momentum.jl b/src/momentum/momentum.jl index 7305c3f22..ff8d67dde 100644 --- a/src/momentum/momentum.jl +++ b/src/momentum/momentum.jl @@ -59,7 +59,7 @@ function momentum( b = force # Residual in Finite Volume form, excluding the pressure contribution - F = @. -c + d + b + cm + F = @. -c + d + b # Closure model isnothing(closure_model) || (F += closure_model(V)) From 6ed9f4d6264c7b1193724cd727bf95957bcbc3bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Mon, 11 Sep 2023 10:16:07 +0200 Subject: [PATCH 049/379] feat(CNN): Add input channels automatically --- scratch/train_model.jl | 8 ++++---- src/closures/cnn.jl | 7 ++++--- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/scratch/train_model.jl b/scratch/train_model.jl index 686bba163..1cf6731a5 100644 --- a/scratch/train_model.jl +++ b/scratch/train_model.jl @@ -115,16 +115,16 @@ arrows(x, y, fx, fy; lengthscale = 1.0f0) # setup, # # # Radius -# [5, 5, 5], +# [2, 2, 2, 2], # # # Channels -# [2, 64, 64, 2], +# [64, 64, 64, 2], # # # Activations -# [leakyrelu, leakyrelu, identity], +# [leakyrelu, leakyrelu, leakyrelu, identity], # # # Bias -# [true, true, false]; +# [true, true, true, false]; # ) closure, θ₀ = fno( diff --git a/src/closures/cnn.jl b/src/closures/cnn.jl index e16244208..196e9afd2 100644 --- a/src/closures/cnn.jl +++ b/src/closures/cnn.jl @@ -23,11 +23,12 @@ function cnn( T = eltype(x) @assert T == Float32 - # Make sure there are two velocity fields in input and output - @assert c[1] == 2 - # @assert c[1] == 4 + # Make sure there are two force fields in output @assert c[end] == 2 + # Add input channel size + c = [2; c] + # Create convolutional closure model NN = Chain( # Unflatten and separate u and v velocities From 26d29e695e6a7acb08b636692eef7b0b8a648c87 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Mon, 11 Sep 2023 15:02:46 +0200 Subject: [PATCH 050/379] docs: Minor changes --- docs/references.bib | 20 +++++----- docs/src/equations/ns.md | 69 ++++++++++++----------------------- docs/src/equations/spatial.md | 2 +- docs/src/equations/time.md | 6 ++- 4 files changed, 39 insertions(+), 58 deletions(-) diff --git a/docs/references.bib b/docs/references.bib index 82b05adab..513805881 100644 --- a/docs/references.bib +++ b/docs/references.bib @@ -154,16 +154,16 @@ @article{Sanderse2020 year = {2020} } @article{Verstappen1997, - doi = {10.1023/a:1004255329158}, - url = {https://doi.org/10.1023%2Fa%3A1004255329158}, - year = 1997, - publisher = {Springer Science and Business Media {LLC}}, - volume = {32}, - number = {2/3}, - pages = {143--159}, - author = {R.W.C.P. Verstappen and A.E.P. Veldman}, - journal = {Journal of Engineering Mathematics}, - title = {Direct Numerical Simulation of Turbulence at Lower Costs} + doi = {10.1023/a:1004255329158}, + url = {https://doi.org/10.1023%2Fa%3A1004255329158}, + year = 1997, + publisher = {Springer Science and Business Media {LLC}}, + volume = {32}, + number = {2/3}, + pages = {143--159}, + author = {R.W.C.P. Verstappen and A.E.P. Veldman}, + journal = {Journal of Engineering Mathematics}, + title = {Direct Numerical Simulation of Turbulence at Lower Costs} } @article{Verstappen2003, author = {Verstappen, R. W. C. P. and Veldman, A. E. P.}, diff --git a/docs/src/equations/ns.md b/docs/src/equations/ns.md index 1904172e5..194670717 100644 --- a/docs/src/equations/ns.md +++ b/docs/src/equations/ns.md @@ -8,7 +8,7 @@ conservative form, they are given by ```math \begin{align*} \nabla \cdot u & = 0, \\ -\frac{\mathrm{d} u}{\mathrm{d} t} + \nabla \cdot (u u^\mathsf{T}) & = -\nabla p + +\frac{\partial u}{\partial t} + \nabla \cdot (u u^\mathsf{T}) & = -\nabla p + \nu \nabla^2 u + f. \end{align*} ``` @@ -23,7 +23,7 @@ written as \sum_{\beta = 1}^d \frac{\partial u^\beta}{\partial x^\beta} & = 0, \\ \frac{\partial u^\alpha}{\partial t} + \sum_{\beta = 1}^d \frac{\partial}{\partial x^\beta} (u^\alpha u^\beta) & = -\frac{\partial -p}{\partial x^\alpha} + \nu \sum_{\beta = 1}^d \frac{\partial^2 u}{\partial +p}{\partial x^\alpha} + \nu \sum_{\beta = 1}^d \frac{\partial^2 u^\alpha}{\partial (x^\beta)^2} + f^\alpha. \end{align*} ``` @@ -39,7 +39,7 @@ be denoted ``\partial \mathcal{O}``, with normal ``n`` and surface element The mass equation in integral form is given by ```math -\int_{\partial \mathcal{O}} u \cdot n \, \mathrm{d} \Gamma = 0. +\int_{\partial \mathcal{O}} u \cdot n \, \mathrm{d} \Gamma = 0, ``` where we have used the divergence theorem to convert the volume integral to a @@ -74,8 +74,8 @@ normal ``n``. We allow for the following types of boundary conditions on - Periodic boundary conditions: ``u(x) = u(x + \tau)`` and ``p(x) = p(x + \tau)`` for ``x \in \Gamma``, where ``\tau`` is a constant translation to another part of the boundary ``\partial \Omega``. This usually requires ``\Gamma`` and its - periodic counterpart ``\Gamma + \tau`` to be flat (including in this software - suite). + periodic counterpart ``\Gamma + \tau`` to be flat and rectangular (including + in this software suite). - Dirichlet boundary conditions: ``u = u_\text{BC}`` on ``\Gamma``. A common situation here is that of a sticky wall, with "no slip boundary conditions. Then ``u_\text{BC} = 0``. @@ -98,32 +98,13 @@ equation for the pressure: \nabla \cdot f ``` -In 2D, this equation becomes +In scalar notation, this becomes ```math -- \left(\frac{\partial^2}{\partial x^2} + \frac{\partial^2}{\partial -y^2}\right) p = \frac{\partial^2 (u u)}{\partial x^2} + 2 \frac{\partial^2 (u -v)}{\partial x \partial y} + \frac{\partial^2 (v v)}{\partial y^2} - \left( \frac{\partial f_u}{\partial x} + \frac{\partial f_v}{\partial y} \right). -``` - -In 3D, this equation becomes - -```math -\begin{split} - - \left( \frac{\partial^2}{\partial x^2} + \frac{\partial^2}{\partial - y^2} + \frac{\partial^2 }{\partial z^2} \right) p - & = \frac{\partial^2 (u u)}{\partial x^2} - + 2 \frac{\partial^2 (u v)}{\partial x \partial y} - + 2 \frac{\partial^2 (u w)}{\partial x \partial z} \\ - & + 2 \frac{\partial^2 (v u)}{\partial y \partial x} - + \frac{\partial^2 (v v)}{\partial y^2} - + 2 \frac{\partial^2 (v w)}{\partial y \partial z} \\ - & + 2 \frac{\partial^2 (v u)}{\partial z \partial x} - + 2 \frac{\partial^2 (w v)}{\partial z \partial y} - + \frac{\partial^2 (w w)}{\partial z^2} \\ - & - \left( \frac{\partial f_u}{\partial x} + \frac{\partial f_v}{\partial - y} + \frac{\partial f_w}{\partial z} \right). -\end{split} +- \sum_{\alpha = 1}^d \frac{\partial^2}{\partial (x^\alpha)^2} p = \sum_{\alpha += 1}^d \sum_{\beta = 1}^d \frac{\partial^2 }{\partial x^\alpha \partial +x^\beta} (u^\alpha u^\beta) - \sum_{\alpha = 1}^d \frac{\partial +f^\alpha}{\partial x^\alpha}. ``` Note the absence of time derivatives in the pressure equation. While the @@ -143,45 +124,43 @@ The vorticity is defined as ``\omega = \nabla \times u``. In 2D, it is a scalar field given by ```math -\omega = -\frac{\partial u}{\partial y} + \frac{\partial v}{\partial x}. +\omega = -\frac{\partial u^1}{\partial x^2} + \frac{\partial u^2}{\partial +x^1}. ``` In 3D, it is a vector field given by ```math \omega = \begin{pmatrix} - - \frac{\partial v}{\partial z} + \frac{\partial w}{\partial y} \\ - \phantom{+} \frac{\partial u}{\partial z} - \frac{\partial w}{\partial x} \\ - - \frac{\partial u}{\partial z} + \frac{\partial w}{\partial x} + - \frac{\partial u^2}{\partial x^3} + \frac{\partial u^3}{\partial x^2} \\ + - \frac{\partial u^3}{\partial x^1} + \frac{\partial u^1}{\partial x^3} \\ + - \frac{\partial u^1}{\partial x^2} + \frac{\partial u^2}{\partial x^1} \end{pmatrix}. ``` Note that the 2D vorticity is equal -to the ``z``-component of the 3D vorticity. +to the ``x^3``-component of the 3D vorticity. ### Stream function -In 2D, the stream function ``\psi`` is a scalar field defined such that - -```math -u = \frac{\partial \psi}{\partial y}, \quad v = \frac{\partial \psi}{\partial x}. -``` - -In 3D, the stream function ``\psi`` is a vector field defined such that +The stream function ``\psi`` is a field (scalar in 2D, vector in 3D) defined +such that ```math -u = \nabla \times \psi +u = \nabla \times \psi. ``` -It is related to the 3D vorticity as +It is related to the vorticity as ```math -\nabla^2 \psi = \nabla \times \omega +\nabla^2 \psi = \nabla \times \omega. ``` ### Kinetic energy -The kinetic energy is defined by ``e = \frac{1}{2} \| u \|^2``. +The local and total kinetic energy are defined by ``k = \frac{1}{2} \| u +\|_2^2`` and ``K = \frac{1}{2} \| u \|_{L^2(\Omega)}^2 = \int_\Omega k \, +\mathrm{d} \Omega``. ### Reynolds number diff --git a/docs/src/equations/spatial.md b/docs/src/equations/spatial.md index 884c5986f..ee13d714f 100644 --- a/docs/src/equations/spatial.md +++ b/docs/src/equations/spatial.md @@ -194,7 +194,7 @@ u^\alpha}{\partial x^\beta} n^\beta \, \mathrm{d} \Gamma ``` where ``n = (n^1, \dots, n^d)`` is the surface normal vector to ``\partial -\Omega``. +\mathcal{O}``. This time, we will not let ``\mathcal{O}`` be the reference finite volume ``\Omega_{I}`` (the ``p``-volume), but rather the shifted ``u^\alpha``-volume. diff --git a/docs/src/equations/time.md b/docs/src/equations/time.md index a1efe44b0..1492d060f 100644 --- a/docs/src/equations/time.md +++ b/docs/src/equations/time.md @@ -29,7 +29,9 @@ it into ``N`` sub-intervals ``[t^n, t^{n + 1}]`` for ``n = 0, \dots, N - 1``, with ``t^0 = 0``, ``t^N = T``, and increment ``\Delta t^n = t^{n + 1} - t^n``. We define ``U^n \approx u_h(t^n)`` as an approximation to the exact discrete velocity field ``u_h(t^n)``, with ``U^0 = u_h(0)`` starting from the exact -initial conditions. +initial conditions. We say that the time integration scheme (definition of +``U^n``) is accurate to the order ``r`` if ``U^n = u_h(t^n) + +\mathcal{O}(\Delta t^r)`` for all ``n``. ## Explicit Runge-Kutta methods @@ -150,7 +152,7 @@ We here require that the time step ``\Delta t`` is constant. Given the velocity ``U_0`` and pressure ``P_0`` at the current time ``t_0`` and their previous values ``U_{-1}`` and ``P_{-1}`` at the time ``t_{-1} = t_0 - \Delta t``, we start by computing the "offstep" values ``V = (1 + \beta) V_0 - \beta V_{-1}`` -and ``Q = (1 + \beta) P_0 - \beta P_{-1}``. +and ``Q = (1 + \beta) P_0 - \beta P_{-1}`` for some ``\beta = \frac{1}{2}``. A tentative velocity field ``W`` is then computed as follows: From 80f6fc0a737ae6feed8d6e7ed1623bfb6688992b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Thu, 14 Sep 2023 00:30:49 +0200 Subject: [PATCH 051/379] refactor: Kernel formulation (works for periodic) --- Project.toml | 2 + docs/src/api/api.md | 52 +- docs/src/equations/time.md | 5 +- examples/DecayingTurbulence2D.jl | 39 +- src/IncompressibleNavierStokes.jl | 63 +- src/boundary_conditions.jl | 218 ++++ src/boundary_conditions/bc_av3.jl | 93 -- src/boundary_conditions/bc_av_stag3.jl | 96 -- src/boundary_conditions/bc_diff3.jl | 96 -- src/boundary_conditions/bc_diff_stag.jl | 88 -- src/boundary_conditions/bc_diff_stag3.jl | 93 -- src/boundary_conditions/bc_div2.jl | 87 -- src/boundary_conditions/bc_general.jl | 121 -- src/boundary_conditions/bc_general_stag.jl | 124 -- .../bc_general_stag_diff.jl | 88 -- src/boundary_conditions/bc_int2.jl | 86 -- src/boundary_conditions/bc_int3.jl | 93 -- src/boundary_conditions/bc_int_mixed2.jl | 74 -- src/boundary_conditions/bc_int_mixed_stag2.jl | 90 -- src/boundary_conditions/bc_int_mixed_stag3.jl | 89 -- src/boundary_conditions/bc_vectors.jl | 87 -- src/boundary_conditions/bc_vort3.jl | 83 -- .../boundary_conditions.jl | 95 -- src/boundary_conditions/get_bc_vectors.jl | 1020 ---------------- src/create_initial_conditions.jl | 130 ++ src/force/force.jl | 28 - src/grid/grid.jl | 921 ++------------ src/grid/max_size.jl | 6 +- src/models/viscosity_models.jl | 8 +- src/momentum.jl | 216 ++++ src/momentum/check_symmetry.jl | 45 - src/momentum/compute_conservation.jl | 124 -- src/momentum/convection.jl | 860 ------------- src/momentum/convection_components.jl | 634 ---------- src/momentum/diffusion.jl | 235 ---- src/momentum/momentum.jl | 177 --- src/momentum/momentumcache.jl | 242 ---- src/momentum/strain_tensor.jl | 163 --- src/momentum/turbulent_K.jl | 17 - src/momentum/turbulent_viscosity.jl | 8 - src/operators/operator_averaging.jl | 411 ------- .../operator_convection_diffusion.jl | 1071 ----------------- src/operators/operator_divergence.jl | 293 ----- src/operators/operator_filter.jl | 125 -- src/operators/operator_interpolation.jl | 857 ------------- src/operators/operator_postprocessing.jl | 173 --- src/operators/operator_regularization.jl | 49 - src/operators/operator_turbulent_diffusion.jl | 309 ----- src/operators/operator_viscosity.jl | 16 - src/operators/operators.jl | 33 - src/postprocess/get_streamfunction.jl | 110 -- src/postprocess/get_velocity.jl | 49 - src/postprocess/get_vorticity.jl | 156 --- src/postprocess/plot_pressure.jl | 19 +- src/postprocess/plot_vorticity.jl | 23 +- src/preprocess/create_initial_conditions.jl | 253 ---- src/processors/real_time_plot.jl | 40 +- src/setup.jl | 160 +-- .../pressure/pressure_additional_solve.jl | 80 +- src/solvers/pressure/pressure_poisson.jl | 14 +- src/solvers/pressure/pressure_solvers.jl | 88 +- src/solvers/solve_unsteady.jl | 56 +- src/time_steppers/step.jl | 4 +- src/time_steppers/step_ab_cn.jl | 8 +- .../step_explicit_runge_kutta.jl | 134 +-- .../step_implicit_runge_kutta.jl | 4 +- src/time_steppers/step_one_leg.jl | 8 +- src/time_steppers/time_stepper_caches.jl | 23 +- src/utils/filter_convection.jl | 28 - src/utils/get_lims.jl | 3 +- 70 files changed, 881 insertions(+), 10512 deletions(-) create mode 100644 src/boundary_conditions.jl delete mode 100644 src/boundary_conditions/bc_av3.jl delete mode 100644 src/boundary_conditions/bc_av_stag3.jl delete mode 100644 src/boundary_conditions/bc_diff3.jl delete mode 100644 src/boundary_conditions/bc_diff_stag.jl delete mode 100644 src/boundary_conditions/bc_diff_stag3.jl delete mode 100644 src/boundary_conditions/bc_div2.jl delete mode 100644 src/boundary_conditions/bc_general.jl delete mode 100644 src/boundary_conditions/bc_general_stag.jl delete mode 100644 src/boundary_conditions/bc_general_stag_diff.jl delete mode 100644 src/boundary_conditions/bc_int2.jl delete mode 100644 src/boundary_conditions/bc_int3.jl delete mode 100644 src/boundary_conditions/bc_int_mixed2.jl delete mode 100644 src/boundary_conditions/bc_int_mixed_stag2.jl delete mode 100644 src/boundary_conditions/bc_int_mixed_stag3.jl delete mode 100644 src/boundary_conditions/bc_vectors.jl delete mode 100644 src/boundary_conditions/bc_vort3.jl delete mode 100644 src/boundary_conditions/boundary_conditions.jl delete mode 100644 src/boundary_conditions/get_bc_vectors.jl create mode 100644 src/create_initial_conditions.jl delete mode 100644 src/force/force.jl create mode 100644 src/momentum.jl delete mode 100644 src/momentum/check_symmetry.jl delete mode 100644 src/momentum/compute_conservation.jl delete mode 100644 src/momentum/convection.jl delete mode 100644 src/momentum/convection_components.jl delete mode 100644 src/momentum/diffusion.jl delete mode 100644 src/momentum/momentum.jl delete mode 100644 src/momentum/momentumcache.jl delete mode 100644 src/momentum/strain_tensor.jl delete mode 100644 src/momentum/turbulent_K.jl delete mode 100644 src/momentum/turbulent_viscosity.jl delete mode 100644 src/operators/operator_averaging.jl delete mode 100644 src/operators/operator_convection_diffusion.jl delete mode 100644 src/operators/operator_divergence.jl delete mode 100644 src/operators/operator_filter.jl delete mode 100644 src/operators/operator_interpolation.jl delete mode 100644 src/operators/operator_postprocessing.jl delete mode 100644 src/operators/operator_regularization.jl delete mode 100644 src/operators/operator_turbulent_diffusion.jl delete mode 100644 src/operators/operator_viscosity.jl delete mode 100644 src/operators/operators.jl delete mode 100644 src/postprocess/get_streamfunction.jl delete mode 100644 src/postprocess/get_velocity.jl delete mode 100644 src/postprocess/get_vorticity.jl delete mode 100644 src/preprocess/create_initial_conditions.jl delete mode 100644 src/utils/filter_convection.jl diff --git a/Project.toml b/Project.toml index c4f778431..b4cb81d31 100644 --- a/Project.toml +++ b/Project.toml @@ -8,10 +8,12 @@ Adapt = "79e6a3ab-5dfb-504d-930d-738a2a938a0e" ComponentArrays = "b0b7db55-cfe3-40fc-9ded-d10e2dbeff66" FFTW = "7a1cc6ca-52ef-59f5-83cd-3a7055c09341" IterativeSolvers = "42fd0dbc-a981-5370-80f2-aaf504508153" +KernelAbstractions = "63c18a36-062a-441e-b654-da1e3ab1ce7c" LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" Lux = "b2108857-7c20-44ae-9111-449ecde12c47" Makie = "ee78f7c6-11fb-53f2-987a-cfe4a2b5a57a" NNlib = "872c559c-99b0-510c-b3b7-b6c96a88d5cd" +Observables = "510215fc-4207-5dde-b226-833fc4488ee2" Optimisers = "3bd65402-5787-11e9-1adc-39752487f4e2" Printf = "de0858da-6303-5e67-8744-51eddeeeb8d7" Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" diff --git a/docs/src/api/api.md b/docs/src/api/api.md index cbf768fa8..4bc0fa8c6 100644 --- a/docs/src/api/api.md +++ b/docs/src/api/api.md @@ -9,18 +9,6 @@ IncompressibleNavierStokes Setup ``` -## Boundary conditions - -```@docs -BoundaryConditions -get_bc_vectors -``` - -## Force - -```@docs -SteadyBodyForce -``` ## Grid @@ -55,46 +43,19 @@ LerayConvectionModel ## Momentum ```@docs -MomentumCache -check_symmetry -compute_conservation -convection +divergence +divergence! +vorticity +vorticity! convection! -convection_components -convection_components! -diffusion diffusion! -momentum +bodyforce! momentum! -momentum_allstage -momentum_allstage! -strain_tensor -turbulent_K -turbulent_viscosity -``` - -## Operators - -```@docs -Operators -operator_averaging -operator_convection_diffusion -operator_divergence -operator_interpolation -operator_postprocessing -operator_regularization -operator_turbulent_diffusion -operator_viscosity -operator_filter ``` ## Postprocess ```@docs -get_streamfunction -get_velocity -get_vorticity -vorticity! plot_force plot_grid plot_pressure @@ -167,7 +128,6 @@ step! ## Utils ```@docs -filter_convection -filter_convection! get_lims +plotmat ``` diff --git a/docs/src/equations/time.md b/docs/src/equations/time.md index 1492d060f..257505eb7 100644 --- a/docs/src/equations/time.md +++ b/docs/src/equations/time.md @@ -51,9 +51,10 @@ are computed as follows: \begin{split} F_i & = \Omega_h^{-1} F(U_{i - 1}, t_{i - 1}) \\ V_i & = U_0 + \Delta t \sum_{j = 1}^i a_{i j} F_j \\ -L P_i & = \frac{(M V_i + y_M(t_i)) - (M U_0 + y_M(t_0))}{\Delta t_i^n} \\ -& = \frac{1}{c_i} \sum_{j = 1}^i a_{i j} F_j + +L P_i & = \frac{1}{c_i} \sum_{j = 1}^i a_{i j} F_j + \frac{y_M(t_i) - y_M(t_0)}{\Delta t_i} \\ +& = \frac{(M V_i + y_M(t_i)) - (M U_0 + y_M(t_0))}{\Delta t_i^n} \\ +& = \frac{M V_i + y_M(t_i)}{\Delta t_i^n} \\ U_i & = V_i - \Delta t_i \Omega_h^{-1} (G P_i + y_G(t_i)), \end{split} ``` diff --git a/examples/DecayingTurbulence2D.jl b/examples/DecayingTurbulence2D.jl index 99e7a541c..a08e1ca09 100644 --- a/examples/DecayingTurbulence2D.jl +++ b/examples/DecayingTurbulence2D.jl @@ -34,54 +34,51 @@ device = identity # To use GPU, use `cu` to move arrays to the GPU. # Note: `cu` converts to Float32 -## using CUDA -## device = cu +using CUDA +device = cu # Viscosity model Re = T(10_000) # A 2D grid is a Cartesian product of two vectors n = 256 -lims = (T(0), T(1)) -x = LinRange(lims..., n + 1) -y = LinRange(lims..., n + 1) -# plot_grid(x, y) +lims = T(0), T(1) +x = LinRange(lims..., n + 1), LinRange(lims..., n + 1) +# plot_grid(x...) # Build setup and assemble operators -setup = Setup(x, y; Re); +setup = device(Setup(x; Re)); # Since the grid is uniform and identical for x and y, we may use a specialized # spectral pressure solver pressure_solver = SpectralPressureSolver(setup); -# Initial conditions -V₀, p₀ = random_field(setup; A = T(1_000_000), σ = T(30), s = 5, pressure_solver); +u₀, p₀ = random_field(setup, T(0); A = T(1_000_000), σ = T(30), s = T(5), pressure_solver); # Iteration processors processors = ( - field_plotter(device(setup); nupdate = 20), - energy_history_plotter(device(setup); nupdate = 20, displayfig = false), - energy_spectrum_plotter(device(setup); nupdate = 20, displayfig = false), + field_plotter(setup; nupdate = 10), + # energy_history_plotter(device(setup); nupdate = 20, displayfig = false), + # energy_spectrum_plotter(device(setup); nupdate = 20, displayfig = false), ## animator(device(setup), "vorticity.mp4"; nupdate = 16), ## vtk_writer(setup; nupdate = 10, dir = "output/$name", filename = "solution"), ## field_saver(setup; nupdate = 10), - step_logger(; nupdate = 100), + step_logger(; nupdate = 10), ); # Time interval -t_start, t_end = tlims = (T(0), T(1.0)) +t_start, t_end = tlims = T(0), T(1.0) # Solve unsteady problem -V, p, outputs = solve_unsteady( +u, p, outputs = solve_unsteady( setup, - V₀, + u₀, p₀, tlims; Δt = T(0.001), processors, pressure_solver, inplace = true, - device, ); # Field plot @@ -98,13 +95,13 @@ outputs[3] # We may visualize or export the computed fields `(V, p)` # Export to VTK -save_vtk(setup, V, p, t_end, "output/solution") +save_vtk(setup, u, p, t_end, "output/solution") # Plot pressure -plot_pressure(setup, p) +plot_pressure(setup, p₀) # Plot velocity -plot_velocity(setup, V, t_end) +plot_velocity(setup, u, t_end) # Plot vorticity -plot_vorticity(setup, V, t_end) +plot_vorticity(setup, u, t_end) diff --git a/src/IncompressibleNavierStokes.jl b/src/IncompressibleNavierStokes.jl index 017406325..305a51c63 100644 --- a/src/IncompressibleNavierStokes.jl +++ b/src/IncompressibleNavierStokes.jl @@ -6,9 +6,10 @@ Energy-conserving solvers for the incompressible Navier-Stokes equations. module IncompressibleNavierStokes using Adapt -using ComponentArrays +using ComponentArrays: ComponentArray using FFTW using IterativeSolvers +using KernelAbstractions using LinearAlgebra using Lux using Makie @@ -24,6 +25,9 @@ using Zygote # Convenience notation const ⊗ = kron +# Boundary condtions +include("boundary_conditions.jl") + # Grid include("grid/dimension.jl") include("grid/grid.jl") @@ -35,48 +39,15 @@ include("grid/max_size.jl") include("models/viscosity_models.jl") include("models/convection_models.jl") -# Types -include("force/force.jl") -include("boundary_conditions/boundary_conditions.jl") -include("operators/operators.jl") +# Setup include("setup.jl") -# Boundary condtions -include("boundary_conditions/bc_av3.jl") -include("boundary_conditions/bc_av_stag3.jl") -include("boundary_conditions/bc_diff3.jl") -include("boundary_conditions/bc_diff_stag.jl") -include("boundary_conditions/bc_diff_stag3.jl") -include("boundary_conditions/bc_div2.jl") -include("boundary_conditions/bc_general.jl") -include("boundary_conditions/bc_general_stag.jl") -include("boundary_conditions/bc_general_stag_diff.jl") -include("boundary_conditions/bc_int2.jl") -include("boundary_conditions/bc_int3.jl") -include("boundary_conditions/bc_int_mixed2.jl") -include("boundary_conditions/bc_int_mixed_stag2.jl") -include("boundary_conditions/bc_int_mixed_stag3.jl") -include("boundary_conditions/bc_vort3.jl") -include("boundary_conditions/get_bc_vectors.jl") - -# Operators -include("operators/operator_averaging.jl") -include("operators/operator_convection_diffusion.jl") -include("operators/operator_divergence.jl") -include("operators/operator_interpolation.jl") -include("operators/operator_postprocessing.jl") -include("operators/operator_regularization.jl") -include("operators/operator_turbulent_diffusion.jl") -include("operators/operator_viscosity.jl") -include("operators/operator_filter.jl") - # Pressure solvers include("solvers/pressure/pressure_solvers.jl") include("solvers/pressure/pressure_poisson.jl") include("solvers/pressure/pressure_additional_solve.jl") # Time steppers -include("momentum/momentumcache.jl") include("time_steppers/methods.jl") include("time_steppers/tableaux.jl") include("time_steppers/nstage.jl") @@ -86,7 +57,7 @@ include("time_steppers/isexplicit.jl") include("time_steppers/lambda_max.jl") # Preprocess -include("preprocess/create_initial_conditions.jl") +include("create_initial_conditions.jl") # Processors include("processors/processors.jl") @@ -94,15 +65,7 @@ include("processors/real_time_plot.jl") include("processors/animator.jl") # Momentum equation -include("momentum/compute_conservation.jl") -include("momentum/check_symmetry.jl") -include("momentum/convection_components.jl") -include("momentum/convection.jl") -include("momentum/diffusion.jl") -include("momentum/momentum.jl") -include("momentum/strain_tensor.jl") -include("momentum/turbulent_K.jl") -include("momentum/turbulent_viscosity.jl") +include("momentum.jl") # Solvers include("solvers/get_timestep.jl") @@ -110,14 +73,10 @@ include("solvers/solve_steady_state.jl") include("solvers/solve_unsteady.jl") # Utils -include("utils/filter_convection.jl") include("utils/get_lims.jl") include("utils/plotmat.jl") # Postprocess -include("postprocess/get_velocity.jl") -include("postprocess/get_vorticity.jl") -include("postprocess/get_streamfunction.jl") include("postprocess/plot_force.jl") include("postprocess/plot_grid.jl") include("postprocess/plot_pressure.jl") @@ -132,6 +91,9 @@ include("closures/fno.jl") include("closures/training.jl") include("closures/create_les_data.jl") +# Boundary conditions +export PeriodicBC, DirichletBC, SymmetricBC, NeumannBC + # Force export SteadyBodyForce @@ -146,7 +108,6 @@ export animator # Setup export Setup -export operator_filter # 1D grids export stretched_grid, cosine_grid @@ -160,7 +121,7 @@ export pressure_poisson, export solve_unsteady, solve_steady_state export momentum, momentum! -export create_initial_conditions, random_field, get_bc_vectors, get_velocity +export create_initial_conditions, random_field, get_velocity export plot_force, plot_grid, plot_pressure, plot_streamfunction, plot_velocity, plot_vorticity, save_vtk diff --git a/src/boundary_conditions.jl b/src/boundary_conditions.jl new file mode 100644 index 000000000..f4cd4e3f6 --- /dev/null +++ b/src/boundary_conditions.jl @@ -0,0 +1,218 @@ +abstract type AbstractBC end + +struct PeriodicBC <: AbstractBC end + +struct DirichletBC{F} <: AbstractBC + u::F + dudt::F +end + +struct SymmetricBC <: AbstractBC end + +struct PressureBC{F} <: AbstractBC + p::F +end + +function ghost_a! end +function ghost_b! end + +# Add opposite boundary ghost volume +# Do everything in first function call for periodic +function ghost_a!(::PeriodicBC, x) + Δx_a = x[2] - x[1] + Δx_b = x[end] - x[end-1] + pushfirst!(x, x[1] - Δx_b) + push!(x, x[end] + Δx_a) +end +ghost_b!(::PeriodicBC, x) = nothing + +# Add infinitely thin boundary volume +ghost_a!(::DirichletBC, x) = pushfirst!(x, x[1]) +ghost_b!(::DirichletBC, x) = push!(x, x[end]) + +# Duplicate boundary volume +ghost_a!(::SymmetricBC, x) = pushfirst!(x, x[1] + (x[2] - x[1])) +ghost_b!(::SymmetricBC, x) = push!(x, x[end] + (x[end] - x[end-1])) + +# Add infinitely thin boundary volume and then symmetric +ghost_a!(::PressureBC, x) = pushfirst!(x, x[1] - (x[2] - x[1]), x[1]) +ghost_b!(::PressureBC, x) = push!(x, x[end], x[end] + (x[end] - x[end-1])) + +""" + offset_u(bc, isnormal, atend) + +Number of non-DOF velocity components at boundary. +If ``isnormal``, then the velocity is normal to the boundary, else parallel. +If ``atend``, it is at the end/right/rear/top boundary, otherwise beginning. +""" +function offset_u end + +""" + offset_p(bc) + +Number of non-DOF pressure components at boundary. +""" +function offset_p end + +offset_u(::PeriodicBC, isnormal, atend) = 1 +offset_p(::PeriodicBC) = 1 + +offset_u(::DirichletBC, isnormal, atend) = 1 +offset_p(::DirichletBC) = 1 + +offset_u(::SymmetricBC, isnormal, atend) = 1 +offset_p(::SymmetricBC) = 1 + +offset_u(::PressureBC, isnormal, atend) = isnormal && atend ? 2 : 1 +offset_p(::PressureBC) = 2 + +function apply_bc_u! end +function apply_bc_p! end + +function apply_bc_u!(u, t, setup; kwargs...) + (; boundary_conditions) = setup + D = length(u) + for β = 1:D + apply_bc_u!(boundary_conditions[β][1], u, β, t, setup; atend = false, kwargs...) + apply_bc_u!(boundary_conditions[β][2], u, β, t, setup; atend = true, kwargs...) + end +end + +function apply_bc_p!(p, t, setup) + (; boundary_conditions, grid) = setup + (; dimension) = grid + D = dimension() + for β = 1:D + apply_bc_p!(boundary_conditions[β][1], p, β, t, setup; atend = false) + apply_bc_p!(boundary_conditions[β][2], p, β, t, setup; atend = true) + end +end + +function apply_bc_u!(::PeriodicBC, u, β, t, setup; atend, kwargs...) + (; grid) = setup + (; dimension, Nu, Iu) = grid + D = dimension() + δ = Offset{D}() + @kernel function _bc_a!(u, α, β, I0) + I = @index(Global, Cartesian) + I = I + I0 + u[α][I] = u[α][I+Nu[α][β]*δ(β)] + end + @kernel function _bc_b!(u, α, β, I0) + I = @index(Global, Cartesian) + I = I + I0 + u[α][I+Nu[α][β]*δ(β)] = u[α][I] + end + for α = 1:D + I0 = first(Iu[α]) + I0 -= oneunit(I0) + ndrange = (Nu[α][1:β-1]..., 1, Nu[α][β+1:end]...) + if atend + _bc_b!(get_backend(u[1]), WORKGROUP)(u, α, β, I0; ndrange) + synchronize(get_backend(u[1])) + else + _bc_a!(get_backend(u[1]), WORKGROUP)(u, α, β, I0 - δ(β); ndrange) + synchronize(get_backend(u[1])) + end + end +end + +function apply_bc_p!(::PeriodicBC, p, β, t, setup; atend) + (; grid) = setup + (; dimension, Np, Ip) = grid + D = dimension() + δ = Offset{D}() + @kernel function _bc_a(p, β, I0) + I = @index(Global, Cartesian) + I = I + I0 + p[I] = p[I+Np[β]*δ(β)] + end + @kernel function _bc_b(p, β, I0) + I = @index(Global, Cartesian) + I = I + I0 + p[I+Np[β]*δ(β)] = p[I] + end + I0 = first(Ip) + I0 -= oneunit(I0) + ndrange = (Np[1:β-1]..., 1, Np[β+1:end]...) + if atend + _bc_b(get_backend(p), WORKGROUP)(p, β, I0; ndrange) + synchronize(get_backend(p)) + else + _bc_a(get_backend(p), WORKGROUP)(p, β, I0 - δ(β); ndrange) + synchronize(get_backend(p)) + end +end + +function apply_bc_u!(bc::DirichletBC, u, β, t, setup; atend, dudt = false, kwargs...) + (; dimension, Nu, x, xp) = setup.grid + D = dimension() + δ = Offset{D}() + bcfunc = dudt ? bc.dudt : bc.u + @kernel function _bc_a(u, α, β) + I = @index(Global, Cartesian) + # u[i][I] = bcfunc[i](ntuple(k -> k == i [I + Nu[i][j] * δ(j)] + # TODO: Apply bcfunc + end + @kernel function _bc_b(u, α, β; xΓ) + I = @index(Global, Cartesian) + # u[α][I] = bcfunc[α](ntuple(xp) + # TODO: Apply bcfunc + end + for α = 1:D + xΓ = (xp[1:β-1]..., xp[β+1:end]...) + if atend + _bc_b(get_backend(u[1]), WORKGROUP)(u, α, β, I0) + synchronize(get_backend(u[1])) + else + _bc_a(get_backend(u[1]), WORKGROUP)(u, α, β, I0) + synchronize(get_backend(u[1])) + end + end +end + +function apply_bc_p!(::DirichletBC, p, β, t, setup; atend) + nothing +end + +function apply_bc_u!(::SymmetricBC, u, β, t, setup; atend, kwargs...) + error("Not implemented") + (; Nu, x, xp) = setup.grid + D = dimension() + δ = Offset{D}() + bcfunc = dudt ? bc.dudt : bc.u + @kernel function _bc_a(u, α, β) + I = @index(Global, Cartesian) + # u[i][I] = bcfunc[i](ntuple(k -> k == i [I + Nu[i][j] * δ(j)] + # TODO: Apply bcfunc + end + @kernel function _bc_b(u, α, β) + I = @index(Global, Cartesian) + # u[α][I] = bcfunc[α](ntuple(xp) + # TODO: Apply bcfunc + end + for α = 1:D + for β = 1:D + xΓ = (xp[1:β-1]..., xp[β+1:end]...) + if atend + _bc_b(get_backend(u[1]), WORKGROUP)(u, α, β, I0) + synchronize(get_backend(u[1])) + else + _bc_a(get_backend(u[1]), WORKGROUP)(u, α, β, I0) + synchronize(get_backend(u[1])) + end + end + end +end + +function apply_bc_p!(::SymmetricBC, p, β, t, setup; atend) + error("Not implemented") +end + +function apply_bc_u!(bc::PressureBC, u, β, t, setup; atend, kwargs...) + error("Not implemented") +end + +function apply_bc_p!(bc::PressureBC, p, β, t, setup; atend) + error("Not implemented") +end diff --git a/src/boundary_conditions/bc_av3.jl b/src/boundary_conditions/bc_av3.jl deleted file mode 100644 index 4e7e3d9bc..000000000 --- a/src/boundary_conditions/bc_av3.jl +++ /dev/null @@ -1,93 +0,0 @@ -function bc_av3(Nt, Nin, Nb, bc1, bc2, h1, h2) - # total solution u is written as u = Bb*ub + Bin*uin - # the boundary conditions can be written as Bbc*u = ybc - # then u can be written entirely in terms of uin and ybc as: - # u = (Bin-Btemp*Bbc*Bin)*uin + Btemp*ybc, where - # Btemp = Bb*(Bbc*Bb)^(-1) - # Bb, Bin and Bbc depend on type of bc (Neumann/Dirichlet/periodic) - - # val1 and val2 can be scalars or vectors with either the value or the - # derivative - - # (ghost) points on boundary / grid lines - - T = typeof(h1) - - # some input checking: - if Nt != Nin + Nb - error("Number of inner points plus boundary points is not equal to total points") - end - - if Nb == 0 - # no boundary points, so simply diagonal matrix without boundary contribution - B1D = I(Nt) - Btemp = spzeros(T, Nt, 2) - ybc1 = zeros(T, 2) - ybc2 = zeros(T, 2) - else - # boundary conditions - Bbc = spzeros(T, Nb, Nt) - ybc1_1D = zeros(T, Nb) - ybc2_1D = zeros(T, Nb) - - if Nb == 6 - # normal situation, 2 boundary points - - # boundary matrices - Bin = spdiagm(Nt, Nin, -3 => ones(T, Nin)) - Bb = spzeros(T, Nt, Nb) - Bb[1:3, 1:3] = I(3) - Bb[(end-2):end, (end-2):end] = I(3) - if bc1 == :dirichlet - Bbc[1, 1] = 1 / 2 - Bbc[1, 5] = 1 / 2 - Bbc[2, 2] = 1 / 2 - Bbc[2, 4] = 1 / 2 - Bbc[3, 3] = 1 # Dirichlet uLe - ybc1_1D[1] = 1 - ybc1_1D[2] = 1 - ybc1_1D[3] = 1 - elseif bc1 == :pressure - error("not implemented") - elseif bc1 == :periodic - Bbc[1:3, 1:3] = -I(3) - Bbc[1:3, (end-5):(end-3)] = I(3) - Bbc[(end-2):end, 4:6] = -I(3) - Bbc[(end-2):end, (end-2):end] = I(3) - else - error("not implemented") - end - - if bc2 == :dirichlet - Bbc[end, end] = 1 / 2 - Bbc[end, end-4] = 1 / 2 - Bbc[end-1, end-1] = 1 / 2 - Bbc[end-1, end-3] = 1 / 2 - Bbc[end-2, end-2] = 1 - ybc2_1D[end-2] = 1 # uRi - ybc2_1D[end-1] = 1 - ybc2_1D[end] = 1 - elseif bc2 == :pressure - error("not implemented") - elseif bc2 == :periodic - Bbc[1:3, 1:3] = -I(3) - Bbc[1:3, (end-5):(end-3)] = I(3) - Bbc[(end-2):end, 4:6] = -I(3) - Bbc[(end-2):end, (end-2):end] = I(3) - else - error("not implemented") - end - elseif Nb == 1 - # one boundary point - error("not implemented") - end - - ybc1 = ybc1_1D - ybc2 = ybc2_1D - - Btemp = Bb / (Bbc * Bb) - B1D = Bin - Btemp * Bbc * Bin - end - - (; B1D, Btemp, ybc1, ybc2) -end diff --git a/src/boundary_conditions/bc_av_stag3.jl b/src/boundary_conditions/bc_av_stag3.jl deleted file mode 100644 index cb7091c57..000000000 --- a/src/boundary_conditions/bc_av_stag3.jl +++ /dev/null @@ -1,96 +0,0 @@ -function bc_av_stag3(Nt, Nin, Nb, bc1, bc2, h1, h2) - # total solution u is written as u = Bb*ub + Bin*uin - # the boundary conditions can be written as Bbc*u = ybc - # then u can be written entirely in terms of uin and ybc as: - # u = (Bin-Btemp*Bbc*Bin)*uin + Btemp*ybc, where - # Btemp = Bb*(Bbc*Bb)^(-1) - # Bb, Bin and Bbc depend on type of bc (Neumann/Dirichlet/periodic) - - # val1 and val2 can be scalars or vectors with either the value or the - # derivative - - # (ghost) points on staggered locations (pressure lines) - - T = typeof(h1) - - # some input checking: - if Nt != Nin + Nb - error("Number of inner points plus boundary points is not equal to total points") - end - - if Nb == 0 - # no boundary points, so simply diagonal matrix without boundary contribution - B1D = I(Nt) - Btemp = spzeros(T, Nt, 2) - ybc1 = zeros(T, 2) - ybc2 = zeros(T, 2) - else - # boundary conditions - Bbc = spzeros(T, Nb, Nt) - ybc1_1D = zeros(T, Nb) - ybc2_1D = zeros(T, Nb) - - if Nb == 6 - # normal situation, 2 boundary points - - # boundary matrices - Bin = spdiagm(Nt, Nin, -3 => ones(T, Nin)) - Bb = spzeros(T, Nt, Nb) - Bb[1:3, 1:3] = I(3) - Bb[(end-2):end, (end-2):end] = I(3) - - if bc1 == :dirichlet - Bbc[1, 1] = -1 - Bbc[1, 6] = 1 - Bbc[2, 2] = 1 / 2 # Dirichlet uLo - Bbc[2, 5] = 1 / 2 - Bbc[3, 3] = -1 - Bbc[3, 4] = 1 - ybc1_1D[1] = 0 - ybc1_1D[2] = 1 - ybc1_1D[3] = 0 - elseif bc1 == :symmetric - error("not implemented") - elseif bc1 == :periodic - Bbc[1:3, 1:3] = -I(3) - Bbc[1:3, (end-5):(end-3)] = I(3) - Bbc[(end-2):end, 4:6] = -I(3) - Bbc[(end-2):end, (end-2):end] = I(3) - else - error("not implemented") - end - - if bc2 == :dirichlet - Bbc[end, end] = -1 - Bbc[end, end-5] = 1 - Bbc[end-1, end-1] = 1 / 2 - Bbc[end-1, end-4] = 1 / 2 - Bbc[end-2, end-2] = -1 - Bbc[end-2, end-3] = 1 - ybc2_1D[end-2] = 0 - ybc2_1D[end-1] = 1 - ybc2_1D[end] = 0 - elseif bc2 == :symmetric - error("not implemented") - elseif bc2 == :periodic - # actually redundant - Bbc[1:3, 1:3] = -I(3) - Bbc[1:3, (end-5):(end-3)] = I(3) - Bbc[(end-2):end, 4:6] = -I(3) - Bbc[(end-2):end, (end-2):end] = I(3) - else - error("not implemented") - end - elseif Nb == 1 # one boundary point - error("not implemented") - end - - ybc1 = ybc1_1D - ybc2 = ybc2_1D - - Btemp = Bb / (Bbc * Bb) - B1D = Bin - Btemp * Bbc * Bin - end - - (; B1D, Btemp, ybc1, ybc2) -end diff --git a/src/boundary_conditions/bc_diff3.jl b/src/boundary_conditions/bc_diff3.jl deleted file mode 100644 index a5dbd34e9..000000000 --- a/src/boundary_conditions/bc_diff3.jl +++ /dev/null @@ -1,96 +0,0 @@ - -""" -total solution u is written as u = Bb*ub + Bin*uin -the boundary conditions can be written as Bbc*u = ybc -then u can be written entirely in terms of uin and ybc as: -u = (Bin-Btemp*Bbc*Bin)*uin + Btemp*ybc, where -Btemp = Bb*(Bbc*Bb)^(-1) -Bb, Bin and Bbc depend on type of bc (Neumann/Dirichlet/periodic) - -val1 and val2 can be scalars or vectors with either the value or the -derivative - -(ghost) points on boundary / grid lines -""" -function bc_diff3(Nt, Nin, Nb, bc1, bc2, h1, h2) - T = typeof(h1) - - # some input checking: - if Nt != Nin + Nb - error("Number of inner points plus boundary points is not equal to total points") - end - - if Nb == 0 # no boundary points, so simply diagonal matrix without boundary contribution - B1D = I(Nt) - Btemp = spzeros(T, Nt, 2) - ybc1 = zeros(T, 2) - ybc2 = zeros(T, 2) - else - # boundary conditions - Bbc = spzeros(T, Nb, Nt) - ybc1_1D = zeros(T, Nb) - ybc2_1D = zeros(T, Nb) - - if Nb == 6 - # normal situation, 2 boundary points - - # boundary matrices - Bin = spdiagm(Nt, Nin, -3 => ones(T, Nin)) - Bb = spzeros(T, Nt, Nb) - Bb[1:3, 1:3] = I(3) - Bb[end-2:end, end-2:end] = I(3) - if bc1 == :dirichlet - Bbc[1, 1] = 1 / 2 # Dirichlet - Bbc[1, 5] = 1 / 2 - Bbc[2, 2] = 1 / 2 # Dirichlet uLe - Bbc[2, 4] = 1 / 2 - Bbc[3, 3] = 1 # Dirichlet uLe - ybc1_1D[1] = 1 - ybc1_1D[2] = 1 - ybc1_1D[3] = 1 - elseif bc1 == :pressure - error("not implemented") - elseif bc1 == :periodic - Bbc[1:3, 1:3] = -I(3) - Bbc[1:3, end-5:end-3] = I(3) - Bbc[end-2:end, 4:6] = -I(3) - Bbc[end-2:end, end-2:end] = I(3) - else - error("not implemented") - end - - if bc2 == :dirichlet - Bbc[end, end] = 1 / 2 - Bbc[end, end-4] = 1 / 2 - Bbc[end-1, end-1] = 1 / 2 - Bbc[end-1, end-3] = 1 / 2 - Bbc[end-2, end-2] = 1 - ybc2_1D[end-2] = 1 # uRi - ybc2_1D[end-1] = 1 # uRi - ybc2_1D[end] = 1 - elseif bc2 == :pressure - error("not implemented") - elseif bc2 == :periodic - # actually redundant - Bbc[1:3, 1:3] = -I(3) - Bbc[1:3, end-5:end-3] = I(3) - Bbc[end-2:end, 4:6] = -I(3) - Bbc[end-2:end, end-2:end] = I(3) - else - error("not implemented") - end - - elseif Nb == 1 - # one boundary point - error("not implemented") - end - - ybc1 = ybc1_1D - ybc2 = ybc2_1D - - Btemp = Bb / (Bbc * Bb) - B1D = Bin - Btemp * Bbc * Bin - end - - (; B1D, Btemp, ybc1, ybc2) -end diff --git a/src/boundary_conditions/bc_diff_stag.jl b/src/boundary_conditions/bc_diff_stag.jl deleted file mode 100644 index a7260f2c6..000000000 --- a/src/boundary_conditions/bc_diff_stag.jl +++ /dev/null @@ -1,88 +0,0 @@ -function bc_diff_stag(Nt, Nin, Nb, bc1, bc2, h1, h2) - # Total solution u is written as u = Bb*ub + Bin*uin - # The boundary conditions can be written as Bbc*u = ybc - # Then u can be written entirely in terms of uin and ybc as: - # u = (Bin-Btemp*Bbc*Bin)*uin + Btemp*ybc, where - # Btemp = Bb*(Bbc*Bb)^(-1) - # Bb, Bin and Bbc depend on type of bc (Neumann/Dirichlet/periodic) - - # Val1 and val2 can be scalars or vectors with either the value or the - # Derivative - # (ghost) points on staggered locations (pressure lines) - - T = typeof(h1) - - # Some input checking: - if Nt != Nin + Nb - error("Number of inner points plus boundary points is not equal to total points") - end - - # Boundary conditions - Bbc = spzeros(T, Nb, Nt) - ybc1_1D = zeros(T, Nb) - ybc2_1D = zeros(T, Nb) - - if Nb == 0 - # No boundary points, so simply diagonal matrix without boundary contribution - B1D = I(Nt) - Btemp = spzeros(T, Nt, 2) - ybc1 = zeros(T, 2) - ybc2 = zeros(T, 2) - elseif Nb == 1 - # One boundary point (should not be unnecessary) - elseif Nb == 2 - # Normal situation, 2 boundary points - - # Boundary matrices - Bin = spdiagm(Nt, Nin, -1 => ones(T, Nin)) - Bb = spzeros(T, Nt, Nb) - Bb[1, 1] = 1 - Bb[end, Nb] = 1 - - if bc1 == :dirichlet - # Zeroth order (standard mirror conditions) - Bbc[1, 1] = 1 / 2 - Bbc[1, 2] = 1 / 2 - ybc1_1D[1] = 1 # ULo - elseif bc1 == :symmetric - Bbc[1, 1] = -1 - Bbc[1, 2] = 1 - ybc1_1D[1] = h1 # DuLo - elseif bc1 == :periodic - Bbc[1, 1] = -1 - Bbc[1, end-1] = 1 - Bbc[2, 2] = -1 - Bbc[2, end] = 1 - else - error("not implemented") - end - - if bc2 == :dirichlet - # Zeroth order (standard mirror conditions) - Bbc[end, end-1] = 1 / 2 - Bbc[end, end] = 1 / 2 - ybc2_1D[2] = 1 # UUp - elseif bc2 == :symmetric - Bbc[2, end-1] = -1 - Bbc[2, end] = 1 - ybc2_1D[2] = h2 # DuUp - elseif bc2 == :periodic - Bbc[1, 1] = -1 - Bbc[1, end-1] = 1 - Bbc[2, 2] = -1 - Bbc[2, end] = 1 - else - error("not implemented") - end - end - - if Nb ∈ (1, 2) - ybc1 = ybc1_1D - ybc2 = ybc2_1D - - Btemp = Bb / (Bbc * Bb) - B1D = Bin - Btemp * Bbc * Bin - end - - (; B1D, Btemp, ybc1, ybc2) -end diff --git a/src/boundary_conditions/bc_diff_stag3.jl b/src/boundary_conditions/bc_diff_stag3.jl deleted file mode 100644 index 69c3deea9..000000000 --- a/src/boundary_conditions/bc_diff_stag3.jl +++ /dev/null @@ -1,93 +0,0 @@ -function bc_diff_stag3(Nt, Nin, Nb, bc1, bc2, h1, h2) - # Total solution u is written as u = Bb*ub + Bin*uin - # The boundary conditions can be written as Bbc*u = ybc - # Then u can be written entirely in terms of uin and ybc as: - # U = (Bin-Btemp*Bbc*Bin)*uin + Btemp*ybc, where - # Btemp = Bb*(Bbc*Bb)^(-1) - # Bb, Bin and Bbc depend on type of bc (Neumann/Dirichlet/periodic) - - # Val1 and val2 can be scalars or vectors with either the value or the - # Derivative - # (ghost) points on staggered locations (pressure lines) - - T = typeof(h1) - - # Some input checking: - if Nt != Nin + Nb - error("Number of inner points plus boundary points is not equal to total points") - end - - # Boundary conditions - Bbc = spzeros(T, Nb, Nt) - ybc1_1D = zeros(T, Nb) - ybc2_1D = zeros(T, Nb) - - if Nb == 0 - # No boundary points, so simply diagonal matrix without boundary contribution - B1D = I(Nt) - Btemp = spzeros(T, Nt, 2) - ybc1 = zeros(T, 2) - ybc2 = zeros(T, 2) - elseif Nb == 6 - # Normal situation, 2 boundary points - # Boundary matrices - Bin = spdiagm(Nt, Nin, -3 => ones(T, Nin)) - Bb = spzeros(T, Nt, Nb) - Bb[1:3, 1:3] = I(3) - Bb[(end-2):end, (end-2):end] = I(3) - - if bc1 == :dirichlet - Bbc[1, 1] = 1 / 2 # Dirichlet - Bbc[1, 6] = 1 / 2 - Bbc[2, 2] = 1 / 2 # Dirichlet uLo - Bbc[2, 5] = 1 / 2 - Bbc[3, 3] = 1 / 2 # Dirichlet uLo - Bbc[3, 4] = 1 / 2 - ybc1_1D[1] = 1 - ybc1_1D[2] = 1 - ybc1_1D[3] = 1 - elseif bc1 == :symmetric - error("not implemented") - elseif bc1 == :periodic - Bbc[1:3, 1:3] = -I(3) - Bbc[1:3, (end-5):(end-3)] = I(3) - Bbc[(end-2):end, 4:6] = -I(3) - Bbc[(end-2):end, (end-2):end] = I(3) - else - error("not implemented") - end - - if bc2 == :dirichlet - Bbc[end, end] = 1 / 2 - Bbc[end, end-5] = 1 / 2 - Bbc[end-1, end-1] = 1 / 2 - Bbc[end-1, end-4] = 1 / 2 - Bbc[end-2, end-2] = 1 / 2 - Bbc[end-2, end-3] = 1 / 2 - ybc2_1D[end-2] = 1 - ybc2_1D[end-1] = 1 - ybc2_1D[end] = 1 - elseif bc2 == :symmetric - error("not implemented") - elseif bc2 == :periodic - # Actually redundant - Bbc[1:3, 1:3] = -I(3) - Bbc[1:3, (end-5):(end-3)] = I(3) - Bbc[(end-2):end, 4:6] = -I(3) - Bbc[(end-2):end, (end-2):end] = I(3) - else - error("not implemented") - end - - ybc1 = ybc1_1D - ybc2 = ybc2_1D - - Btemp = Bb / (Bbc * Bb) - B1D = Bin - Btemp * Bbc * Bin - elseif Nb == 1 - # One boundary point - error("not implemented") - end - - (; B1D, Btemp, ybc1, ybc2) -end diff --git a/src/boundary_conditions/bc_div2.jl b/src/boundary_conditions/bc_div2.jl deleted file mode 100644 index ef0816dee..000000000 --- a/src/boundary_conditions/bc_div2.jl +++ /dev/null @@ -1,87 +0,0 @@ -function bc_div2(Nt, Nin, Nb, bc1, bc2, h1, h2) - # total solution u is written as u = Bb*ub + Bin*uin - # the boundary conditions can be written as Bbc*u = ybc - # then u can be written entirely in terms of uin and ybc as: - # u = (Bin-Btemp*Bbc*Bin)*uin + Btemp*ybc, where - # Btemp = Bb*(Bbc*Bb)^(-1) - # Bb, Bin and Bbc depend on type of bc (Neumann/Dirichlet/periodic) - - # val1 and val2 can be scalars or vectors with either the value or the - # derivative - - # (ghost) points on boundary / grid lines - - T = typeof(h1) - - # some input checking: - if Nt != Nin + Nb - error("Number of inner points plus boundary points is not equal to total points") - end - - if Nb == 0 - # no boundary points, so simply diagonal matrix without boundary contribution - B1D = I(Nt) - Btemp = spzeros(T, Nt, 2) - ybc1 = zeros(T, 2) - ybc2 = zeros(T, 2) - else - # boundary conditions - Bbc = spzeros(T, Nb, Nt) - ybc1_1D = zeros(T, Nb) - ybc2_1D = zeros(T, Nb) - - if Nb == 4 - # normal situation, 4 boundary points - - # boundary matrices - Bin = spdiagm(Nt, Nin, -2 => ones(T, Nin)) - Bb = spzeros(T, Nt, Nb) - Bb[1:2, 1:2] = I(2) - Bb[end-1:end, end-1:end] = I(2) - if bc1 == :dirichlet - Bbc[1, 1] = 1 / 2 - Bbc[1, 3] = 1 / 2 - Bbc[2, 2] = 1 # Dirichlet uLe - ybc1_1D[1] = 1 - ybc1_1D[2] = 1 - elseif bc1 == :pressure - error("not implemented") - elseif bc1 == :periodic - Bbc[1:2, 1:2] = -I(2) - Bbc[1:2, end-3:end-2] = I(2) - Bbc[end-1:end, 3:4] = -I(2) - Bbc[end-1:end, end-1:end] = I(2) - else - error("not implemented") - end - - if bc2 == :dirichlet - Bbc[end, end] = 1 / 2 - Bbc[end, end-2] = 1 / 2 - Bbc[end-1, end-1] = 1 # Dirichlet uRi - ybc2_1D[end-1] = 1 - ybc2_1D[end] = 1 - elseif bc2 == :pressure - error("not implemented") - elseif bc2 == :periodic # actually redundant - Bbc[1:2, 1:2] = -I(2) - Bbc[1:2, end-3:end-2] = I(2) - Bbc[end-1:end, 3:4] = -I(2) - Bbc[end-1:end, end-1:end] = I(2) - else - error("not implemented") - end - else - # one boundary point - error("not implemented") - end - - ybc1 = ybc1_1D - ybc2 = ybc2_1D - - Btemp = Bb / (Bbc * Bb) - B1D = Bin - Btemp * Bbc * Bin - end - - (; B1D, Btemp, ybc1, ybc2) -end diff --git a/src/boundary_conditions/bc_general.jl b/src/boundary_conditions/bc_general.jl deleted file mode 100644 index c466816c0..000000000 --- a/src/boundary_conditions/bc_general.jl +++ /dev/null @@ -1,121 +0,0 @@ -""" - bc_general(Nt, Nin, Nb, bc1, bc2, h1, h2) - -Total solution `u` is written as `u = Bb*ub + Bin*uin` - -The boundary conditions can be written as `Bbc*u = ybc` - -Then `u` can be written entirely in terms of `uin` and `ybc` as: `u = -(Bin-Btemp*Bbc*Bin)*uin + Btemp*ybc`, where `Btemp = Bb/(Bbc*Bb)`. - -`Bb`, `Bin` and `Bbc` depend on type of bc (Neumann/Dirichlet/periodic) `val1` and `val2` -can be scalars or vectors with either the value or the derivative (ghost) points on -boundary/grid lines -""" -function bc_general(Nt, Nin, Nb, bc1, bc2, h1, h2) - if Nt != Nin + Nb - error("Number of inner points plus boundary points is not equal to total points") - end - - T = typeof(h1) - - # Boundary conditions - Bbc = spzeros(T, Nb, Nt) - ybc1_1D = zeros(T, Nb) - ybc2_1D = zeros(T, Nb) - - if Nb == 0 - # No boundary points, so simply diagonal matrix without boundary contribution - B1D = I(Nt) - Btemp = spzeros(T, Nt, 2) - ybc1 = zeros(T, 2) - ybc2 = zeros(T, 2) - elseif Nb == 1 - # One boundary point - Bb = spzeros(T, Nt, Nb) - diagpos = -1 - if bc1 == :dirichlet - Bbc[1, 1] = 1 - ybc1_1D[1] = 1 # uLe - Bb[1, 1] = 1 - elseif bc1 == :pressure - diagpos = 0 - elseif bc1 == :periodic - diagpos = 0 - Bbc[1, 1] = -1 - Bbc[1, end] = 1 - Bb[end, 1] = 1 - else - error("not implemented") - end - - if bc2 == :dirichlet - Bbc[Nb, end] = 1 - ybc2_1D[1] = 1 # uRi - Bb[end, Nb] = 1 - elseif bc2 == :pressure - - elseif bc2 == :periodic # Actually redundant - diagpos = 0 - Bbc[1, 1] = -1 - Bbc[1, end] = 1 - Bb[end, 1] = 1 - else - error("not implemented") - end - - # Boundary matrices - Bin = spdiagm(Nt, Nin, diagpos => ones(T, Nin)) - elseif Nb == 2 - # Normal situation, 2 boundary points - # Boundary matrices - Bin = spdiagm(Nt, Nin, -1 => ones(T, Nin)) - Bb = spzeros(T, Nt, Nb) - Bb[1, 1] = 1 - Bb[end, Nb] = 1 - - if bc1 == :dirichlet - Bbc[1, 1] = 1 - ybc1_1D[1] = 1 # uLe - elseif bc1 == :pressure - Bbc[1, 1] = -1 - Bbc[1, 2] = 1 - ybc1_1D[1] = 2 * h1 # DuLe - elseif bc1 == :periodic - Bbc[1, 1] = -1 - Bbc[1, end-1] = 1 - Bbc[Nb, 2] = -1 - Bbc[Nb, end] = 1 - else - error("not implemented") - end - - if bc2 == :dirichlet - Bbc[Nb, end] = 1 - ybc2_1D[2] = 1 # uRi - elseif bc2 == :pressure - Bbc[Nb, end-1] = -1 - Bbc[Nb, end] = 1 - ybc2_1D[2] = 2 * h2 # duRi - elseif bc2 == :periodic # Actually redundant - Bbc[1, 1] = -1 - Bbc[1, end-1] = 1 - Bbc[Nb, 2] = -1 - Bbc[Nb, end] = 1 - else - error("not implemented") - end - else - error("Nb must be 0, 1, or 2") - end - - if Nb ∈ (1, 2) - ybc1 = ybc1_1D - ybc2 = ybc2_1D - - Btemp = Bb / (Bbc * Bb) - B1D = Bin - Btemp * Bbc * Bin - end - - (; B1D, Btemp, ybc1, ybc2) -end diff --git a/src/boundary_conditions/bc_general_stag.jl b/src/boundary_conditions/bc_general_stag.jl deleted file mode 100644 index b2d9c19c8..000000000 --- a/src/boundary_conditions/bc_general_stag.jl +++ /dev/null @@ -1,124 +0,0 @@ -function bc_general_stag(Nt, Nin, Nb, bc1, bc2, h1, h2) - # Total solution u is written as u = Bb*ub + Bin*uin - # The boundary conditions can be written as Bbc*u = ybc - # Then u can be written entirely in terms of uin and ybc as: - # u = (Bin-Btemp*Bbc*Bin)*uin + Btemp*ybc, where - # Btemp = Bb*(Bbc*Bb)^(-1] - # Bb, Bin and Bbc depend on type of bc (Neumann/Dirichlet/periodic) - # Val1 and val2 can be scalars or vectors with either the value or the - # derivative - # (ghost) points on staggered locations (pressure lines) - - T = typeof(h1) - - # Some input checking: - if Nt != Nin + Nb - error("Number of inner points plus boundary points is not equal to total points") - end - - # Boundary conditions - Bbc = spzeros(T, Nb, Nt) - ybc1_1D = zeros(T, Nb) - ybc2_1D = zeros(T, Nb) - - if Nb == 0 - # No boundary points, so simply diagonal matrix without boundary contribution - B1D = I(Nt) - Btemp = spzeros(T, Nt, 2) - ybc1 = zeros(T, 2) - ybc2 = zeros(T, 2) - elseif Nb == 1 - # One boundary point - Bb = spzeros(T, Nt, Nb) - - diagpos = -1 - if bc1 == :dirichlet - Bbc[1, 1] = 1 / 2 - Bbc[1, 2] = 1 / 2 - ybc1_1D[1] = 1 # ULe - Bb[1, 1] = 1 - elseif bc1 == :symmetric - Bbc[1, 1] = -1 - Bbc[1, 2] = 1 - ybc1_1D[1] = h1 # DuLo - elseif bc1 == :periodic - Bbc[1, 1] = -1 - Bbc[1, end] = 1 - Bb[1, 1] = 1 - else - error("not implemented") - end - - if bc2 == :dirichlet - Bbc[Nb, end-1] = 1 / 2 - Bbc[Nb, end] = 1 / 2 - ybc2_1D[1] = 1 # URi - Bb[end, Nb] = 1 - elseif bc2 == :symmetric - Bbc[Nb, end-1] = -1 - Bbc[Nb, end] = 1 - ybc2_1D[1] = h2 # DuUp - elseif bc2 == :periodic - Bbc[1, 1] = -1 - Bbc[1, end] = 1 - Bb[1, 1] = 1 - else - error("not implemented") - end - - # Boundary matrices - Bin = spdiagm(Nt, Nin, diagpos => ones(T, Nin)) - elseif Nb == 2 - # Normal situation, 2 boundary points - # Boundary matrices - Bin = spdiagm(Nt, Nin, -1 => ones(T, Nin)) - Bb = spzeros(T, Nt, Nb) - Bb[1, 1] = 1 - Bb[end, Nb] = 1 - - if bc1 == :dirichlet - Bbc[1, 1] = 1 / 2 - Bbc[1, 2] = 1 / 2 - ybc1_1D[1] = 1 # ULo - elseif bc1 == :symmetric - Bbc[1, 1] = -1 - Bbc[1, 2] = 1 - ybc1_1D[1] = h1 # DuLo - elseif bc1 == :periodic - Bbc[1, 1] = -1 - Bbc[1, end-1] = 1 - Bbc[2, 2] = -1 - Bbc[2, end] = 1 - else - error("not implemented") - end - - if bc2 == :dirichlet - Bbc[end, end-1] = 1 / 2 - Bbc[end, end] = 1 / 2 - ybc2_1D[2] = 1 # UUp - elseif bc2 == :symmetric - Bbc[2, end-1] = -1 - Bbc[2, end] = 1 - ybc2_1D[2] = h2 # DuUp - elseif bc2 == :periodic - Bbc[1, 1] = -1 - Bbc[1, end-1] = 1 - Bbc[2, 2] = -1 - Bbc[2, end] = 1 - else - error("not implemented") - end - else - error("Nb must be 0, 1, or 2") - end - - if Nb ∈ (1, 2) - ybc1 = ybc1_1D - ybc2 = ybc2_1D - Btemp = Bb / (Bbc * Bb) - B1D = Bin - Btemp * Bbc * Bin - end - - (; B1D, Btemp, ybc1, ybc2) -end diff --git a/src/boundary_conditions/bc_general_stag_diff.jl b/src/boundary_conditions/bc_general_stag_diff.jl deleted file mode 100644 index a488bda8c..000000000 --- a/src/boundary_conditions/bc_general_stag_diff.jl +++ /dev/null @@ -1,88 +0,0 @@ -function bc_general_stag_diff(Nt, Nin, Nb, bc1, bc2, h1, h2) - # Total solution u is written as u = Bb*ub + Bin*uin - # The boundary conditions can be written as Bbc*u = ybc - # Then u can be written entirely in terms of uin and ybc as: - # u = (Bin-Btemp*Bbc*Bin)*uin + Btemp*ybc, where - # Btemp = Bb*(Bbc*Bb)^(-1) - # Bb, Bin and Bbc depend on type of bc (Neumann/Dirichlet/periodic) - - # Val1 and val2 can be scalars or vectors with either the value or the - # Derivative - # (ghost) points on staggered locations (pressure lines) - - T = typeof(h1) - - # Some input checking: - if Nt != Nin + Nb - error("Number of inner points plus boundary points is not equal to total points") - end - - # Boundary conditions - Bbc = spzeros(T, Nb, Nt) - ybc1_1D = zeros(T, Nb) - ybc2_1D = zeros(T, Nb) - - if Nb == 0 - # No boundary points, so simply diagonal matrix without boundary contribution - B1D = I(Nt) - Btemp = spzeros(T, Nt, 2) - ybc1 = zeros(T, 2) - ybc2 = zeros(T, 2) - elseif Nb == 1 - # One boundary point (should not be unnecessary) - elseif Nb == 2 - # Normal situation, 2 boundary points - - # Boundary matrices - Bin = spdiagm(Nt, Nin, -1 => ones(T, Nin)) - Bb = spzeros(T, Nt, Nb) - Bb[1, 1] = 1 - Bb[end, Nb] = 1 - - if bc1 == :dirichlet - Bbc[1, 1] = 3 / 8 - Bbc[1, 2] = 3 / 4 - Bbc[1, 3] = -1 / 8 - ybc1_1D[1] = 1 # ULo - elseif bc1 == :symmetric - Bbc[1, 1] = -1 - Bbc[1, 2] = 1 - ybc1_1D[1] = h1 # DuLo - elseif bc1 == :periodic - Bbc[1, 1] = -1 - Bbc[1, end-1] = 1 - Bbc[2, 2] = -1 - Bbc[2, end] = 1 - else - error("not implemented") - end - - if bc2 == :dirichlet - Bbc[end, end-2] = -1 / 8 - Bbc[end, end-1] = 3 / 4 - Bbc[end, end] = 3 / 8 - ybc2_1D[2] = 1 # UUp - elseif bc2 == :symmetric - Bbc[2, end-1] = -1 - Bbc[2, end] = 1 - ybc2_1D[2] = h2 # DuUp - elseif bc2 == :periodic - Bbc[1, 1] = -1 - Bbc[1, end-1] = 1 - Bbc[2, 2] = -1 - Bbc[2, end] = 1 - else - error("not implemented") - end - end - - if Nb ∈ (1, 2) - ybc1 = ybc1_1D - ybc2 = ybc2_1D - - Btemp = Bb / (Bbc * Bb) - B1D = Bin - Btemp * Bbc * Bin - end - - (; B1D, Btemp, ybc1, ybc2) -end diff --git a/src/boundary_conditions/bc_int2.jl b/src/boundary_conditions/bc_int2.jl deleted file mode 100644 index 2c8c8a5f6..000000000 --- a/src/boundary_conditions/bc_int2.jl +++ /dev/null @@ -1,86 +0,0 @@ -function bc_int2(Nt, Nin, Nb, bc1, bc2, h1, h2) - # total solution u is written as u = Bb*ub + Bin*uin - # the boundary conditions can be written as Bbc*u = ybc - # then u can be written entirely in terms of uin and ybc as: - # u = (Bin-Btemp*Bbc*Bin)*uin + Btemp*ybc, where - # Btemp = Bb*(Bbc*Bb)^(-1) - # Bb, Bin and Bbc depend on type of bc (Neumann/Dirichlet/periodic) - - # val1 and val2 can be scalars or vectors with either the value or the - # derivative - - # (ghost) points on boundary / grid lines - - T = typeof(h1) - - # some input checking: - if Nt != Nin + Nb - error("Number of inner points plus boundary points is not equal to total points") - end - - if Nb == 0 # no boundary points, so simply diagonal matrix without boundary contribution - B1D = I(Nt) - Btemp = spzeros(T, Nt, 2) - ybc1 = zeros(T, 2) - ybc2 = zeros(T, 2) - else - # boundary conditions - Bbc = spzeros(T, Nb, Nt) - ybc1_1D = zeros(T, Nb) - ybc2_1D = zeros(T, Nb) - - if Nb == 4 - # normal situation, 4 boundary points - - # boundary matrices - Bin = spdiagm(Nt, Nin, -2 => ones(T, Nin)) - Bb = spzeros(T, Nt, Nb) - Bb[1:2, 1:2] = I(2) - Bb[end-1:end, end-1:end] = I(2) - if bc1 == :dirichlet - Bbc[1, 1] = 1 / 2 - Bbc[1, 3] = 1 / 2 - Bbc[2, 2] = 1 # Dirichlet uLe - ybc1_1D[1] = 1 - ybc1_1D[2] = 1 - elseif bc1 == :pressure - error("not implemented") - elseif bc1 == :periodic - Bbc[1:2, 1:2] = -I(2) - Bbc[1:2, end-3:end-2] = I(2) - Bbc[end-1:end, 3:4] = -I(2) - Bbc[end-1:end, end-1:end] = I(2) - else - error("not implemented") - end - - if bc2 == :dirichlet - Bbc[end, end] = 1 / 2 - Bbc[end, end-2] = 1 / 2 - Bbc[end-1, end-1] = 1 - ybc2_1D[end-1] = 1 # uRi - ybc2_1D[end] = 1 - elseif bc2 == :pressure - error("not implemented") - elseif bc2 == :periodic # actually redundant - Bbc[1:2, 1:2] = -I(2) - Bbc[1:2, end-3:end-2] = I(2) - Bbc[end-1:end, 3:4] = -I(2) - Bbc[end-1:end, end-1:end] = I(2) - else - error("not implemented") - end - - elseif Nb == 1 # one boundary point - error("not implemented") - end - - ybc1 = ybc1_1D - ybc2 = ybc2_1D - - Btemp = Bb / (Bbc * Bb) - B1D = Bin - Btemp * Bbc * Bin - end - - (; B1D, Btemp, ybc1, ybc2) -end diff --git a/src/boundary_conditions/bc_int3.jl b/src/boundary_conditions/bc_int3.jl deleted file mode 100644 index cf8421650..000000000 --- a/src/boundary_conditions/bc_int3.jl +++ /dev/null @@ -1,93 +0,0 @@ -function bc_int3(Nt, Nin, Nb, bc1, bc2, h1, h2) - # total solution u is written as u = Bb*ub + Bin*uin - # the boundary conditions can be written as Bbc*u = ybc - # then u can be written entirely in terms of uin and ybc as: - # u = (Bin-Btemp*Bbc*Bin)*uin + Btemp*ybc, where - # Btemp = Bb*(Bbc*Bb)^(-1) - # Bb, Bin and Bbc depend on type of bc (Neumann/Dirichlet/periodic) - - # val1 and val2 can be scalars or vectors with either the value or the - # derivative - - # (ghost) points on boundary / grid lines - - T = typeof(h1) - - # some input checking: - if Nt != Nin + Nb - error("Number of inner points plus boundary points is not equal to total points") - end - - if Nb == 0 - # no boundary points, so simply diagonal matrix without boundary contribution - B1D = I(Nt) - Btemp = spzeros(T, Nt, 2) - ybc1 = zeros(T, 2) - ybc2 = zeros(T, 2) - else - # boundary conditions - Bbc = spzeros(T, Nb, Nt) - ybc1_1D = zeros(T, Nb) - ybc2_1D = zeros(T, Nb) - - if Nb == 6 - # normal situation, 2 boundary points - - # boundary matrices - Bin = spdiagm(Nt, Nin, -3 => ones(T, Nin)) - Bb = spzeros(T, Nt, Nb) - Bb[1:3, 1:3] = I(3) - Bb[end-2:end, end-2:end] = I(3) - if bc1 == :dirichlet - Bbc[1, 1] = 1 / 2 # skew-symm - Bbc[1, 5] = 1 / 2 - Bbc[2, 2] = 1 / 2 - Bbc[2, 4] = 1 / 2 - Bbc[3, 3] = 1 # Dirichlet uLe - ybc1_1D[1] = 1 - ybc1_1D[2] = 1 - ybc1_1D[3] = 1 - elseif bc1 == :pressure - error("not implemented") - elseif bc1 == :periodic - Bbc[1:3, 1:3] = -I(3) - Bbc[1:3, end-5:end-3] = I(3) - Bbc[end-2:end, 4:6] = -I(3) - Bbc[end-2:end, end-2:end] = I(3) - else - error("not implemented") - end - - if bc2 == :dirichlet - Bbc[end, end] = 1 / 2 - Bbc[end, end-4] = 1 / 2 - Bbc[end-1, end-1] = 1 / 2 - Bbc[end-1, end-3] = 1 / 2 - Bbc[end-2, end-2] = 1 - ybc2_1D[end-2] = 1 # uRi - ybc2_1D[end-1] = 1 - ybc2_1D[end] = 1 - elseif bc2 == :pressure - error("not implemented") - elseif bc2 == :periodic # actually redundant - Bbc[1:3, 1:3] = -I(3) - Bbc[1:3, end-5:end-3] = I(3) - Bbc[end-2:end, 4:6] = -I(3) - Bbc[end-2:end, end-2:end] = I(3) - else - error("not implemented") - end - elseif Nb == 1 - # one boundary point - error("not implemented") - end - - ybc1 = ybc1_1D - ybc2 = ybc2_1D - - Btemp = Bb / (Bbc * Bb) - B1D = Bin - Btemp * Bbc * Bin - end - - (; B1D, Btemp, ybc1, ybc2) -end diff --git a/src/boundary_conditions/bc_int_mixed2.jl b/src/boundary_conditions/bc_int_mixed2.jl deleted file mode 100644 index 917458ab2..000000000 --- a/src/boundary_conditions/bc_int_mixed2.jl +++ /dev/null @@ -1,74 +0,0 @@ -function bc_int_mixed2(Nt, Nin, Nb, bc1, bc2, h1, h2) - # total solution u is written as u = Bb*ub + Bin*uin - # the boundary conditions can be written as Bbc*u = ybc - # then u can be written entirely in terms of uin and ybc as: - # u = (Bin-Btemp*Bbc*Bin)*uin + Btemp*ybc, where - # Btemp = Bb*(Bbc*Bb)^(-1) - # Bb, Bin and Bbc depend on type of bc (Neumann/Dirichlet/periodic) - - # val1 and val2 can be scalars or vectors with either the value or the - # derivative - - # (ghost) points on boundary / grid lines - - T = typeof(h1) - - # some input checking: - if Nt != Nin + Nb - error("Number of inner points plus boundary points is not equal to total points") - end - - if Nb == 0 # no boundary points, so simply diagonal matrix without boundary contribution - B1D = I(Nt) - Btemp = spzeros(T, Nt, 2) - ybc1 = zeros(T, 2) - ybc2 = zeros(T, 2) - else - # boundary conditions - Bbc = spzeros(T, Nb, Nt) - ybc1_1D = zeros(T, Nb) - ybc2_1D = zeros(T, Nb) - - if bc1 == :dirichlet && bc2 == :dirichlet - # boundary matrices - Bin = spdiagm(Nt, Nin, -2 => ones(T, Nin)) - Bb = spzeros(T, Nt, Nb) - Bb[1:2, 1:2] = I(2) - Bb[end-1:end, end-1:end] = I(2) - - Bbc[1, 1] = -1 - Bbc[1, 3] = 1 - Bbc[2, 2] = 1 # Dirichlet uLe - ybc1_1D[1] = 0 - ybc1_1D[2] = 1 - - Bbc[end, end] = -1 - Bbc[end, end-2] = 1 - Bbc[end-1, end-1] = 1 - ybc2_1D[end-1] = 1 # uRi - ybc2_1D[end] = 0 - elseif bc1 == :periodic && bc2 == :periodic - # boundary matrices - Bin = spdiagm(Nt, Nin, -1 => ones(T, Nin)) - Bb = spzeros(T, Nt, Nb) - Bb[1, 1] = 1 - Bb[end-1:end, end-1:end] = I(2) - Bbc[1, 1] = -1 - Bbc[1, end-2] = 1 - Bbc[end-1, 2] = -1 - Bbc[end-1, end-1] = 1 - Bbc[end, 3] = -1 - Bbc[end, end] = 1 - else - error("not implemented") - end - - ybc1 = ybc1_1D - ybc2 = ybc2_1D - - Btemp = Bb / (Bbc * Bb) - B1D = Bin - Btemp * Bbc * Bin - end - - (; B1D, Btemp, ybc1, ybc2) -end diff --git a/src/boundary_conditions/bc_int_mixed_stag2.jl b/src/boundary_conditions/bc_int_mixed_stag2.jl deleted file mode 100644 index c14843786..000000000 --- a/src/boundary_conditions/bc_int_mixed_stag2.jl +++ /dev/null @@ -1,90 +0,0 @@ -function bc_int_mixed_stag2(Nt, Nin, Nb, bc1, bc2, h1, h2) - # total solution u is written as u = Bb*ub + Bin*uin - # the boundary conditions can be written as Bbc*u = ybc - # then u can be written entirely in terms of uin and ybc as: - # u = (Bin-Btemp*Bbc*Bin)*uin + Btemp*ybc, where - # Btemp = Bb*(Bbc*Bb)^(-1) - # Bb, Bin and Bbc depend on type of bc (Neumann/Dirichlet/periodic) - - # val1 and val2 can be scalars or vectors with either the value or the - # derivative - - # (ghost) points on staggered locations (pressure lines) - - T = typeof(h1) - - # some input checking: - if Nt != Nin + Nb - error("Number of inner points plus boundary points is not equal to total points") - end - - if Nb == 0 - # no boundary points, so simply diagonal matrix without boundary contribution - B1D = I(Nt) - Btemp = spzeros(T, Nt, 2) - ybc1 = zeros(T, 2) - ybc2 = zeros(T, 2) - else - # boundary conditions - Bbc = spzeros(T, Nb, Nt) - ybc1_1D = zeros(T, Nb) - ybc2_1D = zeros(T, Nb) - - if Nb == 4 - # normal situation, 2 boundary points - - # boundary matrices - Bin = spdiagm(Nt, Nin, -2 => ones(T, Nin)) - Bb = spzeros(T, Nt, Nb) - Bb[1:2, 1:2] = I(2) - Bb[end-1:end, end-1:end] = I(2) - - if bc1 == :dirichlet - Bbc[1, 1] = -1 # Neumann type for skew-symmetry - Bbc[1, 4] = 1 - Bbc[2, 2] = -1 - Bbc[2, 3] = 1 - ybc1_1D[1] = 0 - ybc1_1D[2] = 0 - elseif bc1 == :symmetric - error("not implemented") - elseif bc1 == :periodic - Bbc[1:2, 1:2] = -I(2) - Bbc[1:2, end-3:end-2] = I(2) - Bbc[end-1:end, 3:4] = -I(2) - Bbc[end-1:end, end-1:end] = I(2) - else - error("not implemented") - end - - if bc2 == :dirichlet - Bbc[end, end] = -1 # Neumann type for skew-symmetry - Bbc[end, end-3] = 1 - Bbc[end-1, end-1] = -1 - Bbc[end-1, end-2] = 1 - ybc2_1D[end-1] = 0 - ybc2_1D[end] = 0 - elseif bc2 == :symmetric - error("not implemented") - elseif bc2 == :periodic # actually redundant - Bbc[1:2, 1:2] = -I(2) - Bbc[1:2, end-3:end-2] = I(2) - Bbc[end-1:end, 3:4] = -I(2) - Bbc[end-1:end, end-1:end] = I(2) - else - error("not implemented") - end - elseif Nb == 1 - # one boundary point - error("not implemented") - end - - ybc1 = ybc1_1D - ybc2 = ybc2_1D - - Btemp = Bb / (Bbc * Bb) - B1D = Bin - Btemp * Bbc * Bin - end - - (; B1D, Btemp, ybc1, ybc2) -end diff --git a/src/boundary_conditions/bc_int_mixed_stag3.jl b/src/boundary_conditions/bc_int_mixed_stag3.jl deleted file mode 100644 index eb0c448d7..000000000 --- a/src/boundary_conditions/bc_int_mixed_stag3.jl +++ /dev/null @@ -1,89 +0,0 @@ -function bc_int_mixed_stag3(Nt, Nin, Nb, bc1, bc2, h1, h2) - # total solution u is written as u = Bb*ub + Bin*uin - # the boundary conditions can be written as Bbc*u = ybc - # then u can be written entirely in terms of uin and ybc as: - # u = (Bin-Btemp*Bbc*Bin)*uin + Btemp*ybc, where - # Btemp = Bb*(Bbc*Bb)^(-1) - # Bb, Bin and Bbc depend on type of bc (Neumann/Dirichlet/periodic) - - # val1 and val2 can be scalars or vectors with either the value or the - # derivative - - # (ghost) points on staggered locations (pressure lines) - - T = typeof(h1) - - # some input checking: - if Nt != Nin + Nb - error("Number of inner points plus boundary points is not equal to total points") - end - - if Nb == 0 # no boundary points, so simply diagonal matrix without boundary contribution - B1D = I(Nt) - Btemp = spzeros(T, Nt, 2) - ybc1 = zeros(T, 2) - ybc2 = zeros(T, 2) - else - # boundary conditions - Bbc = spzeros(T, Nb, Nt) - ybc1_1D = zeros(T, Nb) - ybc2_1D = zeros(T, Nb) - - if Nb == 4 - # normal situation, 2 boundary points - - # boundary matrices - Bin = spdiagm(Nt, Nin, -2 => ones(T, Nin)) - Bb = spzeros(T, Nt, Nb) - Bb[1:2, 1:2] = I(2) - Bb[end-1:end, end-1:end] = I(2) - - if bc1 == :dirichlet - Bbc[1, 1] = -1 # Neumann type for skew-symmetry - Bbc[1, 4] = 1 - Bbc[2, 2] = -1 - Bbc[2, 3] = 1 - ybc1_1D[1] = 0 - ybc1_1D[2] = 0 - elseif bc1 == :symmetric - error("not implemented") - elseif bc1 == :periodic - Bbc[1:2, 1:2] = -I(2) - Bbc[1:2, end-3:end-2] = I(2) - Bbc[end-1:end, 3:4] = -I(2) - Bbc[end-1:end, end-1:end] = I(2) - else - error("not implemented") - end - - if bc2 == :dirichlet - Bbc[end, end] = -1 # Neumann type for skew-symmetry - Bbc[end, end-3] = 1 - Bbc[end-1, end-1] = -1 - Bbc[end-1, end-2] = 1 - ybc2_1D[end-1] = 0 - ybc2_1D[end] = 0 - elseif bc2 == :symmetric - error("not implemented") - elseif bc2 == :periodic # actually redundant - Bbc[1:2, 1:2] = -I(2) - Bbc[1:2, end-3:end-2] = I(2) - Bbc[end-1:end, 3:4] = -I(2) - Bbc[end-1:end, end-1:end] = I(2) - else - error("not implemented") - end - elseif Nb == 1 - # one boundary point - error("not implemented") - end - - ybc1 = ybc1_1D - ybc2 = ybc2_1D - - Btemp = Bb / (Bbc * Bb) - B1D = Bin - Btemp * Bbc * Bin - end - - (; B1D, Btemp, ybc1, ybc2) -end diff --git a/src/boundary_conditions/bc_vectors.jl b/src/boundary_conditions/bc_vectors.jl deleted file mode 100644 index aec7bcfc7..000000000 --- a/src/boundary_conditions/bc_vectors.jl +++ /dev/null @@ -1,87 +0,0 @@ -Base.@kwdef struct BCVectors{T} - yM::Vector{T} = T[] - y_p::Vector{T} = T[] - - yAu_ux::Vector{T} = T[] - yAu_uy::Vector{T} = T[] - yAu_uz::Vector{T} = T[] - yAv_vx::Vector{T} = T[] - yAv_vy::Vector{T} = T[] - yAv_vz::Vector{T} = T[] - yAw_wx::Vector{T} = T[] - yAw_wy::Vector{T} = T[] - yAw_wz::Vector{T} = T[] - - yDiff::Vector{T} = T[] - - yIu_ux::Vector{T} = T[] - yIv_uy::Vector{T} = T[] - yIw_uz::Vector{T} = T[] - yIu_vx::Vector{T} = T[] - yIv_vy::Vector{T} = T[] - yIw_vz::Vector{T} = T[] - yIu_wx::Vector{T} = T[] - yIv_wy::Vector{T} = T[] - yIw_wz::Vector{T} = T[] - - ySu_ux::Vector{T} = T[] - ySu_uy::Vector{T} = T[] - ySu_uz::Vector{T} = T[] - ySu_vx::Vector{T} = T[] - ySu_wx::Vector{T} = T[] - - ySv_vx::Vector{T} = T[] - ySv_vy::Vector{T} = T[] - ySv_vz::Vector{T} = T[] - ySv_uy::Vector{T} = T[] - ySv_wy::Vector{T} = T[] - - ySw_wx::Vector{T} = T[] - ySw_wy::Vector{T} = T[] - ySw_wz::Vector{T} = T[] - ySw_uz::Vector{T} = T[] - ySw_vz::Vector{T} = T[] - - ydM::Vector{T} = T[] - - yAν_ux::Vector{T} = T[] - yAν_uy::Vector{T} = T[] - yAν_uz::Vector{T} = T[] - yAν_vx::Vector{T} = T[] - yAν_vy::Vector{T} = T[] - yAν_vz::Vector{T} = T[] - yAν_wx::Vector{T} = T[] - yAν_wy::Vector{T} = T[] - yAν_wz::Vector{T} = T[] - - yCux_k::Vector{T} = T[] - yCuy_k::Vector{T} = T[] - yCuz_k::Vector{T} = T[] - yCvx_k::Vector{T} = T[] - yCvy_k::Vector{T} = T[] - yCvz_k::Vector{T} = T[] - yCwx_k::Vector{T} = T[] - yCwy_k::Vector{T} = T[] - yCwz_k::Vector{T} = T[] - - yAuy_k::Vector{T} = T[] - yAuz_k::Vector{T} = T[] - yAvx_k::Vector{T} = T[] - yAvz_k::Vector{T} = T[] - yAwx_k::Vector{T} = T[] - yAwy_k::Vector{T} = T[] - - yDiffu_f::Vector{T} = T[] - yDiffv_f::Vector{T} = T[] - yDiffw_f::Vector{T} = T[] - - yIu_ux3::Vector{T} = T[] - yIv_uy3::Vector{T} = T[] - yIu_vx3::Vector{T} = T[] - yIv_vy3::Vector{T} = T[] - - yAu_ux3::Vector{T} = T[] - yAu_uy3::Vector{T} = T[] - yAv_vx3::Vector{T} = T[] - yAv_vy3::Vector{T} = T[] -end diff --git a/src/boundary_conditions/bc_vort3.jl b/src/boundary_conditions/bc_vort3.jl deleted file mode 100644 index 4fdd9edca..000000000 --- a/src/boundary_conditions/bc_vort3.jl +++ /dev/null @@ -1,83 +0,0 @@ -function bc_vort3(Nt, Nin, Nb, bc1, bc2, h1, h2) - # total solution u is written as u = Bb*ub + Bin*uin - # the boundary conditions can be written as Bbc*u = ybc - # then u can be written entirely in terms of uin and ybc as: - # u = (Bin-Btemp*Bbc*Bin)*uin + Btemp*ybc, where - # Btemp = Bb*(Bbc*Bb)^(-1) - # Bb, Bin and Bbc depend on type of bc (Neumann/Dirichlet/periodic) - - # val1 and val2 can be scalars or vectors with either the value or the - # derivative - - # (ghost) points on boundary / grid lines - - # to calculate vorticity; two ghost points at lower/left boundary, one - # ghost point upper/right boundary - - T = typeof(h1) - - # some input checking: - if Nt != Nin + Nb - error("Number of inner points plus boundary points is not equal to total points") - end - - if Nb == 0 - # no boundary points, so simply diagonal matrix without boundary contribution - B1D = I(Nt) - Btemp = spzeros(T, Nt, 2) - ybc1 = zeros(T, 2) - ybc2 = zeros(T, 2) - else - # boundary conditions - Bbc = spzeros(T, Nb, Nt) - ybc1_1D = zeros(T, Nb) - ybc2_1D = zeros(T, Nb) - - if Nb == 3 - # normal situation, 4 boundary points - - # boundary matrices - Bin = spdiagm(Nt, Nin, -2 => ones(T, Nin)) - Bb = spzeros(T, Nt, Nb) - Bb[1:2, 1:2] = I(2) - Bb[end, end] = 1 - if bc1 == :dirichlet - error("not implemented") - elseif bc1 == :pressure - error("not implemented") - elseif bc1 == :periodic - Bbc[1:2, 1:2] = -I(2) - Bbc[1:2, end-2:end-1] = I(2) - Bbc[end, 3] = -1 - Bbc[end, end] = 1 - else - error("not implemented") - end - - if bc2 == :dirichlet - error("not implemented") - elseif bc2 == :pressure - error("not implemented") - elseif bc2 == :periodic - # actually redundant - Bbc[1:2, 1:2] = -I(2) - Bbc[1:2, end-2:end-1] = I(2) - Bbc[end, 3] = -1 - Bbc[end, end] = 1 - else - error("not implemented") - end - else - # one boundary point - error("not implemented") - end - - ybc1 = ybc1_1D - ybc2 = ybc2_1D - - Btemp = Bb / (Bbc * Bb) - B1D = Bin - Btemp * Bbc * Bin - end - - (; B1D, Btemp, ybc1, ybc2) -end diff --git a/src/boundary_conditions/boundary_conditions.jl b/src/boundary_conditions/boundary_conditions.jl deleted file mode 100644 index 266de1025..000000000 --- a/src/boundary_conditions/boundary_conditions.jl +++ /dev/null @@ -1,95 +0,0 @@ -""" - BoundaryConditions(u_bc, v_bc; T = Float64, bc_unsteady, bc_type, kwargs...) - -Create discrete boundary condtions. - -Values should either be scalars or vectors. All values `(u, v, p, k, e)` are -defined at (x, y) locations, i.e. the corners of pressure volumes, so they -cover the entire domain, including corners. -""" -function BoundaryConditions( - u_bc, - v_bc; - T = Float64, - bc_type, - dudt_bc = nothing, - dvdt_bc = nothing, - bc_unsteady = !isnothing(dudt_bc), - kwargs..., -) - bc_type.u.x[1] ∈ (:dirichlet, :periodic, :pressure) || error("Wrong BC for u-left") - bc_type.u.x[2] ∈ (:dirichlet, :periodic, :pressure) || error("Wrong BC for u-right") - bc_type.u.y[1] ∈ (:dirichlet, :periodic, :symmetric) || error("Wrong BC for u-low") - bc_type.u.y[2] ∈ (:dirichlet, :periodic, :symmetric) || error("Wrong BC for u-up") - - bc_type.v.x[1] ∈ (:dirichlet, :periodic, :symmetric) || error("Wrong BC for v-left") - bc_type.v.x[2] ∈ (:dirichlet, :periodic, :symmetric) || error("Wrong BC for v-right") - bc_type.v.y[1] ∈ (:dirichlet, :periodic, :pressure) || error("Wrong BC for v-low") - bc_type.v.y[2] ∈ (:dirichlet, :periodic, :pressure) || error("Wrong BC for v-up") - - # Pressure (for boundaries marked with `:pressure`) - p∞ = zero(T) - p_bc = (; x = (p∞, p∞), y = (p∞, p∞)) - - (; bc_unsteady, bc_type..., u_bc, v_bc, dudt_bc, dvdt_bc, p_bc, kwargs...) -end - -""" - BoundaryConditions(u_bc, v_bc, w_bc; T = Float64, bc_unsteady, bc_type, kwargs...) - -Create discrete boundary condtions. - -Values should either be scalars or vectors. All values `(u, v, p, k, e)` are -defined at (x, y, z) locations, i.e. the corners of pressure volumes, so they -cover the entire domain, including corners. -""" -function BoundaryConditions( - u_bc, - v_bc, - w_bc; - T = Float64, - bc_type, - dudt_bc = nothing, - dvdt_bc = nothing, - dwdt_bc = nothing, - bc_unsteady = !isnothing(dudt_bc), - kwargs..., -) - bc_type.u.x[1] ∈ (:dirichlet, :periodic, :pressure) || error("Wrong BC for u-left") - bc_type.u.x[2] ∈ (:dirichlet, :periodic, :pressure) || error("Wrong BC for u-right") - bc_type.u.y[1] ∈ (:dirichlet, :periodic, :symmetric) || error("Wrong BC for u-low") - bc_type.u.y[2] ∈ (:dirichlet, :periodic, :symmetric) || error("Wrong BC for u-up") - bc_type.u.z[1] ∈ (:dirichlet, :periodic, :symmetric) || error("Wrong BC for u-back") - bc_type.u.z[2] ∈ (:dirichlet, :periodic, :symmetric) || error("Wrong BC for u-front") - - bc_type.v.x[1] ∈ (:dirichlet, :periodic, :symmetric) || error("Wrong BC for v-left") - bc_type.v.x[2] ∈ (:dirichlet, :periodic, :symmetric) || error("Wrong BC for v-right") - bc_type.v.y[1] ∈ (:dirichlet, :periodic, :pressure) || error("Wrong BC for v-low") - bc_type.v.y[2] ∈ (:dirichlet, :periodic, :pressure) || error("Wrong BC for v-up") - bc_type.v.z[1] ∈ (:dirichlet, :periodic, :symmetric) || error("Wrong BC for v-back") - bc_type.v.z[2] ∈ (:dirichlet, :periodic, :symmetric) || error("Wrong BC for v-front") - - bc_type.w.x[1] ∈ (:dirichlet, :periodic, :symmetric) || error("Wrong BC for w-left") - bc_type.w.x[2] ∈ (:dirichlet, :periodic, :symmetric) || error("Wrong BC for w-right") - bc_type.w.y[1] ∈ (:dirichlet, :periodic, :symmetric) || error("Wrong BC for w-low") - bc_type.w.y[2] ∈ (:dirichlet, :periodic, :symmetric) || error("Wrong BC for w-up") - bc_type.w.z[1] ∈ (:dirichlet, :periodic, :pressure) || error("Wrong BC for w-back") - bc_type.w.z[2] ∈ (:dirichlet, :periodic, :pressure) || error("Wrong BC for w-front") - - # Pressure (for boundaries marked with `:pressure`) - p∞ = zero(T) - p_bc = (; x = (p∞, p∞), y = (p∞, p∞), z = (p∞, p∞)) - - (; - bc_unsteady, - bc_type..., - u_bc, - v_bc, - w_bc, - dudt_bc, - dvdt_bc, - dwdt_bc, - p_bc, - kwargs..., - ) -end diff --git a/src/boundary_conditions/get_bc_vectors.jl b/src/boundary_conditions/get_bc_vectors.jl deleted file mode 100644 index f2c790523..000000000 --- a/src/boundary_conditions/get_bc_vectors.jl +++ /dev/null @@ -1,1020 +0,0 @@ -""" - get_bc_vectors(setup, t) - -Get boundary condition vectors. -""" -function get_bc_vectors end - -get_bc_vectors(setup, t) = get_bc_vectors(setup.grid.dimension, setup, t) - -# 2D version -function get_bc_vectors(::Dimension{2}, setup, t) - (; grid, operators, boundary_conditions, viscosity_model, Re) = setup - - (; Nux_in, Nvy_in, Np, Npx, Npy) = grid - (; xin, yin, x, y, hx, hy, xp, yp) = grid - (; Ω, indu, indv) = grid - (; order4, α) = grid - - (; Dux, Duy, Dvx, Dvy) = operators - (; Au_ux_bc, Au_uy_bc, Av_vx_bc, Av_vy_bc) = operators - (; Su_ux_bc, Su_uy_bc, Sv_vx_bc, Sv_vy_bc) = operators - (; Iu_ux_bc, Iv_uy_bc_lr, Iv_uy_bc_lu) = operators - (; Iu_vx_bc_lr, Iu_vx_bc_lu, Iv_vy_bc) = operators - (; Mx_bc, My_bc) = operators - (; Su_vx_bc_lr, Su_vx_bc_lu, Sv_uy_bc_lr, Sv_uy_bc_lu) = operators - - (; u_bc, v_bc, dudt_bc, dvdt_bc) = boundary_conditions - (; p_bc, bc_unsteady) = boundary_conditions - - if order4 - (; Au_ux_bc3, Au_uy_bc3, Av_vx_bc3, Av_vy_bc3) = operators - (; Iu_ux_bc3, Iv_uy_bc_lu3, Iv_uy_bc_lr3) = operators - (; Iu_vx_bc_lu3, Iu_vx_bc_lr3, Iv_vy_bc3) = operators - (; Su_ux_bc3, Su_uy_bc3, Sv_vx_bc3, Sv_vy_bc3) = operators - (; Diffux_div, Diffuy_div, Diffvx_div, Diffvy_div) = operators - (; Mx_bc3, My_bc3) = operators - end - - T = typeof(t) - - # TODO: Split function into allocating part (constructor?) and mutating `update!` - - ## Get BC values - uLo = u_bc.(x, y[1], t) - uUp = u_bc.(x, y[end], t) - - uLo_i = u_bc.(xin, y[1], t) - uUp_i = u_bc.(xin, y[end], t) - uLe_i = u_bc.(x[1], yp, t) - uRi_i = u_bc.(x[end], yp, t) - - vLe = v_bc.(x[1], y, t) - vRi = v_bc.(x[end], y, t) - - vLo_i = v_bc.(xp, y[1], t) - vUp_i = v_bc.(xp, y[end], t) - vLe_i = v_bc.(x[1], yin, t) - vRi_i = v_bc.(x[end], yin, t) - - if bc_unsteady - dudtLe_i = dudt_bc.(x[1], yp, t) - dudtRi_i = dudt_bc.(x[end], yp, t) - dvdtLo_i = dvdt_bc.(xp, y[1], t) - dvdtUp_i = dvdt_bc.(xp, y[end], t) - end - - ## Boundary conditions for divergence - - # Mx - # ybc = uLe_i ⊗ Mx_bc.ybc1 + uRi_i ⊗ Mx_bc.ybc2 - ybc = vec(Mx_bc.ybc1 * uLe_i') + vec(Mx_bc.ybc2 * uRi_i') - yMx = Mx_bc.Bbc * ybc - if order4 - # ybc3 = uLe_i ⊗ Mx_bc3.ybc1 + uRi_i ⊗ Mx_bc3.ybc2 - ybc3 = vec(Mx_bc3.ybc1 * uLe_i') + vec(Mx_bc3.ybc2 * uRi_i') - yMx3 = Mx_bc3.Bbc * ybc3 - yMx = α * yMx - yMx3 - end - - # My - # ybc = My_bc.ybc1 ⊗ vLo_i + My_bc.ybc2 ⊗ vUp_i - ybc = vec(vLo_i * My_bc.ybc1') + vec(vUp_i * My_bc.ybc2') - yMy = My_bc.Bbc * ybc - if order4 - ybc3 = My_bc3.ybc1 ⊗ vLo_i + My_bc3.ybc2 ⊗ vUp_i - yMy3 = My_bc3.Bbc * ybc3 - yMy = α * yMy - yMy3 - end - - yM = yMx + yMy - - # Time derivative of divergence - if bc_unsteady - ybc = dudtLe_i ⊗ Mx_bc.ybc1 + dudtRi_i ⊗ Mx_bc.ybc2 - ydMx = Mx_bc.Bbc * ybc - if order4 - ybc3 = dudtLe_i ⊗ Mx_bc3.ybc1 + dudtRi_i ⊗ Mx_bc3.ybc2 - ydMx3 = Mx_bc3.Bbc * ybc3 - ydMx = α * ydMx - ydMx3 - end - - # My - ybc = My_bc.ybc1 ⊗ dvdtLo_i + My_bc.ybc2 ⊗ dvdtUp_i - ydMy = My_bc.Bbc * ybc - if order4 - ybc3 = My_bc3.ybc1 ⊗ dvdtLo_i + My_bc3.ybc2 ⊗ dvdtUp_i - ydMy3 = My_bc3.Bbc * ybc3 - ydMy = α * ydMy - ydMy3 - end - - ydM = ydMx + ydMy - else - ydM = zeros(T, Np) - end - - ## Boundary conditions for pressure - - # Left and right side - y1D_le = zeros(T, Nux_in) - y1D_ri = zeros(T, Nux_in) - boundary_conditions.u.x[1] == :pressure && (y1D_le[1] = -1) - boundary_conditions.u.x[2] == :pressure && (y1D_ri[end] = 1) - y_px = (hy .* p_bc.x[1]) ⊗ y1D_le + (hy .* p_bc.x[2]) ⊗ y1D_ri - - # Lower and upper side - y1D_lo = zeros(T, Nvy_in) - y1D_up = zeros(T, Nvy_in) - boundary_conditions.v.y[1] == :pressure && (y1D_lo[1] = -1) - boundary_conditions.v.y[2] == :pressure && (y1D_up[end] = 1) - y_py = y1D_lo ⊗ (hx .* p_bc.y[1]) + y1D_up ⊗ (hx .* p_bc.y[2]) - - y_p = [y_px; y_py] - - ## Boundary conditions for averaging - # Au_ux - ybc = uLe_i ⊗ Au_ux_bc.ybc1 + uRi_i ⊗ Au_ux_bc.ybc2 - yAu_ux = Au_ux_bc.Bbc * ybc - - # Au_uy - ybc = Au_uy_bc.ybc1 ⊗ uLo_i + Au_uy_bc.ybc2 ⊗ uUp_i - yAu_uy = Au_uy_bc.Bbc * ybc - - # Av_vx - ybc = vLe_i ⊗ Av_vx_bc.ybc1 + vRi_i ⊗ Av_vx_bc.ybc2 - yAv_vx = Av_vx_bc.Bbc * ybc - - # Av_vy - ybc = Av_vy_bc.ybc1 ⊗ vLo_i + Av_vy_bc.ybc2 ⊗ vUp_i - yAv_vy = Av_vy_bc.Bbc * ybc - - if order4 - # Au_ux - ybc3 = uLe_i ⊗ Au_ux_bc3.ybc1 + uRi_i ⊗ Au_ux_bc3.ybc2 - yAu_ux3 = Au_ux_bc3.Bbc * ybc3 - - # Au_uy - ybc3 = Au_uy_bc3.ybc1 ⊗ uLo_i + Au_uy_bc3.ybc2 ⊗ uUp_i - yAu_uy3 = Au_uy_bc3.Bbc * ybc3 - - # Av_vx - ybc3 = vLe_i ⊗ Av_vx_bc3.ybc1 + vRi_i ⊗ Av_vx_bc3.ybc2 - yAv_vx3 = Av_vx_bc3.Bbc * ybc3 - - # Av_vy - ybc3 = Av_vy_bc3.ybc1 ⊗ vLo_i + Av_vy_bc3.ybc2 ⊗ vUp_i - yAv_vy3 = Av_vy_bc3.Bbc * ybc3 - end - - ## Boundary conditions for diffusion - if order4 - ybc1 = uLe_i ⊗ Su_ux_bc.ybc1 + uRi_i ⊗ Su_ux_bc.ybc2 - ybc3 = uLe_i ⊗ Su_ux_bc3.ybc1 + uRi_i ⊗ Su_ux_bc3.ybc2 - ySu_ux = α * Su_ux_bc.Bbc * ybc1 - Su_ux_bc3.Bbc * ybc3 - - ybc1 = Su_uy_bc.ybc1 ⊗ uLo_i + Su_uy_bc.ybc2 ⊗ uUp_i - ybc3 = Su_uy_bc3.ybc1 ⊗ uLo_i + Su_uy_bc3.ybc2 ⊗ uUp_i - ySu_uy = α * Su_uy_bc.Bbc * ybc1 - Su_uy_bc3.Bbc * ybc3 - - ybc1 = vLe_i ⊗ Sv_vx_bc.ybc1 + vRi_i ⊗ Sv_vx_bc.ybc2 - ybc3 = vLe_i ⊗ Sv_vx_bc3.ybc1 + vRi_i ⊗ Sv_vx_bc3.ybc2 - ySv_vx = α * Sv_vx_bc.Bbc * ybc1 - Sv_vx_bc3.Bbc * ybc3 - - ybc1 = Sv_vy_bc.ybc1 ⊗ vLo_i + Sv_vy_bc.ybc2 ⊗ vUp_i - ybc3 = Sv_vy_bc3.ybc1 ⊗ vLo_i + Sv_vy_bc3.ybc2 ⊗ vUp_i - ySv_vy = α * Sv_vy_bc.Bbc * ybc1 - Sv_vy_bc3.Bbc * ybc3 - - if viscosity_model isa LaminarModel - yDiffu = Diffux_div * ySu_ux + Diffuy_div * ySu_uy - yDiffv = Diffvx_div * ySv_vx + Diffvy_div * ySv_vy - yDiff = [yDiffu; yDiffv] - else - error("fourth order turbulent diffusion not implemented") - end - else - # Su_ux - ybc = uLe_i ⊗ Su_ux_bc.ybc1 + uRi_i ⊗ Su_ux_bc.ybc2 - ySu_ux = Su_ux_bc.Bbc * ybc - - # Su_uy - ybc = Su_uy_bc.ybc1 ⊗ uLo_i + Su_uy_bc.ybc2 ⊗ uUp_i - ySu_uy = Su_uy_bc.Bbc * ybc - - Sv_uy = Sv_uy_bc_lr.B2D * Sv_uy_bc_lu.B2D - - # Sv_uy (left/right) - ybc = vLe ⊗ Sv_uy_bc_lr.ybc1 + vRi ⊗ Sv_uy_bc_lr.ybc2 - ySv_uy_lr = Sv_uy_bc_lr.Bbc * ybc - - # Iv_uy (low/up) - ybc = Sv_uy_bc_lu.ybc1 ⊗ vLo_i + Sv_uy_bc_lu.ybc2 ⊗ vUp_i - ySv_uy_lu = Sv_uy_bc_lr.B2D * Sv_uy_bc_lu.Bbc * ybc - - ySv_uy = ySv_uy_lr + ySv_uy_lu - - # Su_vx (low/up) - ybc = Su_vx_bc_lu.ybc1 ⊗ uLo + Su_vx_bc_lu.ybc2 ⊗ uUp - ySu_vx_lu = Su_vx_bc_lu.Bbc * ybc - - # Su_vx (left/right) - ybc = uLe_i ⊗ Su_vx_bc_lr.ybc1 + uRi_i ⊗ Su_vx_bc_lr.ybc2 - ySu_vx_lr = Su_vx_bc_lu.B2D * Su_vx_bc_lr.Bbc * ybc - ySu_vx = ySu_vx_lr + ySu_vx_lu - - # Sv_vx - ybc = vLe_i ⊗ Sv_vx_bc.ybc1 + vRi_i ⊗ Sv_vx_bc.ybc2 - ySv_vx = Sv_vx_bc.Bbc * ybc - - # Sv_vy - ybc = Sv_vy_bc.ybc1 ⊗ vLo_i + Sv_vy_bc.ybc2 ⊗ vUp_i - ySv_vy = Sv_vy_bc.Bbc * ybc - - yDiffu = Dux * ySu_ux + Duy * ySu_uy - yDiffv = Dvx * ySv_vx + Dvy * ySv_vy - yDiff = [yDiffu; yDiffv] - - Ωu⁻¹ = 1 ./ Ω[indu] - Ωv⁻¹ = 1 ./ Ω[indv] - - # Diffusive terms in finite-difference setting, without viscosity - yDiffu_f = Ωu⁻¹ .* (Dux * ySu_ux + Duy * ySu_uy) - yDiffv_f = Ωv⁻¹ .* (Dvx * ySv_vx + Dvy * ySv_vy) - end - - ## Boundary conditions for interpolation - - # Iu_ux - ybc = uLe_i ⊗ Iu_ux_bc.ybc1 + uRi_i ⊗ Iu_ux_bc.ybc2 - yIu_ux = Iu_ux_bc.Bbc * ybc - if order4 - ybc3 = uLe_i ⊗ Iu_ux_bc3.ybc1 + uRi_i ⊗ Iu_ux_bc3.ybc2 - yIu_ux3 = Iu_ux_bc3.Bbc * ybc3 - end - - # Iv_uy (left/right) - ybc = vLe ⊗ Iv_uy_bc_lr.ybc1 + vRi ⊗ Iv_uy_bc_lr.ybc2 - yIv_uy_lr = Iv_uy_bc_lr.Bbc * ybc - - # Iv_uy (low/up) - ybc = Iv_uy_bc_lu.ybc1 ⊗ vLo_i + Iv_uy_bc_lu.ybc2 ⊗ vUp_i - yIv_uy_lu = Iv_uy_bc_lr.B2D * Iv_uy_bc_lu.Bbc * ybc - yIv_uy = yIv_uy_lr + yIv_uy_lu - - if order4 - if boundary_conditions.v.y[1] == :dirichlet - vLe_ext = [2 * vLe[1] - vLe[2]; vLe] - vRi_ext = [2 * vRi[1] - vRi[2]; vRi] - elseif boundary_conditions.v.y[1] == :periodic - vLe_ext = [0; vLe] - vRi_ext = [0; vRi] - elseif boundary_conditions.v.y[1] == :pressure - vLe_ext = [vLe[2]; vLe] - vRi_ext = [vRi[2]; vRi] - end - if boundary_conditions.v.y[2] == :dirichlet - vLe_ext = [vLe_ext; 2 * vLe[end] - vLe[end-1]] - vRi_ext = [vRi_ext; 2 * vRi[1] - vRi[2]] - elseif boundary_conditions.v.y[2] == :periodic - vLe_ext = [vLe_ext; 0] - vRi_ext = [vRi_ext; 0] - elseif boundary_conditions.v.y[2] == :pressure - vLe_ext = [vLe_ext; vLe[end-1]] - vRi_ext = [vRi_ext; vRi[end-1]] - end - ybc3 = vLe_ext ⊗ Iv_uy_bc_lr3.ybc1 + vRi_ext ⊗ Iv_uy_bc_lr3.ybc2 - yIv_uy_lr3 = Iv_uy_bc_lr3.Bbc * ybc3 - - ybc3 = Iv_uy_bc_lu3.ybc1 ⊗ vLo_i + Iv_uy_bc_lu3.ybc2 ⊗ vUp_i - yIv_uy_lu3 = Iv_uy_bc_lr3.B2D * Iv_uy_bc_lu3.Bbc * ybc3 - yIv_uy3 = yIv_uy_lr3 + yIv_uy_lu3 - end - - # Iu_vx (low/up) - ybc = Iu_vx_bc_lu.ybc1 ⊗ uLo + Iu_vx_bc_lu.ybc2 ⊗ uUp - yIu_vx_lu = Iu_vx_bc_lu.Bbc * ybc - - # Iu_vx (left/right) - ybc = uLe_i ⊗ Iu_vx_bc_lr.ybc1 + uRi_i ⊗ Iu_vx_bc_lr.ybc2 - yIu_vx_lr = Iu_vx_bc_lu.B2D * Iu_vx_bc_lr.Bbc * ybc - yIu_vx = yIu_vx_lr + yIu_vx_lu - - if order4 - if boundary_conditions.u.x[1] == :dirichlet - uLo_ext = [2 * uLo[1] - uLo[2]; uLo] - uUp_ext = [2 * uUp[1] - uUp[2]; uUp] - elseif boundary_conditions.u.x[1] == :periodic - uLo_ext = [0; uLo] - uUp_ext = [0; uUp] - elseif boundary_conditions.u.x[1] == :pressure - uLo_ext = [uLo[2]; uLo] - uUp_ext = [uUp[2]; uUp] - end - if boundary_conditions.u.x[2] == :dirichlet - uLo_ext = [uLo_ext; 2 * uLo[end] - uLo[end-1]] - uUp_ext = [uUp_ext; 2 * uUp[1] - uUp[2]] - elseif boundary_conditions.u.x[2] == :periodic - uLo_ext = [uLo_ext; 0] - uUp_ext = [uUp_ext; 0] - elseif boundary_conditions.u.x[2] == :pressure - uLo_ext = [uLo_ext; uLo[end-1]] - uUp_ext = [uUp_ext; uUp[end-1]] - end - ybc3 = Iu_vx_bc_lu3.ybc1 ⊗ uLo_ext + Iu_vx_bc_lu3.ybc2 ⊗ uUp_ext - yIu_vx_lu3 = Iu_vx_bc_lu3.Bbc * ybc3 - - ybc3 = uLe_i ⊗ Iu_vx_bc_lr3.ybc1 + uRi_i ⊗ Iu_vx_bc_lr3.ybc2 - yIu_vx_lr3 = Iu_vx_bc_lu3.B2D * Iu_vx_bc_lr3.Bbc * ybc3 - yIu_vx3 = yIu_vx_lr3 + yIu_vx_lu3 - end - - # Iv_vy - ybc = Iv_vy_bc.ybc1 ⊗ vLo_i + Iv_vy_bc.ybc2 ⊗ vUp_i - yIv_vy = Iv_vy_bc.Bbc * ybc - if order4 - ybc3 = Iv_vy_bc3.ybc1 ⊗ vLo_i + Iv_vy_bc3.ybc2 ⊗ vUp_i - yIv_vy3 = Iv_vy_bc3.Bbc * ybc3 - end - - ## Group BC vectors - bc_vectors = (; - yM, - ydM, - y_p, - yAu_ux, - yAu_uy, - yAv_vx, - yAv_vy, - yDiff, - yIu_ux, - yIv_uy, - yIu_vx, - yIv_vy, - ) - - if order4 - bc_vectors = (; - bc_vectors..., - yAu_ux3, - yAu_uy3, - yAv_vx3, - yAv_vy3, - yIu_ux3, - yIv_uy3, - yIu_vx3, - yIv_vy3, - ) - else - # Use values directly (see diffusion.jl and strain_tensor.jl) - bc_vectors = (; - bc_vectors..., - ySu_ux, - ySu_uy, - ySu_vx, - ySv_vx, - ySv_vy, - ySv_uy, - yDiffu_f, - yDiffv_f, - ) - end - - if viscosity_model isa Union{QRModel,SmagorinskyModel,MixingLengthModel} - (; Aν_vy_bc) = operators - (; Cux_k_bc, Cuy_k_bc, Cvx_k_bc, Cvy_k_bc, Auy_k_bc, Avx_k_bc) = operators - - # Set BC for turbulent viscosity nu_t - # In the periodic case, the value of nu_t is not needed - # In all other cases, homogeneous (zero) Neumann conditions are used - nuLe = zeros(T, Npy) - nuRi = zeros(T, Npy) - nuLo = zeros(T, Npx) - nuUp = zeros(T, Npx) - - ## Nu_ux - (; Aν_ux_bc) = operators - ybc = nuLe ⊗ Aν_ux_bc.ybc1 + nuRi ⊗ Aν_ux_bc.ybc2 - yAν_ux = Aν_ux_bc.Bbc * ybc - - ## Nu_uy - (; Aν_uy_bc_lr, Aν_uy_bc_lu) = operators - - nuLe_i = [nuLe[1]; nuLe; nuLe[end]] - nuRi_i = [nuRi[1]; nuRi; nuRi[end]] - - # In x-direction - ybc = nuLe_i ⊗ Aν_uy_bc_lr.ybc1 + nuRi_i ⊗ Aν_uy_bc_lr.ybc2 - yAν_uy_lr = Aν_uy_bc_lr.B2D * ybc - - # In y-direction - ybc = Aν_uy_bc_lu.ybc1 ⊗ nuLo + Aν_uy_bc_lu.ybc2 ⊗ nuUp - yAν_uy_lu = Aν_uy_bc_lu.B2D * ybc - - yAν_uy = yAν_uy_lu + yAν_uy_lr - - ## Nu_vx - (; Aν_vx_bc_lr, Aν_vx_bc_lu) = operators - - nuLo_i = [nuLo[1]; nuLo; nuLo[end]] - nuUp_i = [nuUp[1]; nuUp; nuUp[end]] - - # In y-direction - ybc = Aν_vx_bc_lu.ybc1 ⊗ nuLo_i + Aν_vx_bc_lu.ybc2 ⊗ nuUp_i - yAν_vx_lu = Aν_vx_bc_lu.B2D * ybc - - # In x-direction - ybc = nuLe ⊗ Aν_vx_bc_lr.ybc1 + nuRi ⊗ Aν_vx_bc_lr.ybc2 - yAν_vx_lr = Aν_vx_bc_lr.B2D * ybc - - yAν_vx = yAν_vx_lu + yAν_vx_lr - - ## Nu_vy - ybc = Aν_vy_bc.ybc1 ⊗ nuLo + Aν_vy_bc.ybc2 ⊗ nuUp - yAν_vy = Aν_vy_bc.Bbc * ybc - - # Set BC for getting du/dx, du/dy, dv/dx, dv/dy at cell centers - - uLo_p = u_bc.(xp, y[1], t) - uUp_p = u_bc.(xp, y[end], t) - - vLe_p = v_bc.(x[1], yp, t) - vRi_p = v_bc.(x[end], yp, t) - - ybc = uLe_i ⊗ Cux_k_bc.ybc1 + uRi_i ⊗ Cux_k_bc.ybc2 - yCux_k = Cux_k_bc.Bbc * ybc - - ybc = uLe_i ⊗ Auy_k_bc.ybc1 + uRi_i ⊗ Auy_k_bc.ybc2 - yAuy_k = Auy_k_bc.Bbc * ybc - - ybc = Cuy_k_bc.ybc1 ⊗ uLo_p + Cuy_k_bc.ybc2 ⊗ uUp_p - yCuy_k = Cuy_k_bc.Bbc * ybc - - ybc = Avx_k_bc.ybc1 ⊗ vLo_i + Avx_k_bc.ybc2 ⊗ vUp_i - yAvx_k = Avx_k_bc.Bbc * ybc - - ybc = vLe_p ⊗ Cvx_k_bc.ybc1 + vRi_p ⊗ Cvx_k_bc.ybc2 - yCvx_k = Cvx_k_bc.Bbc * ybc - - ybc = Cvy_k_bc.ybc1 ⊗ vLo_i + Cvy_k_bc.ybc2 ⊗ vUp_i - yCvy_k = Cvy_k_bc.Bbc * ybc - - bc_vectors = (; - bc_vectors..., - yAν_ux, - yAν_uy, - yAν_vx, - yAν_vy, - yCux_k, - yCuy_k, - yCvx_k, - yCvy_k, - yAuy_k, - yAvx_k, - ) - end - - bc_vectors -end - -# 3D version -function get_bc_vectors(::Dimension{3}, setup, t) - (; grid, operators, boundary_conditions, viscosity_model, Re) = setup - - (; u_bc, v_bc, w_bc, dudt_bc, dvdt_bc, dwdt_bc) = boundary_conditions - (; p_bc, bc_unsteady) = boundary_conditions - - (; Ω, indu, indv, indw) = grid - (; Nz, Np, Npx, Npy, Npz) = grid - (; Nux_in, Nux_b, Nux_t, Nuy_in, Nuy_b, Nuy_t, Nuz_in, Nuz_b, Nuz_t) = grid - (; Nvx_in, Nvx_b, Nvx_t, Nvy_in, Nvy_b, Nvy_t, Nvz_in, Nuz_b, Nvz_t) = grid - (; Nwx_in, Nwx_b, Nwx_t, Nwy_in, Nwy_b, Nwy_t, Nwz_in, Nwz_b, Nwz_t) = grid - (; xin, yin, zin, x, y, z, hx, hy, hz, xp, yp, zp) = grid - - (; Dux, Duy, Duz, Dvx, Dvy, Dvz, Dwx, Dwy, Dwz) = operators - - (; Au_ux_bc, Au_uy_bc, Au_uz_bc) = operators - (; Av_vx_bc, Av_vy_bc, Av_vz_bc) = operators - (; Aw_wx_bc, Aw_wy_bc, Aw_wz_bc) = operators - - (; Su_ux_bc, Su_uy_bc, Su_uz_bc) = operators - (; Sv_vx_bc, Sv_vy_bc, Sv_vz_bc) = operators - (; Sw_wx_bc, Sw_wy_bc, Sw_wz_bc) = operators - - (; Mx_bc, My_bc, Mz_bc) = operators - - (; Iu_ux_bc, Iv_vy_bc, Iw_wz_bc) = operators - (; Iv_uy_bc_lr, Iv_uy_bc_lu, Iw_uz_bc_lr, Iw_uz_bc_bf) = operators - (; Iu_vx_bc_lr, Iu_vx_bc_lu, Iw_vz_bc_lu, Iw_vz_bc_bf) = operators - (; Iu_wx_bc_lr, Iu_wx_bc_bf, Iv_wy_bc_lu, Iv_wy_bc_bf) = operators - - (; Sv_uy_bc_lr, Sv_uy_bc_lu, Sw_uz_bc_lr, Sw_uz_bc_bf) = operators - (; Su_vx_bc_lr, Su_vx_bc_lu, Sw_vz_bc_lu, Sw_vz_bc_bf) = operators - (; Su_wx_bc_lr, Su_wx_bc_bf, Sv_wy_bc_lu, Sv_wy_bc_bf) = operators - - T = typeof(t) - - # TODO: Split up function into allocating part (constructor?) and mutating `update!` - - ## Get bc values - uLe_i = reshape(u_bc.(x[1], yp, zp', t), :) - uRi_i = reshape(u_bc.(x[end], yp, zp', t), :) - uLo_i = reshape(u_bc.(xin, y[1], zp', t), :) - uUp_i = reshape(u_bc.(xin, y[end], zp', t), :) - uLo_i2 = reshape(u_bc.(x, y[1], zp', t), :) - uUp_i2 = reshape(u_bc.(x, y[end], zp', t), :) - uBa_i = reshape(u_bc.(xin, yp', z[1], t), :) - uFr_i = reshape(u_bc.(xin, yp', z[end], t), :) - uBa_i2 = reshape(u_bc.(x, yp', z[1], t), :) - uFr_i2 = reshape(u_bc.(x, yp', z[end], t), :) - - vLe_i = reshape(v_bc.(x[1], yin, zp', t), :) - vRi_i = reshape(v_bc.(x[end], yin, zp', t), :) - vLe_i2 = reshape(v_bc.(x[1], y, zp', t), :) - vRi_i2 = reshape(v_bc.(x[end], y, zp', t), :) - vLo_i = reshape(v_bc.(xp, y[1], zp', t), :) - vUp_i = reshape(v_bc.(xp, y[end], zp', t), :) - vBa_i = reshape(v_bc.(xp, yin', z[1], t), :) - vFr_i = reshape(v_bc.(xp, yin', z[end], t), :) - vBa_i2 = reshape(v_bc.(xp, y', z[1], t), :) - vFr_i2 = reshape(v_bc.(xp, y', z[end], t), :) - - wLe_i = reshape(w_bc.(x[1], yp, zin', t), :) - wRi_i = reshape(w_bc.(x[end], yp, zin', t), :) - wLe_i2 = reshape(w_bc.(x[1], yp, z', t), :) - wRi_i2 = reshape(w_bc.(x[end], yp, z', t), :) - wLo_i = reshape(w_bc.(xp, y[1], zin', t), :) - wUp_i = reshape(w_bc.(xp, y[end], zin', t), :) - wLo_i2 = reshape(w_bc.(xp, y[1], z', t), :) - wUp_i2 = reshape(w_bc.(xp, y[end], z', t), :) - wBa_i = reshape(w_bc.(xp, yp', z[1], t), :) - wFr_i = reshape(w_bc.(xp, yp', z[end], t), :) - - if bc_unsteady - dudtLe_i = reshape(dudt_bc.(x[1], yp, zp', t), :) - dudtRi_i = reshape(dudt_bc.(x[end], yp, zp', t), :) - dvdtLo_i = reshape(dvdt_bc.(xp, y[1], zp', t), :) - dvdtUp_i = reshape(dvdt_bc.(xp, y[end], zp', t), :) - dwdtBa_i = reshape(dwdt_bc.(xp, yp', z[1], t), :) - dwdtFr_i = reshape(dwdt_bc.(xp, yp', z[end], t), :) - end - - ## Boundary conditions for divergence - - # Mx - ybc = uLe_i ⊗ Mx_bc.ybc1 + uRi_i ⊗ Mx_bc.ybc2 - yMx = Mx_bc.Bbc * ybc - - # My - ybc = vLo_i ⊗ My_bc.ybc1 + vUp_i ⊗ My_bc.ybc2 - ybc = reshape(permutedims(reshape(ybc, Nvy_b, Nvx_in, Nvz_in), (2, 1, 3)), :) - yMy = My_bc.Bbc * ybc - - # Mz - ybc = Mz_bc.ybc1 ⊗ wBa_i + Mz_bc.ybc2 ⊗ wFr_i - yMz = Mz_bc.Bbc * ybc - - yM = yMx + yMy + yMz - - # Time derivative of divergence - if bc_unsteady - ybc = dudtLe_i ⊗ Mx_bc.ybc1 + dudtRi_i ⊗ Mx_bc.ybc2 - ydMx = Mx_bc.Bbc * ybc - - # My - order of kron is not correct, so reshape - ybc = dvdtLo_i ⊗ My_bc.ybc1 + dvdtUp_i ⊗ My_bc.ybc2 - ybc = reshape(ybc, Nvy_b, Nvx_in, Nvz_in) - ybc = permutedims(ybc, (2, 1, 3)) - ybc = reshape(ybc, :) - ydMy = My_bc.Bbc * ybc - - # Mz - ybc = Mz_bc.ybc1 ⊗ dwdtBa_i + Mz_bc.ybc2 ⊗ dwdtFr_i - ydMz = Mz_bc.Bbc * ybc - - ydM = ydMx + ydMy + ydMz - else - ydM = zeros(T, Np) - end - - ## Boundary conditions for pressure - - # Left and right side - y1D_le = zeros(T, Nux_in) - y1D_ri = zeros(T, Nux_in) - boundary_conditions.u.x[1] == :pressure && (y1D_le[1] = -1) - boundary_conditions.u.x[2] == :pressure && (y1D_ri[end] = 1) - y_px = (p_bc.x[1] .* (hz ⊗ hy)) ⊗ y1D_le + (p_bc.x[2] .* (hz ⊗ hy)) ⊗ y1D_ri - - # Lower and upper side (order of kron not correct, so reshape) - y1D_lo = zeros(T, Nvy_in) - y1D_up = zeros(T, Nvy_in) - boundary_conditions.v.y[1] == :pressure && (y1D_lo[1] = -1) - boundary_conditions.v.y[2] == :pressure && (y1D_up[end] = 1) - y_py = (p_bc.y[1] .* (hz ⊗ hx)) ⊗ y1D_lo + (p_bc.y[2] .* (hz ⊗ hx)) ⊗ y1D_up - y_py = reshape(permutedims(reshape(y_py, Nvy_in, Nvx_in, Nvz_in), (2, 1, 3)), :) - - # Back and front side - y1D_ba = zeros(T, Nwz_in) - y1D_fr = zeros(T, Nwz_in) - boundary_conditions.w.z[1] == :pressure && (y1D_ba[1] = -1) - boundary_conditions.w.z[2] == :pressure && (y1D_fr[end] = 1) - y_pz = y1D_ba ⊗ (p_bc.z[1] .* (hy ⊗ hx)) + y1D_fr ⊗ (p_bc.z[2] .* (hy ⊗ hx)) - - y_p = [y_px; y_py; y_pz] - - ## Boundary conditions for averaging - # Au_ux - ybc = uLe_i ⊗ Au_ux_bc.ybc1 + uRi_i ⊗ Au_ux_bc.ybc2 - yAu_ux = Au_ux_bc.Bbc * ybc - - # Au_uy (order of kron is not correct, so permute) - ybc = Au_uy_bc.ybc1 ⊗ uLo_i + Au_uy_bc.ybc2 ⊗ uUp_i - ybc = reshape(permutedims(reshape(ybc, Nuy_b, Nux_in, Nuz_in), (2, 3, 1)), :) - yAu_uy = Au_uy_bc.Bbc * ybc - - # Au_uz - ybc = Au_uz_bc.ybc1 ⊗ uBa_i + Au_uz_bc.ybc2 ⊗ uFr_i - yAu_uz = Au_uz_bc.Bbc * ybc - - # Av_vx - ybc = vLe_i ⊗ Av_vx_bc.ybc1 + vRi_i ⊗ Av_vx_bc.ybc2 - yAv_vx = Av_vx_bc.Bbc * ybc - - # Av_vy (order of kron is not correct, so permute) - ybc = Av_vy_bc.ybc1 ⊗ vLo_i + Av_vy_bc.ybc2 ⊗ vUp_i - ybc = reshape(permutedims(reshape(ybc, Nvy_b, Nvx_in, Nvz_in), (2, 3, 1)), :) - yAv_vy = Av_vy_bc.Bbc * ybc - - # Av_vz - ybc = Av_vz_bc.ybc1 ⊗ vBa_i + Av_vz_bc.ybc2 ⊗ vFr_i - yAv_vz = Av_vz_bc.Bbc * ybc - - # Aw_wx - ybc = wLe_i ⊗ Aw_wx_bc.ybc1 + wRi_i ⊗ Aw_wx_bc.ybc2 - yAw_wx = Aw_wx_bc.Bbc * ybc - - # Aw_wy (order of kron is not correct, so permute) - ybc = wLo_i ⊗ Aw_wy_bc.ybc1 + wUp_i ⊗ Aw_wy_bc.ybc2 - ybc = reshape(permutedims(reshape(ybc, Nwy_b, Nwx_in, Nwz_in), (2, 3, 1)), :) - yAw_wy = Aw_wy_bc.Bbc * ybc - - # Aw_wz - ybc = Aw_wz_bc.ybc1 ⊗ wBa_i + Aw_wz_bc.ybc2 ⊗ wFr_i - yAw_wz = Aw_wz_bc.Bbc * ybc - - ## Bounary conditions for diffusion - - # Su_ux - ybc = uLe_i ⊗ Su_ux_bc.ybc1 + uRi_i ⊗ Su_ux_bc.ybc2 - ySu_ux = Su_ux_bc.Bbc * ybc - - # Su_uy - ybc = uLo_i ⊗ Su_uy_bc.ybc1 + uUp_i ⊗ Su_uy_bc.ybc2 - ybc = reshape(permutedims(reshape(ybc, Nuy_b, Nux_in, Nuz_in), (2, 1, 3)), :) - ySu_uy = Su_uy_bc.Bbc * ybc - - # Su_uz - ybc = Su_uz_bc.ybc1 ⊗ uBa_i + Su_uz_bc.ybc2 ⊗ uFr_i - ySu_uz = Su_uz_bc.Bbc * ybc - - # Sv_uy (left/right) - ybc = vLe_i2 ⊗ Sv_uy_bc_lr.ybc1 + vRi_i2 ⊗ Sv_uy_bc_lr.ybc2 - ySv_uy_lr = Sv_uy_bc_lr.Bbc * ybc - - # Sv_uy (low/up) (order of kron is not correct, so permute) - ybc = Sv_uy_bc_lu.ybc1 ⊗ vLo_i + Sv_uy_bc_lu.ybc2 ⊗ vUp_i - Nb = Nuy_in + 1 - Nvy_in - Nb != 0 && (ybc = reshape(permutedims(reshape(ybc, Nb, Nvx_in, Nvz_in), (2, 1, 3)), :)) - ySv_uy_lu = Sv_uy_bc_lr.B3D * (Sv_uy_bc_lu.Bbc * ybc) - - # Sv_uy - ySv_uy = ySv_uy_lr + ySv_uy_lu - - # Sw_uz (left/right) - ybc = wLe_i2 ⊗ Sw_uz_bc_lr.ybc1 + wRi_i2 ⊗ Sw_uz_bc_lr.ybc2 - ySw_uz_lr = Sw_uz_bc_lr.Bbc * ybc - - # Sw_uz (back/front) - ybc = Sw_uz_bc_bf.ybc1 ⊗ wBa_i + Sw_uz_bc_bf.ybc2 ⊗ wFr_i - ySw_uz_bf = Sw_uz_bc_lr.B3D * Sw_uz_bc_bf.Bbc * ybc - - # Sw_uz - ySw_uz = ySw_uz_lr + ySw_uz_bf - - # Sv_vx - ybc = vLe_i ⊗ Sv_vx_bc.ybc1 + vRi_i ⊗ Sv_vx_bc.ybc2 - ySv_vx = Sv_vx_bc.Bbc * ybc - - # Sv_vy (order of kron is not correct, so permute) - ybc = vLo_i ⊗ Sv_vy_bc.ybc1 + vUp_i ⊗ Sv_vy_bc.ybc2 - ybc = reshape(permutedims(reshape(ybc, Nvy_b, Nvx_in, Nvz_in), (2, 1, 3)), :) - ySv_vy = Sv_vy_bc.Bbc * ybc - - # Sv_vz - ybc = Sv_vz_bc.ybc1 ⊗ vBa_i + Sv_vz_bc.ybc2 ⊗ vFr_i - ySv_vz = Sv_vz_bc.Bbc * ybc - - # Su_vx (low/up) (order of kron is not correct, so permute) - ybc = uLo_i2 ⊗ Su_vx_bc_lu.ybc1 + uUp_i2 ⊗ Su_vx_bc_lu.ybc2 - ybc = reshape(permutedims(reshape(ybc, Nuy_b, Nvx_t - 1, Nvz_in), (2, 1, 3)), :) - ySu_vx_lu = Su_vx_bc_lu.Bbc * ybc - - # Su_vx (left/right) - ybc = uLe_i ⊗ Su_vx_bc_lr.ybc1 + uRi_i ⊗ Su_vx_bc_lr.ybc2 - ySu_vx_lr = Su_vx_bc_lu.B3D * Su_vx_bc_lr.Bbc * ybc - - # Su_vx - ySu_vx = ySu_vx_lr + ySu_vx_lu - - # Sw_vz (low/up) (order of kron is not correct, so permute) - ybc = wLo_i2 ⊗ Sw_vz_bc_lu.ybc1 + wUp_i2 ⊗ Sw_vz_bc_lu.ybc2 - ybc = reshape(permutedims(reshape(ybc, Nwy_b, Nwx_in, Nz + 1), (2, 1, 3)), :) - ySw_vz_lu = Sw_vz_bc_lu.Bbc * ybc - - # Sw_vz (back/front) - ybc = Sw_vz_bc_bf.ybc1 ⊗ wBa_i + Sw_vz_bc_bf.ybc2 ⊗ wFr_i - ySw_vz_bf = Sw_vz_bc_lu.B3D * Sw_vz_bc_bf.Bbc * ybc - - # Sw_vz - ySw_vz = ySw_vz_lu + ySw_vz_bf - - # Sw_wx - ybc = wLe_i ⊗ Sw_wx_bc.ybc1 + wRi_i ⊗ Sw_wx_bc.ybc2 - ySw_wx = Sw_wx_bc.Bbc * ybc - - # Sw_wy (order of kron is not correct, so permute) - ybc = wLo_i ⊗ Sw_wy_bc.ybc1 + wUp_i ⊗ Sw_wy_bc.ybc2 - ybc = reshape(permutedims(reshape(ybc, Nwy_b, Nwx_in, Nwz_in), (2, 1, 3)), :) - ySw_wy = Sw_wy_bc.Bbc * ybc - - # Sw_wz - ybc = Sw_wz_bc.ybc1 ⊗ wBa_i + Sw_wz_bc.ybc2 ⊗ wFr_i - ySw_wz = Sw_wz_bc.Bbc * ybc - - # Su_wx (back/front) - ybc = Su_wx_bc_bf.ybc1 ⊗ uBa_i2 + Su_wx_bc_bf.ybc2 ⊗ uFr_i2 - ySu_wx_bf = Su_wx_bc_bf.Bbc * ybc - - # Su_wx (left/right) - ybc = uLe_i ⊗ Su_wx_bc_lr.ybc1 + uRi_i ⊗ Su_wx_bc_lr.ybc2 - ySu_wx_lr = Su_wx_bc_bf.B3D * Su_wx_bc_lr.Bbc * ybc - - # Su_wx - ySu_wx = ySu_wx_bf + ySu_wx_lr - - # Sv_wy (back/front) - ybc = Sv_wy_bc_bf.ybc1 ⊗ vBa_i2 + Sv_wy_bc_bf.ybc2 ⊗ vFr_i2 - ySv_wy_bf = Sv_wy_bc_bf.Bbc * ybc - - # Sv_wy (low/up) - ybc = vLo_i ⊗ Sv_wy_bc_lu.ybc1 + vUp_i ⊗ Sv_wy_bc_lu.ybc2 - Nb = Nwy_in + 1 - Nvy_in - Nb != 0 && (ybc = reshape(permutedims(reshape(ybc, Nb, Nvx_in, Nvz_in), (2, 1, 3)), :)) - ySv_wy_lu = Sv_wy_bc_bf.B3D * Sv_wy_bc_lu.Bbc * ybc - - # Sv_wy - ySv_wy = ySv_wy_bf + ySv_wy_lu - - yDiffu = Dux * ySu_ux + Duy * ySu_uy + Duz * ySu_uz - yDiffv = Dvx * ySv_vx + Dvy * ySv_vy + Dvz * ySv_vz - yDiffw = Dwx * ySw_wx + Dwy * ySw_wy + Dwz * ySw_wz - yDiff = [yDiffu; yDiffv; yDiffw] - - Ωu⁻¹ = 1 ./ Ω[indu] - Ωv⁻¹ = 1 ./ Ω[indv] - Ωw⁻¹ = 1 ./ Ω[indw] - - # Diffusive terms in finite-difference setting, without viscosity - yDiffu_f = Ωu⁻¹ .* (Dux * ySu_ux + Duy * ySu_uy + Duz * ySu_uz) - yDiffv_f = Ωv⁻¹ .* (Dvx * ySv_vx + Dvy * ySv_vy + Dvz * ySv_vz) - yDiffw_f = Ωw⁻¹ .* (Dwx * ySw_wx + Dwy * ySw_wy + Dwz * ySw_wz) - - ## Boundary conditions for interpolation - - # Iu_ux - ybc = uLe_i ⊗ Iu_ux_bc.ybc1 + uRi_i ⊗ Iu_ux_bc.ybc2 - yIu_ux = Iu_ux_bc.Bbc * ybc - - # Iv_uy (left/right) - ybc = vLe_i2 ⊗ Iv_uy_bc_lr.ybc1 + vRi_i2 ⊗ Iv_uy_bc_lr.ybc2 - yIv_uy_lr = Iv_uy_bc_lr.Bbc * ybc - - # Iv_uy (low/up) (order of kron is not correct, so permute) - ybc = vLo_i ⊗ Iv_uy_bc_lu.ybc1 + vUp_i ⊗ Iv_uy_bc_lu.ybc2 - Nb = Nuy_in + 1 - Nvy_in - Nb != 0 && (ybc = reshape(permutedims(reshape(ybc, Nb, Nvx_in, Nvz_in), (2, 1, 3)), :)) - yIv_uy_lu = Iv_uy_bc_lr.B3D * Iv_uy_bc_lu.Bbc * ybc - - yIv_uy = yIv_uy_lr + yIv_uy_lu - - # Iw_uz (left/right) - ybc = wLe_i2 ⊗ Iw_uz_bc_lr.ybc1 + wRi_i2 ⊗ Iw_uz_bc_lr.ybc2 - yIw_uz_lr = Iw_uz_bc_lr.Bbc * ybc - - # Iw_uz (back/front) - ybc = Iw_uz_bc_bf.ybc1 ⊗ wBa_i + Iw_uz_bc_bf.ybc2 ⊗ wFr_i - yIw_uz_bf = Iw_uz_bc_lr.B3D * Iw_uz_bc_bf.Bbc * ybc - - # Iw_uz - yIw_uz = yIw_uz_lr + yIw_uz_bf - - # Iu_vx (low/up) (order of kron is not correct, so permute) - ybc = uLo_i2 ⊗ Iu_vx_bc_lu.ybc1 + uUp_i2 ⊗ Iu_vx_bc_lu.ybc2 - ybc = reshape(permutedims(reshape(ybc, Nuy_b, Nvx_t - 1, Nvz_in), (2, 1, 3)), :) - yIu_vx_lu = Iu_vx_bc_lu.Bbc * ybc - - # Iu_vx (left/right) - ybc = uLe_i ⊗ Iu_vx_bc_lr.ybc1 + uRi_i ⊗ Iu_vx_bc_lr.ybc2 - yIu_vx_lr = Iu_vx_bc_lu.B3D * Iu_vx_bc_lr.Bbc * ybc - - # Iu_vx - yIu_vx = yIu_vx_lr + yIu_vx_lu - - # Iv_vy (order of kron is not correct) - ybc = vLo_i ⊗ Iv_vy_bc.ybc1 + vUp_i ⊗ Iv_vy_bc.ybc2 - ybc = reshape(permutedims(reshape(ybc, Nvy_b, Nvx_in, Nvz_in), (2, 1, 3)), :) - yIv_vy = Iv_vy_bc.Bbc * ybc - - # Iw_vz (low/up) (order of kron is not correct, so permute) - ybc = wLo_i2 ⊗ Iw_vz_bc_lu.ybc1 + wUp_i2 ⊗ Iw_vz_bc_lu.ybc2 - ybc = reshape(permutedims(reshape(ybc, Nwy_b, Nwx_in, Nz + 1), (2, 1, 3)), :) - yIw_vz_lu = Iw_vz_bc_lu.Bbc * ybc - - # Iw_vz (back/front) - ybc = Iw_vz_bc_bf.ybc1 ⊗ wBa_i + Iw_vz_bc_bf.ybc2 ⊗ wFr_i - yIw_vz_bf = Iw_vz_bc_lu.B3D * Iw_vz_bc_bf.Bbc * ybc - - # Iw_vz - yIw_vz = yIw_vz_lu + yIw_vz_bf - - # Iu_wx (back/front) - ybc = Iu_wx_bc_bf.ybc1 ⊗ uBa_i2 + Iu_wx_bc_bf.ybc2 ⊗ uFr_i2 - yIu_wx_bf = Iu_wx_bc_bf.Bbc * ybc - - # Iu_wx (left/right) - ybc = uLe_i ⊗ Iu_wx_bc_lr.ybc1 + uRi_i ⊗ Iu_wx_bc_lr.ybc2 - yIu_wx_lr = Iu_wx_bc_bf.B3D * Iu_wx_bc_lr.Bbc * ybc - - yIu_wx = yIu_wx_bf + yIu_wx_lr - - # Iv_wy (back/front) - ybc = Iv_wy_bc_bf.ybc1 ⊗ vBa_i2 + Iv_wy_bc_bf.ybc2 ⊗ vFr_i2 - yIv_wy_bf = Iv_wy_bc_bf.Bbc * ybc - - # Iv_wy (low/up) - ybc = vLo_i ⊗ Iv_wy_bc_lu.ybc1 + vUp_i ⊗ Iv_wy_bc_lu.ybc2 - Nb = Nwy_in + 1 - Nvy_in - Nb != 0 && (ybc = reshape(permutedims(reshape(ybc, Nb, Nvx_in, Nvz_in), (2, 1, 3)), :)) - yIv_wy_lu = Iv_wy_bc_bf.B3D * Iv_wy_bc_lu.Bbc * ybc - - # Iv_wy - yIv_wy = yIv_wy_bf + yIv_wy_lu - - # Iw_wz - ybc = Iw_wz_bc.ybc1 ⊗ wBa_i + Iw_wz_bc.ybc2 ⊗ wFr_i - yIw_wz = Iw_wz_bc.Bbc * ybc - - # Group vectors - bc_vectors = (; - yM, - ydM, - y_p, - yAu_ux, - yAu_uy, - yAu_uz, - yAv_vx, - yAv_vy, - yAv_vz, - yAw_wx, - yAw_wy, - yAw_wz, - yDiff, - ySu_ux, - ySu_uy, - ySu_uz, - ySv_vx, - ySv_vy, - ySv_vz, - ySw_wx, - ySw_wy, - ySw_wz, - ySu_vx, - ySu_wx, - ySv_uy, - ySv_wy, - ySw_uz, - ySw_vz, - yDiffu_f, - yDiffv_f, - yDiffw_f, - yIu_ux, - yIv_uy, - yIw_uz, - yIu_vx, - yIv_vy, - yIw_vz, - yIu_wx, - yIv_wy, - yIw_wz, - ) - - if viscosity_model isa Union{QRModel,SmagorinskyModel,MixingLengthModel} - (; Aν_vy_bc) = operators - (; Cux_k_bc, Cuy_k_bc, Cuz_k_bc) = operators - (; Cvx_k_bc, Cvy_k_bc, Cvz_k_bc) = operators - (; Cwx_k_bc, Cwy_k_bc, Cwz_k_bc) = operators - (; Auy_k_bc, Avx_k_bc) = operators - (; Auz_k_bc, Awx_k_bc) = operators - (; Awy_k_bc, Avz_k_bc) = operators - - # Set bc for turbulent viscosity nu_t - # In the periodic case, the value of nu_t is not needed - # In all other cases, homogeneous (zero) Neumann conditions are used - nuLe = zeros(T, Npy) - nuRi = zeros(T, Npy) - nuLo = zeros(T, Npx) - nuUp = zeros(T, Npx) - - ## Nu_ux - (; Aν_ux_bc) = operators - ybc = nuLe ⊗ Aν_ux_bc.ybc1 + nuRi ⊗ Aν_ux_bc.ybc2 - yAν_ux = Aν_ux_bc.Bbc * ybc - - ## Nu_uy - (; Aν_uy_bc_lr, Aν_uy_bc_lu) = operators - - nuLe_i = [nuLe[1]; nuLe; nuLe[end]] - nuRi_i = [nuRi[1]; nuRi; nuRi[end]] - - # In x-direction - ybc = nuLe_i ⊗ Aν_uy_bc_lr.ybc1 + nuRi_i ⊗ Aν_uy_bc_lr.ybc2 - yAν_uy_lr = Aν_uy_bc_lr.B3D * ybc - - # In y-direction - ybc = Aν_uy_bc_lu.ybc1 ⊗ nuLo + Aν_uy_bc_lu.ybc2 ⊗ nuUp - yAν_uy_lu = Aν_uy_bc_lu.B3D * ybc - - yAν_uy = yAν_uy_lu + yAν_uy_lr - - ## Nu_vx - (; Aν_vx_bc_lr, Aν_vx_bc_lu) = operators - - nuLo_i = [nuLo[1]; nuLo; nuLo[end]] - nuUp_i = [nuUp[1]; nuUp; nuUp[end]] - - # In y-direction - ybc = Aν_vx_bc_lu.ybc1 ⊗ nuLo_i + Aν_vx_bc_lu.ybc2 ⊗ nuUp_i - yAν_vx_lu = Aν_vx_bc_lu.B3D * ybc - - # In x-direction - ybc = nuLe ⊗ Aν_vx_bc_lr.ybc1 + nuRi ⊗ Aν_vx_bc_lr.ybc2 - yAν_vx_lr = Aν_vx_bc_lr.B3D * ybc - - yAν_vx = yAν_vx_lu + yAν_vx_lr - - ## Nu_vy - ybc = Aν_vy_bc.ybc1 ⊗ nuLo + Aν_vy_bc.ybc2 ⊗ nuUp - yAν_vy = Aν_vy_bc.Bbc * ybc - - # Set bc for getting du/dx, du/dy, dv/dx, dv/dy at cell centers - - uLo_p = u_bc.(xp, y[1], t) - uUp_p = u_bc.(xp, y[end], t) - - vLe_p = v_bc.(x[1], yp, t) - vRi_p = v_bc.(x[end], yp, t) - - ybc = uLe_i ⊗ Cux_k_bc.ybc1 + uRi_i ⊗ Cux_k_bc.ybc2 - yCux_k = Cux_k_bc.Bbc * ybc - - ybc = uLe_i ⊗ Auy_k_bc.ybc1 + uRi_i ⊗ Auy_k_bc.ybc2 - yAuy_k = Auy_k_bc.Bbc * ybc - - ybc = Cuy_k_bc.ybc1 ⊗ uLo_p + Cuy_k_bc.ybc2 ⊗ uUp_p - yCuy_k = Cuy_k_bc.Bbc * ybc - - ybc = Avx_k_bc.ybc1 ⊗ vLo_i + Avx_k_bc.ybc2 ⊗ vUp_i - yAvx_k = Avx_k_bc.Bbc * ybc - - ybc = vLe_p ⊗ Cvx_k_bc.ybc1 + vRi_p ⊗ Cvx_k_bc.ybc2 - yCvx_k = Cvx_k_bc.Bbc * ybc - - ybc = Cvy_k_bc.ybc1 ⊗ vLo_i + Cvy_k_bc.ybc2 ⊗ vUp_i - yCvy_k = Cvy_k_bc.Bbc * ybc - - bc_vectors = (; - bc_vectors..., - yAν_ux, - yAν_uy, - yAν_vx, - yAν_vy, - yCux_k, - yCuy_k, - yCvx_k, - yCvy_k, - yAuy_k, - yAvx_k, - ) - end - - bc_vectors -end diff --git a/src/create_initial_conditions.jl b/src/create_initial_conditions.jl new file mode 100644 index 000000000..730fcf7b0 --- /dev/null +++ b/src/create_initial_conditions.jl @@ -0,0 +1,130 @@ +""" + create_initial_conditions( + setup, + initial_velocity, + t; + pressure_solver = DirectPressureSolver(setup), + ) + +Create initial vectors `(u, p)` at starting time `t`. +""" +function create_initial_conditions( + setup, + initial_velocity, + t; + pressure_solver = DirectPressureSolver(setup), +) + (; grid) = setup + (; dimension, N, Iu, Ip, x, xp, Ωu) = grid + + T = eltype(x[1]) + D = dimension() + + # Allocate velocity and pressure + u = ntuple(d -> KernelAbstractions.zeros(get_backend(x[1]), T, N...), D) + p = KernelAbstractions.zeros(get_backend(x[1]), T, N...) + + # Initial velocities + for α = 1:D + xin = ntuple(β -> reshape(α == β ? x[β][2:end] : xp[β], ntuple(Returns(1), β - 1)..., :), D) + u[α][Iu[α]] .= initial_velocity[α].(xin...)[Iu[α]] + end + + apply_bc_u!(u, t, setup) + + # Kinetic energy and momentum of initial velocity field + # Iteration 1 corresponds to t₀ = 0 (for unsteady simulations) + maxdiv = maximum(divergence(u, setup)) + + # TODO: Maybe eps(T)^(3//4) + if maxdiv > 1e-12 + @warn "Initial velocity field not (discretely) divergence free: $maxdiv.\n" * + "Performing additional projection." + + # Make velocity field divergence free + f = divergence(u, setup) + Δp = pressure_poisson(pressure_solver, f[Ip][:]) + p[Ip][:] .= Δp + apply_bc_p!(p, t, setup) + G = pressuregradient(p, setup) + for α = 1:D + u[α] .-= 1 ./ Ωu[α] .* G[α] + end + end + + p = pressure_additional_solve(pressure_solver, u, t, setup) + apply_bc_p!(p, t, setup) + + # Initial conditions, including initial boundary condititions + u, p +end + +function create_spectrum(N; A, σ, s, backend) + T = typeof(A) + D = length(N) + K = N .÷ 2 + k = ntuple(α -> reshape(1:K[α], ntuple(Returns(1), α - 1)..., :, ntuple(Returns(1), D-α)...), D) + a = KernelAbstractions.ones(backend, Complex{T}, K) + AT = typeof(a) + # k = AT.(Array{Complex{T}}.(k)) + # k = AT.(k) + τ = T(2π) + a .*= A / sqrt(τ^2 * 2σ^2) + for α = 1:D + kα = k[α] + @. a *= exp(-(kα - s)^2 / 2σ^2 - im * τ * rand(T)) + end + for α = 1:D + a = cat(a, reverse(a; dims = α); dims = α) + end + a +end + +""" + random_field( + setup, t; + A = 1_000_000, + σ = 30, + s = 5, + pressure_solver = DirectPressureSolver(setup), + ) + +Create random field. + +- `K`: Maximum wavenumber +- `A`: Eddy amplitude +- `σ`: Variance +- `s` Wavenumber offset before energy starts decaying +""" +function random_field( + setup, t; + A = convert(eltype(setup.grid.x), 1_000_000), + σ = convert(eltype(setup.grid.x), 30), + s = convert(eltype(setup.grid.x), 5), + pressure_solver = DirectPressureSolver(setup), +) + (; dimension, x, N, Ip, Ωu) = setup.grid + + D = dimension() + T = eltype(x[1]) + backend = get_backend(x[1]) + + u = ntuple(α -> real.(ifft(create_spectrum(N; A, σ, s, backend))), D) + M = divergence(u, setup) + p = zero(M) + + # Make velocity field divergence free + Min = view(M, Ip) + pin = view(p, Ip) + pressure_poisson!(pressure_solver, pin, Min) + apply_bc_p!(p, t, setup) + G = pressuregradient(p, setup) + for α = 1:D + @. u[α] -= 1 / Ωu[α] * G[α] + end + apply_bc_u!(u, t, setup) + p = pressure_additional_solve(pressure_solver, u, t, setup) + apply_bc_p!(p, t, setup) + + u, p +end diff --git a/src/force/force.jl b/src/force/force.jl deleted file mode 100644 index 5a4a7c111..000000000 --- a/src/force/force.jl +++ /dev/null @@ -1,28 +0,0 @@ -""" - SteadyBodyForce(fu, fv, grid) - -Two-dimensional steady body force `f(x, y) = [fu(x, y), fv(x, y)]`. -""" -function SteadyBodyForce(fu, fv, grid) - (; NV, indu, indv, xu, yu, xv, yv) = grid - T = eltype(xu) - F = zeros(T, NV) - F[indu] .= reshape(fu.(xu, yu), :) - F[indv] .= reshape(fv.(xv, yv), :) - F -end - -""" - SteadyBodyForce(fu, fv, fw, grid) - -Three-dimensional steady body force `f(x, y, z) = [fu(x, y, z), fv(x, y, z), fw(x, y, z)]`. -""" -function SteadyBodyForce(fu, fv, fw, grid) - (; NV, indu, indv, indw, xu, yu, zu, xv, yv, zv, xw, yw, zw) = grid - T = eltype(xu) - F = zeros(T, NV) - F[indu] .= reshape(fu.(xu, yu, zu), :) - F[indv] .= reshape(fv.(xv, yv, zv), :) - F[indw] .= reshape(fw.(xw, yw, zw), :) - F -end diff --git a/src/grid/grid.jl b/src/grid/grid.jl index aadcb43db..5a7a1b624 100644 --- a/src/grid/grid.jl +++ b/src/grid/grid.jl @@ -1,878 +1,95 @@ """ - Grid(x, y; boundary_conditions, order4 = false) + Grid(x, boundary_conditions) -Create nonuniform Cartesian box mesh `x` × `y` with boundary conditions `boundary_conditions`. -If `order4` is `true`, a fourth order mesh is created. +Create nonuniform Cartesian box mesh `x[1]` × ... × `x[d]` with boundary +conditions `boundary_conditions`. """ -function Grid(x, y; boundary_conditions, order4 = false) - dimension = Dimension(2) - - T = eltype(x) - - α = 81 - β = T(9 // 8) - - Nx = length(x) - 1 - Ny = length(y) - 1 - xlims = (x[1], x[end]) - ylims = (y[1], y[end]) - - # Pressure positions - xp = (x[1:(end-1)] + x[2:end]) / 2 - yp = (y[1:(end-1)] + y[2:end]) / 2 - - # Distance between velocity points - hx = diff(x) - hy = diff(y) - - # Distance between pressure points - gx = zeros(T, Nx + 1) - gx[1] = hx[1] / 2 - gx[2:Nx] = (hx[1:(Nx-1)] + hx[2:Nx]) / 2 - gx[Nx+1] = hx[end] / 2 - - gy = zeros(T, Ny + 1) - gy[1] = hy[1] / 2 - gy[2:Ny] = (hy[1:(Ny-1)] + hy[2:Ny]) / 2 - gy[Ny+1] = hy[end] / 2 - - # Number of pressure points - Npx = Nx - Npy = Ny - Np = Npx * Npy - - ## u-volumes - # x[1] x[2] x[3] .... x[Nx] x[Nx+1] - # | | | | | - # | | | | | - # Dirichlet BC: - # ULe u[1] u[2] .... u(Nx-1) uRi - # Periodic BC: - # u[1] u[2] u[3] .... u[Nx] u[1] - # Pressure BC: - # u[1] u[2] u[3] .... u[Nx] u[Nx+1] - - # x-dir - Nux_b = 2 # Boundary points - Nux_in = Nx + 1 # Inner points - Nux_in -= boundary_conditions.u.x[1] ∈ [:dirichlet, :symmetric] - Nux_in -= boundary_conditions.u.x[2] ∈ [:dirichlet, :symmetric] - Nux_in -= boundary_conditions.u.x == (:periodic, :periodic) - Nux_t = Nux_in + Nux_b # Total number - - # Y-dir - Nuy_b = 2 # Boundary points - Nuy_in = Ny # Inner points - Nuy_t = Nuy_in + Nuy_b # Total number - - # Total number - Nu = Nux_in * Nuy_in - - ## v-volumes - - # X-dir - Nvx_b = 2 # Boundary points - Nvx_in = Nx # Inner points - Nvx_t = Nvx_in + Nvx_b # Total number - - # Y-dir - Nvy_b = 2 # Boundary points - Nvy_in = Ny + 1 # Inner points - Nvy_in -= boundary_conditions.v.y[1] ∈ [:dirichlet, :symmetric] - Nvy_in -= boundary_conditions.v.y[2] ∈ [:dirichlet, :symmetric] - Nvy_in -= boundary_conditions.v.y == (:periodic, :periodic) - Nvy_t = Nvy_in + Nvy_b # Total number - - # Total number - Nv = Nvx_in * Nvy_in - - # Total number of velocity points - NV = Nu + Nv - - ## For a grid with three times larger volumes: - if order4 - hx3 = zeros(T, Nx) - hx3[2:(end-1)] = hx[1:(end-2)] + hx[2:(end-1)] + hx[3:end] - if boundary_conditions.u.x[1] == :periodic && - boundary_conditions.u.x[2] == :periodic - hx3[1] = hx[end] + hx[1] + hx[2] - hx3[end] = hx[end-1] + hx[end] + hx[1] - else - hx3[1] = 2 * hx[1] + hx[2] - hx3[end] = hx[end-1] + 2 * hx[end] - end - - hy3 = zeros(T, Ny) - hy3[2:(end-1)] = hy[1:(end-2)] + hy[2:(end-1)] + hy[3:end] - if boundary_conditions.v.y[1] == :periodic && - boundary_conditions.v.y[2] == :periodic - hy3[1] = hy[end] + hy[1] + hy[2] - hy3[end] = hy[end-1] + hy[end] + hy[1] - else - hy3[1] = 2 * hy[1] + hy[2] - hy3[end] = hy[end-1] + 2 * hy[end] - end - - hxi3 = copy(hx3) - hyi3 = copy(hy3) - - # Distance between pressure points - gx3 = zeros(T, Nx + 1) - gx3[3:(Nx-1)] = gx[2:(end-3)] + gx[3:(end-2)] + gx[4:(end-1)] - if boundary_conditions.u.x[1] == :periodic && - boundary_conditions.u.x[2] == :periodic - gx3[1] = gx[end-1] + gx[end] + gx[1] + gx[2] - gx3[2] = gx[end] + gx[1] + gx[2] + gx[3] - gx3[end-1] = gx[end-2] + gx[end-1] + gx[end] + gx[1] - gx3[end] = gx[end-1] + gx[end] + gx[1] + gx[2] - else - gx3[1] = 2 * gx[1] + 2 * gx[2] - gx3[2] = 2 * gx[1] + gx[2] + gx[3] - gx3[end-1] = 2 * gx[end] + gx[end-1] + gx[end-2] - gx3[end] = 2 * gx[end] + 2 * gx[end-1] - end - - # Distance between pressure points - gy3 = zeros(T, Ny + 1) - gy3[3:(Ny-1)] = gy[2:(end-3)] + gy[3:(end-2)] + gy[4:(end-1)] - if boundary_conditions.v.y[1] == :periodic && - boundary_conditions.v.y[2] == :periodic - gy3[1] = gy[end-1] + gy[end] + gy[1] + gy[2] - gy3[2] = gy[end] + gy[1] + gy[2] + gy[3] - gy3[end-1] = gy[end-2] + gy[end-1] + gy[end] + gy[1] - gy3[end] = gy[end-1] + gy[end] + gy[1] + gy[2] - else - gy3[1] = 2 * gy[1] + 2 * gy[2] - gy3[2] = 2 * gy[1] + gy[2] + gy[3] - gy3[end-1] = 2 * gy[end] + gy[end-1] + gy[end-2] - gy3[end] = 2 * gy[end] + 2 * gy[end-1] - end +function Grid(x, boundary_conditions) + # Kill all LinRanges etc. + x = Array.(x) + xlims = extrema.(x) + + D = length(x) + dimension = Dimension(D) + + T = eltype(x[1]) + + # Add offset positions for ghost volumes + # For all BC, there is one ghost volume on each side, + # but not all of the ``d + 1`` fields have a component inside this ghost + # volume. + for d = 1:D + a, b = boundary_conditions[d] + ghost_a!(a, x[d]) + ghost_b!(b, x[d]) end - ## Adapt mesh metrics depending on number of volumes - - ## X-direction - - # gxd: differentiation - gxd = copy(gx) - gxd[1] = hx[1] - gxd[end] = hx[end] - - # hxi: integration and hxd: differentiation - # Map to find suitable size - hxi = copy(hx) + # Number of finite volumes in each dimension, including ghost volumes + N = length.(x) .- 1 - # Restrict Nx+2 to Nux_in+1 points - if boundary_conditions.u.x == (:dirichlet, :dirichlet) - xin = x[2:(end-1)] - hxd = copy(hx) - gxi = gx[2:(end-1)] - diagpos = 1 - - if order4 - hxd3 = [hx3[1]; hx3; hx3[end]] - hxd13 = [hx[1]; hx; hx[end]] - gxd3 = [2 * gx[1] + gx[2] + gx[3]; gx3; 2 * gx[end] + gx[end-1] + gx[end-2]] - gxd13 = [gx[2]; 2 * gx[1]; gx[2:(end-1)]; 2 * gx[end]; gx[end-1]] - gxi3 = gx3[2:(end-1)] - end - elseif boundary_conditions.u.x == (:dirichlet, :pressure) - xin = x[2:end] - hxd = [hx; hx[end]] - gxi = gx[2:end] - diagpos = 1 - elseif boundary_conditions.u.x == (:pressure, :dirichlet) - xin = x[1:(end-1)] - hxd = [hx[1]; hx] - gxi = gx[1:(end-1)] - diagpos = 0 - elseif boundary_conditions.u.x == (:pressure, :pressure) - xin = x[1:end] - hxd = [hx[1]; hx; hx[end]] - gxi = copy(gx) - diagpos = 0 - elseif boundary_conditions.u.x == (:periodic, :periodic) - xin = x[1:(end-1)] - hxd = [hx[end]; hx] - gxi = [gx[1] + gx[end]; gx[2:(end-1)]] - gxd[1] = (hx[1] + hx[end]) / 2 - gxd[end] = (hx[1] + hx[end]) / 2 - diagpos = 0 - - if order4 - hxd3 = [hx3[end-1]; hx3[end]; hx3; hx3[1]] - hxd13 = [hx[end-1]; hx[end]; hx; hx[1]] - gxd3 = [gx3[end-1]; gx3; gx3[2]] - gxd13 = [gx[end-1]; gx[1] + gx[end]; gx[2:(end-1)]; gx[end] + gx[1]; gx[2]] - gxi3 = gx3[1:(end-1)] + # Number of velocity DOFs in each dimension + Nu = ntuple(D) do α + ntuple(D) do β + na = offset_u(boundary_conditions[β][1], α == β, false) + nb = offset_u(boundary_conditions[β][2], α == β, true) + N[β] - na - nb end end - Bmap = spdiagm(Nux_in + 1, Nx + 2, diagpos => ones(T, Nux_in + 1)) - - # Matrix to map from Nvx_t-1 to Nux_in points - # (used in interpolation, convection_diffusion, viscosity) - Bvux = spdiagm(Nux_in, Nvx_t - 1, diagpos => ones(T, Nux_in)) - - # Map from Npx+2 points to Nux_t-1 points (ux faces) - Bkux = copy(Bmap) - - ## Y-direction - - # gyi: integration and gyd: differentiation - gyd = copy(gy) - gyd[1] = hy[1] - gyd[end] = hy[end] - - # hyi: integration and hyd: differentiation - # Map to find suitable size - hyi = copy(hy) - - # Restrict Ny+2 to Nvy_in+1 points - if boundary_conditions.v.y == (:dirichlet, :dirichlet) - yin = y[2:(end-1)] - hyd = copy(hy) - gyi = gy[2:(end-1)] - diagpos = 1 - - if order4 - hyd3 = [hy3[1]; hy3; hy3[end]] - hyd13 = [hy[1]; hy; hy[end]] - gyd3 = [2 * gy[1] + gy[2] + gy[3]; gy3; 2 * gy[end] + gy[end-1] + gy[end-2]] - gyd13 = [gy[2]; 2 * gy[1]; gy[2:(end-1)]; 2 * gy[end]; gy[end-1]] - gyi3 = gy3[2:(end-1)] - end - elseif boundary_conditions.v.y == (:dirichlet, :pressure) - yin = y[2:end] - hyd = [hy; hy[end]] - gyi = gy[2:end] - diagpos = 1 - elseif boundary_conditions.v.y == (:pressure, :dirichlet) - yin = y[1:(end-1)] - hyd = [hy[1]; hy] - gyi = gy[1:(end-1)] - diagpos = 0 - elseif boundary_conditions.v.y == (:pressure, :pressure) - yin = y[1:end] - hyd = [hy[1]; hy; hy[end]] - gyi = copy(gy) - diagpos = 0 - elseif boundary_conditions.v.y == (:periodic, :periodic) - yin = y[1:(end-1)] - hyd = [hy[end]; hy] - gyi = [gy[1] + gy[end]; gy[2:(end-1)]] - gyd[1] = (hy[1] + hy[end]) / 2 - gyd[end] = (hy[1] + hy[end]) / 2 - diagpos = 0 - - if order4 - hyd3 = [hy3[end-1]; hy3[end]; hy3; hy3[1]] - hyd13 = [hy[end-1]; hy[end]; hy; hy[1]] - gyd3 = [gy3[end-1]; gy3; gy3[2]] - gyd13 = [gy[end-1]; gy[1] + gy[end]; gy[2:(end-1)]; gy[end] + gy[1]; gy[2]] - gyi3 = gy3[1:(end-1)] + # Cartesian index ranges of velocity DOFs + Iu = ntuple(D) do α + Iuα = ntuple(D) do β + na = offset_u(boundary_conditions[β][1], α == β, false) + nb = offset_u(boundary_conditions[β][2], α == β, true) + 1+na:N[β]-nb end + CartesianIndices(Iuα) end - Bmap = spdiagm(Nvy_in + 1, Ny + 2, diagpos => ones(T, Nvy_in + 1)) - - # Matrix to map from Nuy_t-1 to Nvy_in points - # (used in interpolation, convection_diffusion) - Buvy = spdiagm(Nvy_in, Nuy_t - 1, diagpos => ones(T, Nvy_in)) - - # Map from Npy+2 points to Nvy_t-1 points (vy faces) - Bkvy = copy(Bmap) - - ## - # Volume (area) of pressure control volumes - Ωp = hyi ⊗ hxi - - # Volume (area) of u control volumes - Ωu = hyi ⊗ gxi - - # Volume of ux volumes - Ωux = hyi ⊗ hxd - - # Volume of uy volumes - Ωuy = gyd ⊗ gxi - - # Volume (area) of v control volumes - Ωv = gyi ⊗ hxi - - # Volume of vx volumes - Ωvx = gyi ⊗ gxd - - # Volume of vy volumes - Ωvy = hyd ⊗ hxi - - Ω = [Ωu; Ωv] - - if order4 - # Differencing for second order operators on the fourth order mesh - Ωux1 = hyi ⊗ hxd13 - Ωuy1 = gyd13 ⊗ gxi - Ωvx1 = gyi ⊗ gxd13 - Ωvy1 = hyd13 ⊗ hxi - - # Volume (area) of pressure control volumes - Ωp3 = hyi3 ⊗ hxi3 - - # Volume (area) of u-vel control volumes - Ωu3 = hyi3 ⊗ gxi3 - - # Volume (area) of v-vel control volumes - Ωv3 = gyi3 ⊗ hxi3 - - # Volume (area) of dudx control volumes - Ωux3 = hyi3 ⊗ hxd3 - - # Volume (area) of dudy control volumes - Ωuy3 = gyd3 ⊗ gxi3 - - # Volume (area) of dvdx control volumes - Ωvx3 = gyi3 ⊗ gxd3 - - # Volume (area) of dvdy control volumes - Ωvy3 = hyd3 ⊗ hxi3 - - Ωu1 = copy(Ωu) - Ωv1 = copy(Ωv) - - Ωu = α * Ωu1 - Ωu3 - Ωv = α * Ωv1 - Ωv3 - Ω = [Ωu; Ωv] - - Ωux = @. α * Ωux1 - Ωux3 - Ωuy = @. α * Ωuy1 - Ωuy3 - Ωvx = @. α * Ωvx1 - Ωvx3 - Ωvy = @. α * Ωvy1 - Ωvy3 + # Number of p DOFs in each dimension + Np = ntuple(D) do α + na = offset_p(boundary_conditions[α][1]) + nb = offset_p(boundary_conditions[α][2]) + N[α] - na - nb end - # Metrics that can be useful for initialization: - xu = ones(T, 1, Nuy_in) ⊗ xin - yu = yp ⊗ ones(T, Nux_in) - xu = reshape(xu, Nux_in, Nuy_in) - yu = reshape(yu, Nux_in, Nuy_in) + # Cartesian index range of pressure DOFs + Ip = CartesianIndices(ntuple(D) do α + na = offset_p(boundary_conditions[α][1]) + nb = offset_p(boundary_conditions[α][2]) + 1+na:N[α]-nb + end) - xv = ones(T, 1, Nvy_in) ⊗ xp - yv = yin ⊗ ones(T, Nvx_in) - xv = reshape(xv, Nvx_in, Nvy_in) - yv = reshape(yv, Nvx_in, Nvy_in) + xp = ntuple(d -> (x[d][1:end-1] .+ x[d][2:end]) ./ 2, D) - xpp = ones(T, Ny) ⊗ xp - ypp = yp ⊗ ones(T, Nx) - xpp = reshape(xpp, Nx, Ny) - ypp = reshape(ypp, Nx, Ny) + # Volume widths + Δ = ntuple(d -> diff(x[d]), D) + Δu = ntuple(d -> push!(diff(xp[d]), Δ[d][end] / 2), D) - # Indices of unknowns in velocity vector - indu = 1:Nu - indv = Nu .+ (1:Nv) - indV = 1:NV - indp = NV .+ (1:Np) - - ## Store quantities in the structure - params = (; - Npx, - Npy, - Np, - Nux_in, - Nux_b, - Nux_t, - Nuy_in, - Nuy_b, - Nuy_t, - Nvx_in, - Nvx_b, - Nvx_t, - Nvy_in, - Nvy_b, - Nvy_t, - Nu, - Nv, - NV, - Ωp, - Ω, - Ωux, - Ωvx, - Ωuy, - Ωvy, - hxi, - hyi, - hxd, - hyd, - gxi, - gyi, - gxd, - gyd, - Buvy, - Bvux, - Bkux, - Bkvy, - xin, - yin, - xu, - yu, - xv, - yv, - xpp, - ypp, - indu, - indv, - indV, - indp, - ) - - if order4 - params = (; - params..., - hx3, - hy3, - hxi3, - hyi3, - gxi3, - gyi3, - hxd13, - hxd3, - hyd13, - hyd3, - gxd13, - gxd3, - gyd13, - gyd3, - Ωux1, - Ωux3, - Ωuy1, - Ωuy3, - Ωvx1, - Ωvx3, - Ωvy1, - Ωvy3, - ) + # Reference volume sizes + Ω = KernelAbstractions.ones(get_backend(x[1]), T, N...) + for d = 1:D + Ω .*= reshape(Δ[d], ntuple(Returns(1), d - 1)..., :) end - (; - dimension, - α, - β, - order4, - Nx, - Ny, - xlims, - ylims, - x, - y, - xp, - yp, - hx, - hy, - gx, - gy, - params..., - ) -end - -""" - Grid(x, y, z; boundary_conditions, order4 = false) - -Create nonuniform Cartesian box mesh `x` × `y` × `z` with boundary conditions `boundary_conditions`. -If `order4` is `true`, a fourth order mesh is created. -""" -function Grid(x, y, z; boundary_conditions, order4 = false) - dimension = Dimension(3) - - T = eltype(x) - - order4 && error("Fourth order grids not yet implemented for 3D") - α = 81 - β = T(9 // 8) - - Nx = length(x) - 1 - Ny = length(y) - 1 - Nz = length(z) - 1 - xlims = (x[1], x[end]) - ylims = (y[1], y[end]) - zlims = (z[1], z[end]) - - # Pressure positions - xp = (x[1:(end-1)] + x[2:end]) / 2 - yp = (y[1:(end-1)] + y[2:end]) / 2 - zp = (z[1:(end-1)] + z[2:end]) / 2 - - # Distance between velocity points - hx = diff(x) - hy = diff(y) - hz = diff(z) - - # Distance between pressure points - gx = zeros(T, Nx + 1) - gx[1] = hx[1] / 2 - gx[2:Nx] = (hx[1:(Nx-1)] + hx[2:Nx]) / 2 - gx[Nx+1] = hx[end] / 2 - - gy = zeros(T, Ny + 1) - gy[1] = hy[1] / 2 - gy[2:Ny] = (hy[1:(Ny-1)] + hy[2:Ny]) / 2 - gy[Ny+1] = hy[end] / 2 - - gz = zeros(T, Nz + 1) - gz[1] = hz[1] / 2 - gz[2:Nz] = (hz[1:(Nz-1)] + hz[2:Nz]) / 2 - gz[Nz+1] = hz[end] / 2 - - # Number of pressure points - Npx = Nx - Npy = Ny - Npz = Nz - Np = Npx * Npy * Npz - - ## u-volumes - # x[1] x[2] x[3] .... x[Nx] x[Nx+1] - # | | | | | - # | | | | | - # Dirichlet BC: - # ULe u[1] u[2] .... u(Nx-1) uRi - # Periodic BC: - # u[1] u[2] u[3] .... u[Nx] u[1] - # Pressure BC: - # u[1] u[2] u[3] .... u[Nx] u[Nx+1] - - # x-dir - Nux_b = 2 # Boundary points - Nux_in = Nx + 1 # Inner points - Nux_in -= boundary_conditions.u.x[1] ∈ [:dirichlet, :symmetric] - Nux_in -= boundary_conditions.u.x[2] ∈ [:dirichlet, :symmetric] - Nux_in -= boundary_conditions.u.x == (:periodic, :periodic) - Nux_t = Nux_in + Nux_b # Total number - - # y-dir - Nuy_b = 2 # Boundary points - Nuy_in = Ny # Inner points - Nuy_t = Nuy_in + Nuy_b # Total number - - # z-dir - Nuz_b = 2 # Boundary points - Nuz_in = Nz # Inner points - Nuz_t = Nuz_in + Nuz_b # Total number - - # Total number - Nu = Nux_in * Nuy_in * Nuz_in - - ## v-volumes - - # x-dir - Nvx_b = 2 # Boundary points - Nvx_in = Nx # Inner points - Nvx_t = Nvx_in + Nvx_b # Total number - - # y-dir - Nvy_b = 2 # Boundary points - Nvy_in = Ny + 1 # Inner points - Nvy_in -= boundary_conditions.v.y[1] ∈ [:dirichlet, :symmetric] - Nvy_in -= boundary_conditions.v.y[2] ∈ [:dirichlet, :symmetric] - Nvy_in -= boundary_conditions.v.y == (:periodic, :periodic) - Nvy_t = Nvy_in + Nvy_b # Total number - - # z-dir - Nvz_b = 2 # Boundary points - Nvz_in = Nz # Inner points - Nvz_t = Nvz_in + Nvz_b # Total number - - # Total number - Nv = Nvx_in * Nvy_in * Nvz_in - - ## w-volumes - - # x-dir - Nwx_b = 2 # Boundary points - Nwx_in = Nx # Inner points - Nwx_t = Nwx_in + Nwx_b # Total number - - # y-dir - Nwy_b = 2 # Boundary points - Nwy_in = Ny # Inner points - Nwy_t = Nwy_in + Nwy_b # Total number - - # z-dir - Nwz_b = 2 # Boundary points - Nwz_in = Nz + 1 # Inner points - Nwz_in -= boundary_conditions.w.z[1] ∈ [:dirichlet, :symmetric] - Nwz_in -= boundary_conditions.w.z[2] ∈ [:dirichlet, :symmetric] - Nwz_in -= boundary_conditions.w.z == (:periodic, :periodic) - Nwz_t = Nwz_in + Nwz_b # Total number - - # Total number - Nw = Nwx_in * Nwy_in * Nwz_in - - # Total number of velocity points - NV = Nu + Nv + Nw - - ## Adapt mesh metrics depending on number of volumes - - ## X-direction - - # gxd: differentiation - gxd = copy(gx) - gxd[1] = hx[1] - gxd[end] = hx[end] - - # hxi: integration and hxd: differentiation - # Map to find suitable size - hxi = copy(hx) - hxd = [hx[1]; hx; hx[end]] - - # Restrict Nx+2 to Nux_in+1 points - boundary_conditions.u.x == (:dirichlet, :dirichlet) && (diagpos = 1) - boundary_conditions.u.x == (:dirichlet, :pressure) && (diagpos = 1) - boundary_conditions.u.x == (:pressure, :dirichlet) && (diagpos = 0) - boundary_conditions.u.x == (:pressure, :pressure) && (diagpos = 0) - boundary_conditions.u.x == (:periodic, :periodic) && (diagpos = 0) - - Bmap = spdiagm(Nux_in + 1, Nx + 2, diagpos => ones(T, Nux_in + 1)) - Bmap_x_xin = spdiagm(Nux_in, Nx + 1, diagpos => ones(T, Nux_in)) - hxd = Bmap * hxd - gxi = Bmap_x_xin * gx - xin = Bmap_x_xin * x - - if boundary_conditions.u.x == (:periodic, :periodic) - gxd[1] = (hx[1] + hx[end]) / 2 - gxd[end] = gxd[1] - gxi[1] = gxd[1] - hxd[1] = hx[end] + # Velocity volume sizes + Ωu = ntuple(α -> KernelAbstractions.ones(get_backend(x[1]), T, N), D) + for α = 1:D, β = 1:D + Ωu[α] .*= reshape((α == β ? Δu : Δ)[β], ntuple(Returns(1), β - 1)..., :) end - # Matrix to map from Nvx_t-1 to Nux_in points - Bvux = spdiagm(Nux_in, Nvx_t - 1, diagpos => ones(T, Nux_in)) - - # Matrix to map from Nwx_t-1 to Nux_in points - Bwux = spdiagm(Nux_in, Nwx_t - 1, diagpos => ones(T, Nux_in)) - - # Map from Npx+2 points to Nux_t-1 points (ux faces) - Bkux = copy(Bmap) - - ## y-direction - - # gyi: integration and gyd: differentiation - gyd = copy(gy) - gyd[1] = hy[1] - gyd[end] = hy[end] - - # hyi: integration and hyd: differentiation - # Map to find suitable size - hyi = copy(hy) - hyd = [hy[1]; hy; hy[end]] - - # Restrict Ny+2 to Nvy_in+1 points - boundary_conditions.v.y == (:dirichlet, :dirichlet) && (diagpos = 1) - boundary_conditions.v.y == (:dirichlet, :pressure) && (diagpos = 1) - boundary_conditions.v.y == (:pressure, :dirichlet) && (diagpos = 0) - boundary_conditions.v.y == (:pressure, :pressure) && (diagpos = 0) - boundary_conditions.v.y == (:periodic, :periodic) && (diagpos = 0) - - Bmap = spdiagm(Nvy_in + 1, Ny + 2, diagpos => ones(T, Nvy_in + 1)) - Bmap_y_yin = spdiagm(Nvy_in, Ny + 1, diagpos => ones(T, Nvy_in)) - - hyd = Bmap * hyd - gyi = Bmap_y_yin * gy - yin = Bmap_y_yin * y - - if boundary_conditions.v.y == (:periodic, :periodic) - gyd[1] = (hy[1] + hy[end]) / 2 - gyd[end] = gyd[1] - gyi[1] = gyd[1] - hyd[1] = hy[end] + # Vorticity volume sizes + Ωω = KernelAbstractions.ones(get_backend(x[1]), T, N) + for α = 1:D + Ωω .*= reshape(Δu[α], ntuple(Returns(1), α - 1)..., :) end - # Matrix to map from Nuy_t-1 to Nvy_in points - Buvy = spdiagm(Nvy_in, Nuy_t - 1, diagpos => ones(T, Nvy_in)) - - # matrix to map from Nwy_t-1 to Nvy_in points - Bwvy = spdiagm(Nvy_in, Nwy_t - 1, diagpos => ones(T, Nvy_in)) - - # Map from Npy+2 points to Nvy_t-1 points (vy faces) - Bkvy = copy(Bmap) - - ## z-direction - - # gzi: integration and gzd: differentiation - gzd = copy(gz) - gzd[1] = hz[1] - gzd[end] = hz[end] - - # hzi: integration and hzd: differentiation - # Map to find suitable size - hzi = copy(hz) - hzd = [hz[1]; hz; hz[end]] - - # Restrict Nz+2 to Nvz_in+1 points - boundary_conditions.w.z == (:dirichlet, :dirichlet) && (diagpos = 1) - boundary_conditions.w.z == (:dirichlet, :pressure) && (diagpos = 1) - boundary_conditions.w.z == (:pressure, :dirichlet) && (diagpos = 0) - boundary_conditions.w.z == (:pressure, :pressure) && (diagpos = 0) - boundary_conditions.w.z == (:periodic, :periodic) && (diagpos = 0) - - shape = (Nwz_in + 1, Nz + 2) - Bmap = spdiagm(shape..., diagpos => ones(T, Nwz_in + 1)) - Bmap_z_zin = spdiagm(Nwz_in, Nz + 1, diagpos => ones(T, Nwz_in)) - hzd = Bmap * hzd - gzi = Bmap_z_zin * gz - zin = Bmap_z_zin * z - - if boundary_conditions.w.z == (:periodic, :periodic) - gzd[1] = (hz[1] + hz[end]) / 2 - gzd[end] = gzd[1] - gzi[1] = gzd[1] - hzd[1] = hz[end] + # Velocity volume mid-sections + Γu = ntuple(α -> ntuple(β -> KernelAbstractions.ones(get_backend(x[1]), T, N), D), D) + for α = 1:D, β = 1:D, γ = ((1:β-1)..., (β+1:D)...) + Γu[α][β] .*= reshape(γ == β ? 1 : γ == α ? Δu[γ] : Δ[γ], ntuple(Returns(1), γ - 1)..., :) end - # Matrix to map from Nuz_t-1 to Nwz_in points - Buwz = spdiagm(Nwz_in, Nuz_t - 1, diagpos => ones(T, Nwz_in)) - - # Matrix to map from Nvz_t-1 to Nwz_in points - Bvwz = spdiagm(Nwz_in, Nvz_t - 1, diagpos => ones(T, Nwz_in)) - - # Map from Npy+2 points to Nvy_t-1 points (vy faces) - Bkwz = copy(Bmap) - - ## Volumes - # Volume (area) of pressure control volumes - Ωp = hzi ⊗ hyi ⊗ hxi - - # Volume (area) of u control volumes - Ωu = hzi ⊗ hyi ⊗ gxi - - # Volume (area) of v control volumes - Ωv = hzi ⊗ gyi ⊗ hxi - - # Volume (area) of w control volumes - Ωw = gzi ⊗ hyi ⊗ hxi - - # Total volumes - Ω = [Ωu; Ωv; Ωw] - - # Metrics that can be useful for initialization: - xu = reshape(ones(T, Nuz_in) ⊗ ones(T, Nuy_in) ⊗ xin, Nux_in, Nuy_in, Nuz_in) - yu = reshape(ones(T, Nuz_in) ⊗ yp ⊗ ones(T, Nux_in), Nux_in, Nuy_in, Nuz_in) - zu = reshape(zp ⊗ ones(T, Nuy_in) ⊗ ones(T, Nux_in), Nux_in, Nuy_in, Nuz_in) - - xv = reshape(ones(T, Nvz_in) ⊗ ones(T, Nvy_in) ⊗ xp, Nvx_in, Nvy_in, Nvz_in) - yv = reshape(ones(T, Nvz_in) ⊗ yin ⊗ ones(T, Nvx_in), Nvx_in, Nvy_in, Nvz_in) - zv = reshape(zp ⊗ ones(T, Nvy_in) ⊗ ones(T, Nvx_in), Nvx_in, Nvy_in, Nvz_in) - - xw = reshape(ones(T, Nwz_in) ⊗ ones(T, Nwy_in) ⊗ xp, Nwx_in, Nwy_in, Nwz_in) - yw = reshape(ones(T, Nwz_in) ⊗ yp ⊗ ones(T, Nwx_in), Nwx_in, Nwy_in, Nwz_in) - zw = reshape(zin ⊗ ones(T, Nwy_in) ⊗ ones(T, Nwx_in), Nwx_in, Nwy_in, Nwz_in) - - xpp = reshape(ones(T, Nz) ⊗ ones(T, Ny) ⊗ xp, Nx, Ny, Nz) - ypp = reshape(ones(T, Nz) ⊗ yp ⊗ ones(T, Nx), Nx, Ny, Nz) - zpp = reshape(zp ⊗ ones(T, Ny) ⊗ ones(T, Nx), Nx, Ny, Nz) - - # Indices of unknowns in velocity vector - indu = 1:Nu - indv = Nu .+ (1:Nv) - indw = Nu + Nv .+ (1:Nw) - indV = 1:NV - indp = NV .+ (1:Np) - - (; - dimension, - order4, - α, - β, - Nx, - Ny, - Nz, - xlims, - ylims, - zlims, - x, - y, - z, - xp, - yp, - zp, - hx, - hy, - hz, - gx, - gy, - gz, - Npx, - Npy, - Npz, - Np, - Nux_in, - Nux_b, - Nux_t, - Nuy_in, - Nuy_b, - Nuy_t, - Nuz_in, - Nuz_b, - Nuz_t, - Nvx_in, - Nvx_b, - Nvx_t, - Nvy_in, - Nvy_b, - Nvy_t, - Nvz_in, - Nvz_b, - Nvz_t, - Nwx_in, - Nwx_b, - Nwx_t, - Nwy_in, - Nwy_b, - Nwy_t, - Nwz_in, - Nwz_b, - Nwz_t, - Nu, - Nv, - Nw, - NV, - Ωp, - Ω, - hxi, - hyi, - hzi, - hxd, - hyd, - hzd, - gxi, - gyi, - gzi, - gxd, - gyd, - gzd, - Buvy, - Bvux, - Buwz, - Bwux, - Bvwz, - Bwvy, - Bkux, - Bkvy, - Bkwz, - xin, - yin, - zin, - xu, - yu, - zu, - xv, - yv, - zv, - xw, - yw, - zw, - xpp, - ypp, - zpp, - indu, - indv, - indw, - indV, - indp, - ) + # Grid quantities + (; dimension, N, Nu, Np, Iu, Ip, xlims, x, xp, Δ, Δu, Ω, Ωu, Ωω, Γu) end diff --git a/src/grid/max_size.jl b/src/grid/max_size.jl index 40961a1fa..23c05bce5 100644 --- a/src/grid/max_size.jl +++ b/src/grid/max_size.jl @@ -4,7 +4,7 @@ Get size of the largest grid element. """ function max_size(grid) - (; hx, hy) = grid - Δx, Δy = maximum(hx), maximum(hy) - √(Δx^2 + Δy^2) + (; Δ) = grid + m = maximum.(Δ) + sqrt(sum(m .^ 2)) end diff --git a/src/models/viscosity_models.jl b/src/models/viscosity_models.jl index a76d0490b..10fe03d56 100644 --- a/src/models/viscosity_models.jl +++ b/src/models/viscosity_models.jl @@ -10,7 +10,7 @@ abstract type AbstractViscosityModel end Laminar model. """ -Base.@kwdef struct LaminarModel <: AbstractViscosityModel +struct LaminarModel <: AbstractViscosityModel end """ @@ -18,16 +18,16 @@ end Mixing-length model with mixing length `lm`. """ -Base.@kwdef struct MixingLengthModel{T} <: AbstractViscosityModel +@kwdef struct MixingLengthModel{T} <: AbstractViscosityModel lm::T = 1 # Mixing length end """ - SmagorinskyModel(Re, C_s = 0.17) + SmagorinskyModel(C_s = 0.17) Smagorinsky-Lilly model with constant `C_s`. """ -Base.@kwdef struct SmagorinskyModel{T} <: AbstractViscosityModel +@kwdef struct SmagorinskyModel{T} <: AbstractViscosityModel C_s::T = 0.17 # Smagorinsky constant end diff --git a/src/momentum.jl b/src/momentum.jl new file mode 100644 index 000000000..d791d5e77 --- /dev/null +++ b/src/momentum.jl @@ -0,0 +1,216 @@ +# Let this be constant for now +# const WORKGROUP = 64 +const WORKGROUP = 256 + +# See https://b-fg.github.io/2023/05/07/waterlily-on-gpu.html +# for writing kernel loops + +struct Offset{D} end +(::Offset{D})(i) where {D} = CartesianIndex(ntuple(j -> j == i ? 1 : 0, D)) + +function divergence!(M, u, setup) + (; boundary_conditions, grid) = setup + (; Δ, Np, Ip, Ω) = grid + D = length(u) + δ = Offset{D}() + @kernel function _divergence!(M, u, α, I0) + I = @index(Global, Cartesian) + I = I + I0 + # D = length(I) + # δ = Offset{D}() + M[I] += Ω[I] / Δ[α][I[α]] * (u[α][I] - u[α][I-δ(α)]) + end + M .= 0 + I0 = first(Ip) + I0 -= oneunit(I0) + for α = 1:D + _divergence!(get_backend(M), WORKGROUP)(M, u, α, I0; ndrange = Np) + synchronize(get_backend(M)) + end + M +end + +divergence(u, setup) = divergence!( + KernelAbstractions.zeros(get_backend(u[1]), eltype(u[1]), setup.grid.N...), + u, + setup, +) + +vorticity(u, setup) = vorticity!( + setup.grid.dimension() == 2 ? + KernelAbstractions.zeros(get_backend(u[1]), eltype(u[1]), setup.grid.N) : + ntuple( + α -> KernelAbstractions.zeros(get_backend(u[1]), eltype(u[1]), setup.grid.N), + setup.grid.dimension(), + ), + u, + setup, +) +vorticity!(ω, u, setup) = vorticity!(setup.grid.dimension, ω, u, setup) + +function vorticity!(::Dimension{2}, ω, u, setup) + (; boundary_conditions, grid) = setup + (; dimension, Δu, N) = grid + D = dimension() + δ = Offset{D}() + @kernel function _vorticity!(ω, u, I0) + I = @index(Global, Cartesian) + I = I + I0 + ω[I] = + -Δu[1][I[1]] * (u[1][I+δ(2)] - u[1][I]) + Δu[2][I[2]] * (u[2][I+δ(1)] - u[2][I]) + end + I0 = CartesianIndex(ntuple(Returns(1), D)) + I0 -= oneunit(I0) + _vorticity!(get_backend(ω), WORKGROUP)(ω, u, I0; ndrange = N .- 1) + synchronize(get_backend(ω)) + ω +end + +function vorticity!(::Dimension{3}, ω, u, setup) + (; boundary_conditions, grid) = setup + (; dimension, Δu, Nu, Iω, Ωω) = grid + D = dimension() + δ = Offset{D}() + @kernel function _vorticity!(ω, u, α, I0; Δu) + T = eltype(ω) + I = @index(Global, Cartesian) + I = I + I0 + β = mod1(α + 1, D) + γ = mod1(α - 1, D) + ω[α][I] = + -Ωω[I] / Δu[γ] * (u[β][I+δ(γ)] - u[β][I]) + + Ωω[I] / Δu[β] * (u[γ][I+δ(β)] - u[γ][I]) + end + for α = 1:D + I0 = first(Iu[α]) + I0 -= oneunit(I0) + _vorticity!(get_backend(ω), WORKGROUP)(ω, u, α, I0; Δu, ndrange = N .- 1) + synchronize(get_backend(ω[1])) + end + ω +end + +function convection!(F, u, setup) + (; boundary_conditions, grid, Re, bodyforce) = setup + (; dimension, Δ, Δu, Nu, Iu, Γu) = grid + D = dimension() + δ = Offset{D}() + @kernel function _convection!(F, u, α, β, I0) + I = @index(Global, Cartesian) + I = I + I0 + F[α][I] -= + Γu[α][β][I] * ( + (u[α][I] + u[α][I+δ(β)]) / 2 * (u[β][I] + u[β][I+δ(α)]) / 2 - + (u[α][I-δ(β)] + u[α][I]) / 2 * (u[β][I-δ(β)] + u[β][I-δ(β)+δ(α)]) / 2 + ) + end + for α = 1:D + I0 = first(Iu[α]) + I0 -= oneunit(I0) + for β = 1:D + _convection!(get_backend(F[1]), WORKGROUP)(F, u, α, β, I0; ndrange = Nu[α]) + synchronize(get_backend(F[1])) + end + end + F +end + +# Add ϵ in denominator for "infinitely thin" volumes +function diffusion!(F, u, setup; ϵ = eps(eltype(F[1]))) + (; boundary_conditions, grid, Re, bodyforce) = setup + (; dimension, Δ, Δu, Nu, Iu, Ωu, Γu) = grid + D = dimension() + δ = Offset{D}() + ν = 1 / Re + @kernel function _diffusion!(F, u, α, β, I0) + I = @index(Global, Cartesian) + I = I + I0 + F[α][I] += + ν * + Γu[α][β][I] * + ( + (u[α][I+δ(β)] - u[α][I]) / ((β == α ? Δ : Δu)[β][I[β]] + ϵ) - + (u[α][I] - u[α][I-δ(β)]) / ((β == α ? Δ : Δu)[β][(I-δ(β))[β]] + ϵ) + ) + end + for α = 1:D + I0 = first(Iu[α]) + I0 -= oneunit(I0) + for β = 1:D + _diffusion!(get_backend(F[1]), WORKGROUP)(F, u, α, β, I0; ndrange = Nu[α]) + synchronize(get_backend(F[1])) + end + end + F +end + +function bodyforce!(F, u, t, setup) + (; boundary_conditions, grid, Re, bodyforce) = setup + (; dimension, Δ, Δu, Nu, Iu, x, xp) = grid + D = dimension() + δ = Offset{D}() + @kernel function _bodyforce!(F, force, α, t, I0) + I = @index(Global, Cartesian) + I = I + I0 + vol = prod(β -> (β == α ? Δu : Δ)[k][I[k]], ntuple(identity, D)) + # vol = Δu[i] * prod(Δ) / Δ[i] + F[α][I] += force[α](ntuple(β -> α == β ? x[β][1+I[β]] : xp[β][I[β]], D)..., t) + end + for α = 1:D + I0 = first(Iu[α]) + I0 -= oneunit(I0) + isnothing(bodyforce) || _bodyforce!(get_backend(F[1]), WORKGROUP)( + F, + bodyforce, + α, + t, + I0; + ndrange = Nu[α], + ) + synchronize(get_backend(F[1])) + end + F +end + +function momentum!(F, u, t, setup) + (; grid, closure_model) = setup + (; dimension) = grid + D = dimension() + for α = 1:D + F[α] .= 0 + end + convection!(F, u, setup) + diffusion!(F, u, setup) + bodyforce!(F, u, t, setup) + isnothing(closure_model) || (F .+= closure_model(u)) + F +end + +function pressuregradient!(G, p, setup) + (; boundary_conditions, grid) = setup + (; dimension, Δ, Np, Iu, Ω) = grid + D = dimension() + δ = Offset{D}() + @kernel function _pressuregradient!(G, p, α, I0) + I = @index(Global, Cartesian) + I = I0 + I + G[α][I] = Ω[I] / Δ[α][I[α]] * (p[I+δ(α)] - p[I]) + end + D = dimension() + for α = 1:D + I0 = first(Iu[α]) + I0 -= oneunit(I0) + _pressuregradient!(get_backend(G[1]), WORKGROUP)(G, p, α, I0; ndrange = Np) + synchronize(get_backend(G[1])) + end + G +end + +pressuregradient(p, setup) = pressuregradient!( + ntuple( + α -> KernelAbstractions.zeros(get_backend(p), eltype(p), setup.grid.N), + setup.grid.dimension(), + ), + p, + setup, +) diff --git a/src/momentum/check_symmetry.jl b/src/momentum/check_symmetry.jl deleted file mode 100644 index 44e5def85..000000000 --- a/src/momentum/check_symmetry.jl +++ /dev/null @@ -1,45 +0,0 @@ -""" - check_symmetry(V, t, setup, ϵ = 1e-14) - -Check symmetry of convection operator. - -`flag = 0`: no symmetry error -`flag = 1`: symmetry error -""" -function check_symmetry(V, t, setup, ϵ = 1e-14; bc_vectors) - (; grid, operators, boundary_conditions) = setup - (; indu, indv) = grid - (; Cux, Cuy, Cvx, Cvy) = operators - (; Au_ux, Au_uy, Av_vx, Av_vy) = operators - (; Iu_ux, Iv_uy, Iu_vx, Iv_vy) = operators - (; yIu_ux, yIv_uy, yIu_vx, yIv_vy) = bc_vectors - - uₕ = @view V[indu] - vₕ = @view V[indv] - - Cu = - Cux * spdiagm(Iu_ux * uₕ + yIu_ux) * Au_ux + - Cuy * spdiagm(Iv_uy * vₕ + yIv_uy) * Au_uy - Cv = - Cvx * spdiagm(Iu_vx * uₕ + yIu_vx) * Av_vx + - Cvy * spdiagm(Iv_vy * vₕ + yIv_vy) * Av_vy - - error_u = maximum(abs.(Cu + Cu')) - error_v = maximum(abs.(Cv + Cv')) - - symmetry_error = max(error_u, error_v) - - flag = 0 - if symmetry_error > ϵ - if boundary_conditions.u.x[1] != :pressure && - boundary_conditions.u.x[2] != :pressure - flag = 1 - end - if boundary_conditions.v.y[1] != :pressure && - boundary_conditions.v.y[2] != :pressure - flag = 1 - end - end - - flag, symmetry_error -end diff --git a/src/momentum/compute_conservation.jl b/src/momentum/compute_conservation.jl deleted file mode 100644 index 5b8c7757a..000000000 --- a/src/momentum/compute_conservation.jl +++ /dev/null @@ -1,124 +0,0 @@ -""" - compute_conservation(V, t, setup; bc_vectors = nothing) - -Compute mass, momentum and energy conservation properties of velocity field. -""" -function compute_conservation end - -compute_conservation(V, t, setup; bc_vectors = nothing) = - compute_conservation(setup.grid.dimension, V, t, setup; bc_vectors = nothing) - -# 2D version -function compute_conservation(::Dimension{2}, V, t, setup; bc_vectors = nothing) - (; grid, operators, boundary_conditions) = setup - (; indu, indv, Ω, x, y, xp, yp, hx, hy, gx, gy) = grid - (; M) = operators - (; bc_unsteady, u_bc, v_bc) = boundary_conditions - - uₕ = @view V[indu] - vₕ = @view V[indv] - - Ωu = @view Ω[indu] - Ωv = @view Ω[indv] - - if isnothing(bc_vectors) || bc_unsteady - bc_vectors = get_bc_vectors(setup, t) - end - (; yM) = bc_vectors - - uLe_i = reshape(u_bc.(x[1], yp, t), :) - uRi_i = reshape(u_bc.(x[end], yp, t), :) - vLo_i = reshape(v_bc.(xp, y[1], t), :) - vUp_i = reshape(v_bc.(xp, y[end], t), :) - - # Check if new velocity field is divergence free (mass conservation) - maxdiv = maximum(abs.(M * V + yM)) - - # Calculate total momentum - umom = sum(Ωu .* uₕ) - vmom = sum(Ωv .* vₕ) - - # Add boundary contributions in case of Dirichlet BC - boundary_conditions.u.x[1] == :dirichlet && (umom += sum(uLe_i .* hy) * gx[1]) - boundary_conditions.u.x[2] == :dirichlet && (umom += sum(uRi_i .* hy) * gx[end]) - boundary_conditions.v.y[1] == :dirichlet && (vmom += sum(vLo_i .* hx) * gy[1]) - boundary_conditions.v.y[2] == :dirichlet && (vmom += sum(vUp_i .* hx) * gy[end]) - - # Calculate total kinetic energy - k = 1 // 2 * sum(Ω .* V .^ 2) - - # Add boundary contributions in case of Dirichlet BC - boundary_conditions.u.x[1] == :dirichlet && - (k += 1 // 2 * sum(uLe_i .^ 2 .* hy) * gx[1]) - boundary_conditions.u.x[2] == :dirichlet && - (k += 1 // 2 * sum(uRi_i .^ 2 .* hy) * gx[end]) - boundary_conditions.v.y[1] == :dirichlet && - (k += 1 // 2 * sum(vLo_i .^ 2 .* hx) * gy[1]) - boundary_conditions.v.y[2] == :dirichlet && - (k += 1 // 2 * sum(vUp_i .^ 2 .* hx) * gy[end]) - - maxdiv, umom, vmom, k -end - -# 3D version -function compute_conservation(::Dimension{3}, V, t, setup; bc_vectors = nothing) - (; grid, operators, boundary_conditions) = setup - (; indu, indv, indw, Ω, x, y, z, xp, yp, zp, hx, hy, hz, gx, gy, gz) = grid - (; M) = operators - (; bc_unsteady, u_bc, v_bc, w_bc) = boundary_conditions - - uₕ = @view V[indu] - vₕ = @view V[indv] - wₕ = @view V[indw] - - Ωu = @view Ω[indu] - Ωv = @view Ω[indv] - Ωw = @view Ω[indw] - - if isnothing(bc_vectors) || bc_unsteady - bc_vectors = get_bc_vectors(setup, t) - end - (; yM) = bc_vectors - - uLe_i = reshape(u_bc.(x[1], yp, zp', t), :) - uRi_i = reshape(u_bc.(x[end], yp, zp', t), :) - vLo_i = reshape(v_bc.(xp, y[1], zp', t), :) - vUp_i = reshape(v_bc.(xp, y[end], zp', t), :) - wBa_i = reshape(w_bc.(xp, yp', z[1], t), :) - wFr_i = reshape(w_bc.(xp, yp', z[end], t), :) - - # Check if new velocity field is divergence free (mass conservation) - maxdiv = maximum(abs.(M * V + yM)) - - # Calculate total momentum - umom = sum(Ωu .* uₕ) - vmom = sum(Ωv .* vₕ) - wmom = sum(Ωw .* wₕ) - - # Add boundary contributions in case of Dirichlet BC - boundary_conditions.u.x[1] == :dirichlet && (umom += sum(uLe_i .* (hz ⊗ hy)) * gx[1]) - boundary_conditions.u.x[2] == :dirichlet && (umom += sum(uRi_i .* (hz ⊗ hy)) * gx[end]) - boundary_conditions.v.y[1] == :dirichlet && (vmom += sum(vLo_i .* (hz ⊗ hx)) * gy[1]) - boundary_conditions.v.y[2] == :dirichlet && (vmom += sum(vUp_i .* (hz ⊗ hx)) * gy[end]) - boundary_conditions.w.z[1] == :dirichlet && (wmom += sum(wBa_i .* (hy ⊗ hx)) * gz[1]) - boundary_conditions.w.z[2] == :dirichlet && (wmom += sum(wFr_i .* (hy ⊗ hx)) * gz[end]) - - # Calculate total kinetic energy - k = 1 // 2 * sum(Ω .* V .^ 2) - - # Add boundary contributions in case of Dirichlet BC - boundary_conditions.u.x[1] == :dirichlet && - (k += 1 // 2 * sum(uLe_i .^ 2 .* (hz ⊗ hy)) * gx[1]) - boundary_conditions.u.x[2] == :dirichlet && - (k += 1 // 2 * sum(uRi_i .^ 2 .* (hz ⊗ hy)) * gx[end]) - boundary_conditions.v.y[1] == :dirichlet && - (k += 1 // 2 * sum(vLo_i .^ 2 .* (hz ⊗ hx)) * gy[1]) - boundary_conditions.v.y[2] == :dirichlet && - (k += 1 // 2 * sum(vUp_i .^ 2 .* (hz ⊗ hx)) * gy[end]) - boundary_conditions.w.z[1] == :dirichlet && - (k += 1 // 2 * sum(wBa_i .^ 2 .* (hy ⊗ hx)) * gz[1]) - boundary_conditions.w.z[2] == :dirichlet && - (k += 1 // 2 * sum(wFr_i .^ 2 .* (hy ⊗ hx)) * gz[end]) - - maxdiv, umom, vmom, wmom, k -end diff --git a/src/momentum/convection.jl b/src/momentum/convection.jl deleted file mode 100644 index bcc33544f..000000000 --- a/src/momentum/convection.jl +++ /dev/null @@ -1,860 +0,0 @@ -""" - convection( - model, V, ϕ, setup; - bc_vectors, - get_jacobian = false, - newton_factor = false, - ) - -Evaluate convective terms `c` and, optionally, Jacobian `∇c = ∂c/∂V`, using the convection -model `model`. The convected quantity is `ϕ` (usually `ϕ = V`). - -Non-mutating/allocating/out-of-place version. - -See also [`convection!`](@ref). -""" -function convection end - -convection(m, V, ϕ, setup; kwargs...) = - convection(setup.grid.dimension, m, V, ϕ, setup; kwargs...) - -function convection( - dimension, - ::NoRegConvectionModel, - V, - ϕ, - setup; - bc_vectors, - get_jacobian = false, - newton_factor = false, -) - (; order4, α) = setup.grid - - # No regularization - c, ∇c = convection_components( - V, - ϕ, - setup; - bc_vectors, - get_jacobian, - newton_factor, - order4 = false, - ) - - if order4 - c3, ∇c3 = convection_components( - V, - ϕ, - setup; - bc_vectors, - get_jacobian, - newton_factor, - order4, - ) - c = @. α * c - c3 - get_jacobian && (∇c = @. α * ∇c - ∇c3) - end - - c, ∇c -end - -# 2D version -function convection( - ::Dimension{2}, - ::C2ConvectionModel, - V, - ϕ, - setup; - bc_vectors, - get_jacobian = false, - newton_factor = false, -) - (; grid, operators) = setup - (; indu, indv) = grid - (; Diffu_f, Diffv_f, M, α_reg) = operators - (; yDiffu_f, yDiffv_f, yM) = bc_vectors - - uₕ = @view V[indu] - vₕ = @view V[indv] - - ϕu = @view ϕ[indu] - ϕv = @view ϕ[indv] - - ϕ̄u = filter_convection(ϕu, Diffu_f, yDiffu_f, α_reg) - ϕ̄v = filter_convection(ϕv, Diffv_f, yDiffv_f, α_reg) - - ūₕ = filter_convection(uₕ, Diffu_f, yDiffu_f, α_reg) - v̄ₕ = filter_convection(vₕ, Diffv_f, yDiffv_f, α_reg) - - ϕ̄ = [ϕ̄u; ϕ̄v] - V̄ = [ūₕ; v̄ₕ] - - # Divergence of filtered velocity field; should be zero! - maxdiv_f = maximum(abs.(M * ϕ̄ + yM)) - - c, ∇c = convection_components(V̄, ϕ̄, setup; bc_vectors, get_jacobian, newton_factor) - - cu = @view c[indu] - cv = @view c[indv] - - cu = filter_convection(cu, Diffu_f, yDiffu_f, α_reg) - cv = filter_convection(cv, Diffv_f, yDiffv_f, α_reg) - - c = [cu; cv] - - c, ∇c -end - -# 3D version -function convection( - ::Dimension{3}, - ::C2ConvectionModel, - V, - ϕ, - setup; - bc_vectors, - get_jacobian = false, - newton_factor = false, -) - (; grid, operators) = setup - (; indu, indv, indw) = grid - (; Diffu_f, Diffv_f, Diffw_f, M, α_reg) = operators - (; yDiffu_f, yDiffv_f, yDiffw_f, yM) = bc_vectors - - uₕ = @view V[indu] - vₕ = @view V[indv] - wₕ = @view V[indw] - - ϕu = @view ϕ[indu] - ϕv = @view ϕ[indv] - ϕw = @view ϕ[indw] - - ϕ̄u = filter_convection(ϕu, Diffu_f, yDiffu_f, α_reg) - ϕ̄v = filter_convection(ϕv, Diffv_f, yDiffv_f, α_reg) - ϕ̄w = filter_convection(ϕw, Diffw_f, yDiffw_f, α_reg) - - ūₕ = filter_convection(uₕ, Diffu_f, yDiffu_f, α_reg) - v̄ₕ = filter_convection(vₕ, Diffv_f, yDiffv_f, α_reg) - w̄ₕ = filter_convection(wₕ, Diffw_f, yDiffw_f, α_reg) - - ϕ̄ = [ϕ̄u; ϕ̄v; ϕ̄w] - V̄ = [ūₕ; v̄ₕ; w̄ₕ] - - # Divergence of filtered velocity field; should be zero! - maxdiv_f = maximum(abs.(M * ϕ̄ + yM)) - - c, ∇c = - convection_components(V̄, ϕ̄, setup, cache; bc_vectors, get_jacobian, newton_factor) - - cu = @view c[indu] - cv = @view c[indv] - cw = @view c[indw] - - cu = filter_convection(cu, Diffu_f, yDiffu_f, α_reg) - cv = filter_convection(cv, Diffv_f, yDiffv_f, α_reg) - cw = filter_convection(cw, Diffw_f, yDiffw_f, α_reg) - - c = [cu; cv; cw] - - c, ∇c -end - -# 2D version -function convection( - ::Dimension{2}, - ::C4ConvectionModel, - V, - ϕ, - setup; - bc_vectors, - get_jacobian = false, - newton_factor = false, -) - (; grid, operators) = setup - (; indu, indv) = grid - (; Diffu_f, Diffv_f, M, α_reg) = operators - (; yDiffu_f, yDiffv_f, yM) = bc_vectors - - uₕ = @view V[indu] - vₕ = @view V[indv] - - ϕu = @view ϕ[indu] - ϕv = @view ϕ[indv] - - # C4 consists of 3 terms: - # C4 = conv(filter(u), filter(u)) + filter(conv(filter(u), u') + - # filter(conv(u', filter(u))) - # where u' = u - filter(u) - - # Filter both convecting and convected velocity - ūₕ = filter_convection(uₕ, Diffu_f, yDiffu_f, α_reg) - v̄ₕ = filter_convection(vₕ, Diffv_f, yDiffv_f, α_reg) - - V̄ = [ūₕ; v̄ₕ] - ΔV = V - V̄ - - ϕ̄u = filter_convection(ϕu, Diffu_f, yDiffu_f, α_reg) - ϕ̄v = filter_convection(ϕv, Diffv_f, yDiffv_f, α_reg) - - ϕ̄ = [ϕ̄u; ϕ̄v] - Δϕ = ϕ - ϕ̄ - - # Divergence of filtered velocity field; should be zero! - maxdiv_f = maximum(abs.(M * V̄ + yM)) - - c, ∇c = convection_components(V̄, ϕ̄, setup; bc_vectors, get_jacobian, newton_factor) - c2, ∇c2 = convection_components(ΔV, ϕ̄, setup; bc_vectors, get_jacobian, newton_factor) - c3, ∇c3 = convection_components(V̄, Δϕ, setup; bc_vectors, get_jacobian, newton_factor) - - cu = @view c[indu] - cv = @view c[indv] - - cu2 = @view c2[indu] - cv2 = @view c2[indv] - - cu3 = @view c3[indu] - cv3 = @view c3[indv] - - cu += filter_convection(cu2 + cu3, Diffu_f, yDiffu_f, α_reg) - cv += filter_convection(cv2 + cv3, Diffv_f, yDiffv_f, α_reg) - - c = [cu; cv] - - c, ∇c -end - -# 3D version -function convection( - ::Dimension{3}, - ::C4ConvectionModel, - V, - ϕ, - setup; - bc_vectors, - get_jacobian = false, - newton_factor = false, -) - (; grid, operators) = setup - (; indu, indv, indw) = grid - (; Diffu_f, Diffv_f, Diffw_f, M, α_reg) = operators - (; yDiffu_f, yDiffv_f, yDiffw_f, yM) = bc_vectors - (; c2, ∇c2, c3, ∇c3) = cache - - uₕ = @view V[indu] - vₕ = @view V[indv] - wₕ = @view V[indw] - - ϕu = @view ϕ[indu] - ϕv = @view ϕ[indv] - ϕw = @view ϕ[indw] - - # C4 consists of 3 terms: - # C4 = conv(filter(u), filter(u)) + filter(conv(filter(u), u') + - # filter(conv(u', filter(u))) - # where u' = u - filter(u) - - # Filter both convecting and convected velocity - ūₕ = filter_convection(uₕ, Diffu_f, yDiffu_f, α_reg) - v̄ₕ = filter_convection(vₕ, Diffv_f, yDiffv_f, α_reg) - w̄ₕ = filter_convection(wₕ, Diffw_f, yDiffw_f, α_reg) - - V̄ = [ūₕ; v̄ₕ; w̄ₕ] - ΔV = V - V̄ - - ϕ̄u = filter_convection(ϕu, Diffu_f, yDiffu_f, α_reg) - ϕ̄v = filter_convection(ϕv, Diffv_f, yDiffv_f, α_reg) - ϕ̄w = filter_convection(ϕw, Diffw_f, yDiffw_f, α_reg) - - ϕ̄ = [ϕ̄u; ϕ̄v; ϕ̄w] - Δϕ = ϕ - ϕ̄ - - # Divergence of filtered velocity field; should be zero! - maxdiv_f = maximum(abs.(M * V̄ + yM)) - - c, ∇c = convection_components(V̄, ϕ̄, setup; bc_vectors, get_jacobian, newton_factor) - c2, ∇c2 = convection_components(ΔV, ϕ̄, setup; bc_vectors, get_jacobian, newton_factor) - c3, ∇c3 = convection_components(V̄, Δϕ, setup; bc_vectors, get_jacobian, newton_factor) - - cu = @view c[indu] - cv = @view c[indv] - cw = @view c[indw] - - cu2 = @view c2[indu] - cv2 = @view c2[indv] - cw2 = @view c2[indw] - - cu3 = @view c3[indu] - cv3 = @view c3[indv] - cw3 = @view c3[indw] - - cu += filter_convection(cu2 + cu3, Diffu_f, yDiffu_f, α_reg) - cv += filter_convection(cv2 + cv3, Diffv_f, yDiffv_f, α_reg) - cw += filter_convection(cw2 + cw3, Diffw_f, yDiffw_f, α_reg) - - c = [cu; cv; cw] - - c, ∇c -end - -# 2D version -function convection( - ::Dimension{2}, - ::LerayConvectionModel, - V, - ϕ, - setup; - bc_vectors, - get_jacobian = false, - newton_factor = false, -) - (; grid, operators) = setup - (; indu, indv) = grid - (; Diffu_f, Diffv_f, M, α_reg) = operators - (; yDiffu_f, yDiffv_f, yM) = bc_vectors - - ϕu = @view ϕ[indu] - ϕv = @view ϕ[indv] - - # TODO: needs finishing - error("Leray convection not implemented yet") - - # Filter the convecting field - ϕ̄u = filter_convection(ϕu, Diffu_f, yDiffu_f, α_reg) - ϕ̄v = filter_convection(ϕv, Diffv_f, yDiffv_f, α_reg) - - ϕ̄ = [ϕ̄u; ϕ̄v] - - # Divergence of filtered velocity field; should be zero! - maxdiv_f = maximum(abs.(M * ϕ̄ + yM)) - - convection_components(V, ϕ̄, setup; bc_vectors, get_jacobian, newton_factor) -end - -# 3D version -function convection( - ::Dimension{3}, - ::LerayConvectionModel, - c, - ∇c, - V, - ϕ, - setup, - bc_vectors, - cache; - get_jacobian = false, - newton_factor = false, -) - (; grid, operators) = setup - (; indu, indv, indw) = grid - (; Diffu_f, Diffv_f, Diffw_f, M, α_reg) = operators - (; yDiffu_f, yDiffv_f, yDiffw_f, yM) = bc_vectors - - ϕu = @view ϕ[indu] - ϕv = @view ϕ[indv] - ϕw = @view ϕ[indw] - - # TODO: needs finishing - error("Leray convection not implemented yet") - - # Filter the convecting field - ϕ̄u = filter_convection(ϕu, Diffu_f, yDiffu_f, α_reg) - ϕ̄v = filter_convection(ϕv, Diffv_f, yDiffv_f, α_reg) - ϕ̄w = filter_convection(ϕw, Diffw_f, yDiffw_f, α_reg) - - ϕ̄ = [ϕ̄u; ϕ̄v; ϕ̄w] - - # Divergence of filtered velocity field; should be zero! - maxdiv_f = maximum(abs.(M * ϕ̄ + yM)) - - convection_components(V, ϕ̄, setup; bc_vectors, get_jacobian, newton_factor) -end - -""" - convection!( - model, c, ∇c, V, ϕ, setup, cache; - bc_vectors, - get_jacobian = false, - newton_factor = false, - ) - -Evaluate convective terms `c` and, optionally, Jacobian `∇c = ∂c/∂V`, using the convection -model `model`. The convected quantity is `ϕ` (usually `ϕ = V`). - -Mutating/non-allocating/in-place version. - -See also [`convection`](@ref). -""" -function convection! end - -convection!(m, c, ∇c, V, ϕ, setup, cache; kwargs...) = - convection!(setup.grid.dimension, m, c, ∇c, V, ϕ, setup, cache; kwargs...) - -function convection!( - dimension, - ::NoRegConvectionModel, - c, - ∇c, - V, - ϕ, - setup, - cache; - bc_vectors, - get_jacobian = false, - newton_factor = false, -) - (; order4, α) = setup.grid - (; c3, ∇c3) = cache - - # No regularization - convection_components!( - c, - ∇c, - V, - ϕ, - setup, - cache; - bc_vectors, - get_jacobian, - newton_factor, - order4 = false, - ) - - if order4 - convection_components!( - c3, - ∇c3, - V, - ϕ, - setup, - cache; - bc_vectors, - get_jacobian, - newton_factor, - order4, - ) - @. c = α * c - c3 - get_jacobian && (@. ∇c = α * ∇c - ∇c3) - end - - c, ∇c -end - -# 2D version -function convection!( - ::Dimension{2}, - ::C2ConvectionModel, - c, - ∇c, - V, - ϕ, - setup, - cache; - bc_vectors, - get_jacobian = false, - newton_factor = false, -) - (; grid, operators) = setup - (; indu, indv) = grid - (; Diffu_f, Diffv_f, M, α_reg) = operators - (; yDiffu_f, yDiffv_f, yM) = bc_vectors - - cu = @view c[indu] - cv = @view c[indv] - - uₕ = @view V[indu] - vₕ = @view V[indv] - - ϕu = @view ϕ[indu] - ϕv = @view ϕ[indv] - - ϕ̄u = filter_convection(ϕu, Diffu_f, yDiffu_f, α_reg) - ϕ̄v = filter_convection(ϕv, Diffv_f, yDiffv_f, α_reg) - - ūₕ = filter_convection(uₕ, Diffu_f, yDiffu_f, α_reg) - v̄ₕ = filter_convection(vₕ, Diffv_f, yDiffv_f, α_reg) - - ϕ̄ = [ϕ̄u; ϕ̄v] - V̄ = [ūₕ; v̄ₕ] - - # Divergence of filtered velocity field; should be zero! - maxdiv_f = maximum(abs.(M * ϕ̄ + yM)) - - convection_components!( - c, - ∇c, - V̄, - ϕ̄, - setup, - cache; - bc_vectors, - get_jacobian, - newton_factor, - ) - - cu .= filter_convection(cu, Diffu_f, yDiffu_f, α_reg) - cv .= filter_convection(cv, Diffv_f, yDiffv_f, α_reg) - - c, ∇c -end - -# 3D version -function convection!( - ::Dimension{3}, - ::C2ConvectionModel, - c, - ∇c, - V, - ϕ, - setup, - cache; - bc_vectors, - get_jacobian = false, - newton_factor = false, -) - (; grid, operators) = setup - (; indu, indv, indw) = grid - (; Diffu_f, Diffv_f, Diffw_f, M, α_reg) = operators - (; yDiffu_f, yDiffv_f, yDiffw_f, yM) = bc_vectors - - cu = @view c[indu] - cv = @view c[indv] - cw = @view c[indw] - - uₕ = @view V[indu] - vₕ = @view V[indv] - wₕ = @view V[indw] - - ϕu = @view ϕ[indu] - ϕv = @view ϕ[indv] - ϕw = @view ϕ[indw] - - ϕ̄u = filter_convection(ϕu, Diffu_f, yDiffu_f, α_reg) - ϕ̄v = filter_convection(ϕv, Diffv_f, yDiffv_f, α_reg) - ϕ̄w = filter_convection(ϕw, Diffw_f, yDiffw_f, α_reg) - - ūₕ = filter_convection(uₕ, Diffu_f, yDiffu_f, α_reg) - v̄ₕ = filter_convection(vₕ, Diffv_f, yDiffv_f, α_reg) - w̄ₕ = filter_convection(wₕ, Diffw_f, yDiffw_f, α_reg) - - ϕ̄ = [ϕ̄u; ϕ̄v; ϕ̄w] - V̄ = [ūₕ; v̄ₕ; w̄ₕ] - - # Divergence of filtered velocity field; should be zero! - maxdiv_f = maximum(abs.(M * ϕ̄ + yM)) - - convection_components!( - c, - ∇c, - V̄, - ϕ̄, - setup, - cache; - bc_vectors, - get_jacobian, - newton_factor, - ) - - cu .= filter_convection(cu, Diffu_f, yDiffu_f, α_reg) - cv .= filter_convection(cv, Diffv_f, yDiffv_f, α_reg) - cw .= filter_convection(cw, Diffw_f, yDiffw_f, α_reg) - - c, ∇c -end - -# 2D version -function convection!( - ::Dimension{2}, - ::C4ConvectionModel, - c, - ∇c, - V, - ϕ, - setup, - cache; - bc_vectors, - get_jacobian = false, - newton_factor = false, -) - (; grid, operators) = setup - (; indu, indv) = grid - (; Diffu_f, Diffv_f, M, α_reg) = operators - (; yDiffu_f, yDiffv_f, yM) = bc_vectors - (; c2, ∇c2, c3, ∇c3) = cache - - cu = @view c[indu] - cv = @view c[indv] - cu2 = @view c2[indu] - cv2 = @view c2[indv] - cu3 = @view c3[indu] - cv3 = @view c3[indv] - - uₕ = @view V[indu] - vₕ = @view V[indv] - - ϕu = @view ϕ[indu] - ϕv = @view ϕ[indv] - - # C4 consists of 3 terms: - # C4 = conv(filter(u), filter(u)) + filter(conv(filter(u), u') + - # filter(conv(u', filter(u))) - # where u' = u - filter(u) - - # Filter both convecting and convected velocity - ūₕ = filter_convection(uₕ, Diffu_f, yDiffu_f, α_reg) - v̄ₕ = filter_convection(vₕ, Diffv_f, yDiffv_f, α_reg) - - V̄ = [ūₕ; v̄ₕ] - ΔV = V - V̄ - - ϕ̄u = filter_convection(ϕu, Diffu_f, yDiffu_f, α_reg) - ϕ̄v = filter_convection(ϕv, Diffv_f, yDiffv_f, α_reg) - - ϕ̄ = [ϕ̄u; ϕ̄v] - Δϕ = ϕ - ϕ̄ - - # Divergence of filtered velocity field; should be zero! - maxdiv_f = maximum(abs.(M * V̄ + yM)) - - convection_components!( - c, - ∇c, - V̄, - ϕ̄, - setup, - cache; - bc_vectors, - get_jacobian, - newton_factor, - ) - convection_components!( - c2, - ∇c2, - ΔV, - ϕ̄, - setup, - cache; - bc_vectors, - get_jacobian, - newton_factor, - ) - convection_components!( - c3, - ∇c3, - V̄, - Δϕ, - setup, - cache; - bc_vectors, - get_jacobian, - newton_factor, - ) - - cu .+= filter_convection(cu2 + cu3, Diffu_f, yDiffu_f, α_reg) - cv .+= filter_convection(cv2 + cv3, Diffv_f, yDiffv_f, α_reg) - - c, ∇c -end - -# 3D version -function convection!( - ::Dimension{3}, - ::C4ConvectionModel, - c, - ∇c, - V, - ϕ, - setup, - cache; - bc_vectors, - get_jacobian = false, - newton_factor = false, -) - (; grid, operators) = setup - (; indu, indv, indw) = grid - (; Diffu_f, Diffv_f, Diffw_f, M, α_reg) = operators - (; yDiffu_f, yDiffv_f, yDiffw_f, yM) = bc_vectors - (; c2, ∇c2, c3, ∇c3) = cache - - cu = @view c[indu] - cv = @view c[indv] - cw = @view c[indw] - - cu2 = @view c2[indu] - cv2 = @view c2[indv] - cw2 = @view c2[indw] - - cu3 = @view c3[indu] - cv3 = @view c3[indv] - cw3 = @view c3[indw] - - uₕ = @view V[indu] - vₕ = @view V[indv] - wₕ = @view V[indw] - - ϕu = @view ϕ[indu] - ϕv = @view ϕ[indv] - ϕw = @view ϕ[indw] - - # C4 consists of 3 terms: - # C4 = conv(filter(u), filter(u)) + filter(conv(filter(u), u') + - # filter(conv(u', filter(u))) - # where u' = u - filter(u) - - # Filter both convecting and convected velocity - ūₕ = filter_convection(uₕ, Diffu_f, yDiffu_f, α_reg) - v̄ₕ = filter_convection(vₕ, Diffv_f, yDiffv_f, α_reg) - w̄ₕ = filter_convection(wₕ, Diffw_f, yDiffw_f, α_reg) - - V̄ = [ūₕ; v̄ₕ; w̄ₕ] - ΔV = V - V̄ - - ϕ̄u = filter_convection(ϕu, Diffu_f, yDiffu_f, α_reg) - ϕ̄v = filter_convection(ϕv, Diffv_f, yDiffv_f, α_reg) - ϕ̄w = filter_convection(ϕw, Diffw_f, yDiffw_f, α_reg) - - ϕ̄ = [ϕ̄u; ϕ̄v; ϕ̄w] - Δϕ = ϕ - ϕ̄ - - # Divergence of filtered velocity field; should be zero! - maxdiv_f = maximum(abs.(M * V̄ + yM)) - - convection_components!( - c, - ∇c, - V̄, - ϕ̄, - setup, - cache; - bc_vectors, - get_jacobian, - newton_factor, - ) - convection_components!( - c2, - ∇c2, - ΔV, - ϕ̄, - setup, - cache; - bc_vectors, - get_jacobian, - newton_factor, - ) - convection_components!( - c3, - ∇c3, - V̄, - Δϕ, - setup, - cache; - bc_vectors, - get_jacobian, - newton_factor, - ) - - cu .+= filter_convection(cu2 + cu3, Diffu_f, yDiffu_f, α_reg) - cv .+= filter_convection(cv2 + cv3, Diffv_f, yDiffv_f, α_reg) - cw .+= filter_convection(cw2 + cw3, Diffw_f, yDiffw_f, α_reg) - - c, ∇c -end - -# 2D version -function convection!( - ::Dimension{2}, - ::LerayConvectionModel, - c, - ∇c, - V, - ϕ, - setup, - cache; - bc_vectors, - get_jacobian = false, - newton_factor = false, -) - (; grid, operators) = setup - (; indu, indv) = grid - (; Diffu_f, Diffv_f, M, α_reg) = operators - (; yDiffu_f, yDiffv_f, yM) = bc_vectors - - ϕu = @view ϕ[indu] - ϕv = @view ϕ[indv] - - # TODO: needs finishing - error("Leray convection not implemented yet") - - # Filter the convecting field - ϕ̄u = filter_convection(ϕu, Diffu_f, yDiffu_f, α_reg) - ϕ̄v = filter_convection(ϕv, Diffv_f, yDiffv_f, α_reg) - - ϕ̄ = [ϕ̄u; ϕ̄v] - - # Divergence of filtered velocity field; should be zero! - maxdiv_f = maximum(abs.(M * ϕ̄ + yM)) - - convection_components!( - c, - ∇c, - V, - ϕ̄, - setup, - cache; - bc_vectors, - get_jacobian, - newton_factor, - ) - - c, ∇c -end - -# 3D version -function convection!( - ::Dimension{3}, - ::LerayConvectionModel, - c, - ∇c, - V, - ϕ, - setup, - cache; - bc_vectors, - get_jacobian = false, - newton_factor = false, -) - (; grid, operators) = setup - (; indu, indv, indw) = grid - (; Diffu_f, Diffv_f, Diffw_f, M, α) = operators - (; yDiffu_f, yDiffv_f, yDiffw_f, yM) = bc_vectors - - ϕu = @view ϕ[indu] - ϕv = @view ϕ[indv] - ϕw = @view ϕ[indw] - - # TODO: needs finishing - error("Leray convection not implemented yet") - - # Filter the convecting field - ϕ̄u = filter_convection(ϕu, Diffu_f, yDiffu_f, α) - ϕ̄v = filter_convection(ϕv, Diffv_f, yDiffv_f, α) - ϕ̄w = filter_convection(ϕw, Diffw_f, yDiffw_f, α) - - ϕ̄ = [ϕ̄u; ϕ̄v; ϕ̄w] - - # Divergence of filtered velocity field; should be zero! - maxdiv_f = maximum(abs.(M * ϕ̄ + yM)) - - convection_components!( - c, - ∇c, - V, - ϕ̄, - setup, - cache; - bc_vectors, - get_jacobian, - newton_factor, - ) - - c, ∇c -end diff --git a/src/momentum/convection_components.jl b/src/momentum/convection_components.jl deleted file mode 100644 index 6783b7a3b..000000000 --- a/src/momentum/convection_components.jl +++ /dev/null @@ -1,634 +0,0 @@ -""" - convection_components( - V, ϕ, setup; - bc_vectors, - get_jacobian = false, - newton_factor = false, - order4 = false, - ) - -Compute convection components. - -Non-mutating/allocating/out-of-place version. - -See also [`convection_components!`](@ref). -""" -function convection_components end - -convection_components(V, ϕ, setup; kwargs...) = - convection_components(setup.grid.dimension, V, ϕ, setup; kwargs...) - -# 2D version -function convection_components( - ::Dimension{2}, - V, - ϕ, - setup; - bc_vectors, - get_jacobian = false, - newton_factor = false, - order4 = false, -) - (; grid, operators) = setup - - if order4 - Cux = operators.Cux3 - Cuy = operators.Cuy3 - Cvx = operators.Cvx3 - Cvy = operators.Cvy3 - - Au_ux = operators.Au_ux3 - Au_uy = operators.Au_uy3 - Av_vx = operators.Av_vx3 - Av_vy = operators.Av_vy3 - - Iu_ux = operators.Iu_ux3 - Iv_uy = operators.Iv_uy3 - Iu_vx = operators.Iu_vx3 - Iv_vy = operators.Iv_vy3 - - yAu_ux = bc_vectors.yAu_ux3 - yAu_uy = bc_vectors.yAu_uy3 - yAv_vx = bc_vectors.yAv_vx3 - yAv_vy = bc_vectors.yAv_vy3 - - yIu_ux = bc_vectors.yIu_ux3 - yIv_uy = bc_vectors.yIv_uy3 - yIu_vx = bc_vectors.yIu_vx3 - yIv_vy = bc_vectors.yIv_vy3 - else - (; Cux, Cuy, Cvx, Cvy) = operators - (; Au_ux, Au_uy, Av_vx, Av_vy) = operators - (; Iu_ux, Iv_uy, Iu_vx, Iv_vy) = operators - (; yAu_ux, yAu_uy, yAv_vx, yAv_vy) = bc_vectors - (; yIu_ux, yIv_uy, yIu_vx, yIv_vy) = bc_vectors - end - - (; indu, indv) = grid - - uₕ = @view V[indu] - vₕ = @view V[indv] - - ϕu = @view ϕ[indu] - ϕv = @view ϕ[indv] - - # Convection components - u_ux = Au_ux * uₕ + yAu_ux # u at ux - ū_ux = Iu_ux * ϕu + yIu_ux # ū at ux - ∂uū∂x = Cux * (u_ux .* ū_ux) - - u_uy = Au_uy * uₕ + yAu_uy # u at uy - v̄_uy = Iv_uy * ϕv + yIv_uy # ū at uy - ∂uv̄∂y = Cuy * (u_uy .* v̄_uy) - - v_vx = Av_vx * vₕ + yAv_vx # v at vx - ū_vx = Iu_vx * ϕu + yIu_vx # ū at vx - ∂vū∂x = Cvx * (v_vx .* ū_vx) - - v_vy = Av_vy * vₕ + yAv_vy # v at vy - v̄_vy = Iv_vy * ϕv + yIv_vy # ū at vy - ∂vv̄∂y = Cvy * (v_vy .* v̄_vy) - - cu = ∂uū∂x + ∂uv̄∂y - cv = ∂vū∂x + ∂vv̄∂y - - c = [cu; cv] - - if get_jacobian - ## Convective terms, u-component - C1 = Cux * Diagonal(ū_ux) - C2 = Cux * Diagonal(u_ux) * newton_factor - Conv_ux_11 = C1 * Au_ux .+ C2 * Iu_ux - - C1 = Cuy * Diagonal(v̄_uy) - C2 = Cuy * Diagonal(u_uy) * newton_factor - Conv_uy_11 = C1 * Au_uy - Conv_uy_12 = C2 * Iv_uy - - ## Convective terms, v-component - C1 = Cvx * Diagonal(ū_vx) - C2 = Cvx * Diagonal(v_vx) * newton_factor - Conv_vx_21 = C2 * Iu_vx - Conv_vx_22 = C1 * Av_vx - - C1 = Cvy * Diagonal(v̄_vy) - C2 = Cvy * Diagonal(v_vy) * newton_factor - Conv_vy_22 = C1 * Av_vy .+ C2 * Iv_vy - - ∇c = [ - (Conv_ux_11+Conv_uy_11) Conv_uy_12 - Conv_vx_21 (Conv_vx_22+Conv_vy_22) - ] - else - ∇c = nothing - end - - c, ∇c -end - -# 3D version -function convection_components( - ::Dimension{3}, - V, - ϕ, - setup; - bc_vectors, - get_jacobian = false, - newton_factor = false, - order4 = false, -) - order4 && error("order4 not implemented for 3D") - - (; grid, operators) = setup - (; indu, indv, indw) = grid - (; Cux, Cuy, Cuz, Cvx, Cvy, Cvz, Cwx, Cwy, Cwz) = operators - - (; Au_ux, Au_uy, Au_uz) = operators - (; Av_vx, Av_vy, Av_vz) = operators - (; Aw_wx, Aw_wy, Aw_wz) = operators - - (; Iu_ux, Iv_uy, Iw_uz) = operators - (; Iu_vx, Iv_vy, Iw_vz) = operators - (; Iu_wx, Iv_wy, Iw_wz) = operators - - (; yAu_ux, yAu_uy, yAu_uz) = bc_vectors - (; yAv_vx, yAv_vy, yAv_vz) = bc_vectors - (; yAw_wx, yAw_wy, yAw_wz) = bc_vectors - - (; yIu_ux, yIv_uy, yIw_uz) = bc_vectors - (; yIu_vx, yIv_vy, yIw_vz) = bc_vectors - (; yIu_wx, yIv_wy, yIw_wz) = bc_vectors - - uₕ = @view V[indu] - vₕ = @view V[indv] - wₕ = @view V[indw] - - ϕu = @view ϕ[indu] - ϕv = @view ϕ[indv] - ϕw = @view ϕ[indw] - - u_ux = Au_ux * uₕ + yAu_ux # u at ux - ū_ux = Iu_ux * ϕu + yIu_ux # ū at ux - ∂uū∂x = Cux * (u_ux .* ū_ux) - - u_uy = Au_uy * uₕ + yAu_uy # u at uy - v̄_uy = Iv_uy * ϕv + yIv_uy # v̄ at uy - ∂uv̄∂y = Cuy * (u_uy .* v̄_uy) - - u_uz = Au_uz * uₕ + yAu_uz # u at uz - w̄_uz = Iw_uz * ϕw + yIw_uz # ū at uz - ∂uw̄∂z = Cuz * (u_uz .* w̄_uz) - - v_vx = Av_vx * vₕ + yAv_vx # v at vx - ū_vx = Iu_vx * ϕu + yIu_vx # ū at vx - ∂vū∂x = Cvx * (v_vx .* ū_vx) - - v_vy = Av_vy * vₕ + yAv_vy # v at vy - v̄_vy = Iv_vy * ϕv + yIv_vy # v̄ at vy - ∂vv̄∂y = Cvy * (v_vy .* v̄_vy) - - v_vz = Av_vz * vₕ + yAv_vz # v at vz - w̄_vz = Iw_vz * ϕw + yIw_vz # w̄ at vz - ∂vw̄∂z = Cvz * (v_vz .* w̄_vz) - - w_wx = Aw_wx * wₕ + yAw_wx # w at wx - ū_wx = Iu_wx * ϕu + yIu_wx # ū at wx - ∂wū∂x = Cwx * (w_wx .* ū_wx) - - w_wy = Aw_wy * wₕ + yAw_wy # w at wy - v̄_wy = Iv_wy * ϕv + yIv_wy # v̄ at wy - ∂wv̄∂y = Cwy * (w_wy .* v̄_wy) - - w_wz = Aw_wz * wₕ + yAw_wz # w at wz - w̄_wz = Iw_wz * ϕw + yIw_wz # w̄ at wz - ∂ww̄∂z = Cwz * (w_wz .* w̄_wz) - - cu = @. ∂uū∂x + ∂uv̄∂y + ∂uw̄∂z - cv = @. ∂vū∂x + ∂vv̄∂y + ∂vw̄∂z - cw = @. ∂wū∂x + ∂wv̄∂y + ∂ww̄∂z - - c = [cu; cv; cw] - - if get_jacobian - ## Convective terms, u-component - C1 = Cux * Diagonal(ū_ux) - C2 = Cux * Diagonal(u_ux) * newton_factor - Conv_ux_11 = C1 * Au_ux .+ C2 * Iu_ux - - C1 = Cuy * Diagonal(v̄_uy) - C2 = Cuy * Diagonal(u_uy) * newton_factor - Conv_uy_11 = C1 * Au_uy - Conv_uy_12 = C2 * Iv_uy - - C1 = Cuz * Diagonal(w̄_uz) - C2 = Cuz * Diagonal(u_uz) * newton_factor - Conv_uz_11 = C1 * Au_uz - Conv_uz_13 = C2 * Iw_uz - - ## Convective terms, v-component - C1 = Cvx * Diagonal(ū_vx) - C2 = Cvx * Diagonal(v_vx) * newton_factor - Conv_vx_21 = C2 * Iu_vx - Conv_vx_22 = C1 * Av_vx - - C1 = Cvy * Diagonal(v̄_vy) - C2 = Cvy * Diagonal(v_vy) * newton_factor - Conv_vy_22 = C1 * Av_vy .+ C2 * Iv_vy - - C1 = Cvz * Diagonal(w̄_vz) - C2 = Cvz * Diagonal(v_vz) * newton_factor - Conv_vz_23 = C2 * Iw_vz - Conv_vz_22 = C1 * Av_vz - - ## Convective terms, w-component - C1 = Cwx * Diagonal(ū_wx) - C2 = Cwx * Diagonal(w_wx) * newton_factor - Conv_wx_31 = C2 * Iu_wx - Conv_wx_33 = C1 * Aw_wx - - C1 = Cwy * Diagonal(v̄_wy) - C2 = Cwy * Diagonal(w_wy) * newton_factor - Conv_wy_32 = C2 * Iv_wy - Conv_wy_33 = C1 * Aw_wy - - C1 = Cwz * Diagonal(w̄_wz) - C2 = Cwz * Diagonal(w_wz) * newton_factor - Conv_wz_33 = C1 * Aw_wz .+ C2 * Iw_wz - - ## Jacobian - ∇c = [ - (Conv_ux_11+Conv_uy_11+Conv_uz_11) Conv_uy_12 Conv_uz_13 - Conv_vx_21 (Conv_vx_22+Conv_vy_22+Conv_vz_22) Conv_vz_23 - Conv_wx_31 Conv_wy_32 (Conv_wx_33+Conv_wy_33+Conv_wz_33) - ] - else - ∇c = nothing - end - - c, ∇c -end - -""" - convection_components!( - c, ∇c, V, ϕ, setup, cache; - bc_vectors, - get_jacobian = false, - newton_factor = false, - order4 = false, - ) - -Compute convection components. - -Mutating/non-allocating/in-place version. - -See also [`convection_components`](@ref). -""" -function convection_components! end - -convection_components!(c, ∇c, V, ϕ, setup, cache; kwargs...) = - convection_components!(setup.grid.dimension, c, ∇c, V, ϕ, setup, cache; kwargs...) - -# 2D version -function convection_components!( - ::Dimension{2}, - c, - ∇c, - V, - ϕ, - setup, - cache; - bc_vectors, - get_jacobian = false, - newton_factor = false, - order4 = false, -) - (; grid, operators) = setup - - if order4 - Cux = operators.Cux3 - Cuy = operators.Cuy3 - Cvx = operators.Cvx3 - Cvy = operators.Cvy3 - - Au_ux = operators.Au_ux3 - Au_uy = operators.Au_uy3 - Av_vx = operators.Av_vx3 - Av_vy = operators.Av_vy3 - - Iu_ux = operators.Iu_ux3 - Iv_uy = operators.Iv_uy3 - Iu_vx = operators.Iu_vx3 - Iv_vy = operators.Iv_vy3 - - yAu_ux = bc_vectors.yAu_ux3 - yAu_uy = bc_vectors.yAu_uy3 - yAv_vx = bc_vectors.yAv_vx3 - yAv_vy = bc_vectors.yAv_vy3 - - yIu_ux = bc_vectors.yIu_ux3 - yIv_uy = bc_vectors.yIv_uy3 - yIu_vx = bc_vectors.yIu_vx3 - yIv_vy = bc_vectors.yIv_vy3 - else - (; Cux, Cuy, Cvx, Cvy) = operators - (; Au_ux, Au_uy, Av_vx, Av_vy) = operators - (; Iu_ux, Iv_uy, Iu_vx, Iv_vy) = operators - (; yAu_ux, yAu_uy, yAv_vx, yAv_vy) = bc_vectors - (; yIu_ux, yIv_uy, yIu_vx, yIv_vy) = bc_vectors - end - - (; indu, indv) = grid - - (; u_ux, ū_ux, uū_ux, u_uy, v̄_uy, uv̄_uy) = cache - (; v_vx, ū_vx, vū_vx, v_vy, v̄_vy, vv̄_vy) = cache - - (; ∂uū∂x, ∂uv̄∂y, ∂vū∂x, ∂vv̄∂y) = cache - (; Conv_ux_11, Conv_uy_11, Conv_uy_12, Conv_vx_21, Conv_vx_22, Conv_vy_22) = cache - - cu = @view c[indu] - cv = @view c[indv] - - uₕ = @view V[indu] - vₕ = @view V[indv] - - ϕu = @view ϕ[indu] - ϕv = @view ϕ[indv] - - # Convection components - if order4 - # TODO: preallocated arrays for order4 - u_ux = Au_ux * uₕ + yAu_ux # U at ux - ū_ux = Iu_ux * ϕu + yIu_ux # Ū at ux - # ∂uū∂x = Cux * (u_ux .* ū_ux) - uū_ux = @. u_ux = u_ux * ū_ux - - u_uy = Au_uy * uₕ + yAu_uy # U at uy - v̄_uy = Iv_uy * ϕv + yIv_uy # Ū at uy - # ∂uv̄∂y = Cuy * (u_uy .* v̄_uy) - uv̄_uy = @. u_uy = u_uy * v̄_uy - - v_vx = Av_vx * vₕ + yAv_vx # V at vx - ū_vx = Iu_vx * ϕu + yIu_vx # Ū at vx - ∂vū∂x = Cvx * (v_vx .* ū_vx) - vū_vx = @. v_vx = v_vx * ū_vx - - v_vy = Av_vy * vₕ + yAv_vy # V at vy - v̄_vy = Iv_vy * ϕv + yIv_vy # Ū at vy - # ∂vv̄∂y = Cvy * (v_vy .* v̄_vy) - vv̄_vy = @. v_vy = v_vy * v̄_vy - else - # Fill preallocated arrays - mul!(u_ux, Au_ux, uₕ) - mul!(ū_ux, Iu_ux, ϕu) - mul!(u_uy, Au_uy, uₕ) - mul!(v̄_uy, Iv_uy, ϕv) - mul!(v_vx, Av_vx, vₕ) - mul!(ū_vx, Iu_vx, ϕu) - mul!(v_vy, Av_vy, vₕ) - mul!(v̄_vy, Iv_vy, ϕv) - - u_ux .+= yAu_ux - ū_ux .+= yIu_ux - @. uū_ux = u_ux * ū_ux - - u_uy .+= yAu_uy - v̄_uy .+= yIv_uy - @. uv̄_uy = u_uy * v̄_uy - - v_vx .+= yAv_vx - ū_vx .+= yIu_vx - @. vū_vx = v_vx * ū_vx - - v_vy .+= yAv_vy - v̄_vy .+= yIv_vy - @. vv̄_vy = v_vy * v̄_vy - end - - mul!(∂uū∂x, Cux, uū_ux) - mul!(∂uv̄∂y, Cuy, uv̄_uy) - mul!(∂vū∂x, Cvx, vū_vx) - mul!(∂vv̄∂y, Cvy, vv̄_vy) - - @. cu = ∂uū∂x + ∂uv̄∂y - @. cv = ∂vū∂x + ∂vv̄∂y - - if get_jacobian - ## Convective terms, u-component - C1 = Cux * Diagonal(ū_ux) - C2 = Cux * Diagonal(u_ux) * newton_factor - Conv_ux_11 .= C1 * Au_ux .+ C2 * Iu_ux - - C1 = Cuy * Diagonal(v̄_uy) - C2 = Cuy * Diagonal(u_uy) * newton_factor - Conv_uy_11 .= C1 * Au_uy - Conv_uy_12 .= C2 * Iv_uy - - ## Convective terms, v-component - C1 = Cvx * Diagonal(ū_vx) - C2 = Cvx * Diagonal(v_vx) * newton_factor - Conv_vx_21 .= C2 * Iu_vx - Conv_vx_22 .= C1 * Av_vx - - C1 = Cvy * Diagonal(v̄_vy) - C2 = Cvy * Diagonal(v_vy) * newton_factor - Conv_vy_22 .= C1 * Av_vy .+ C2 * Iv_vy - - ∇c[indu, indu] = Conv_ux_11 + Conv_uy_11 - ∇c[indu, indv] = Conv_uy_12 - ∇c[indv, indu] = Conv_vx_21 - ∇c[indv, indv] = Conv_vx_22 + Conv_vy_22 - end - - c, ∇c -end - -# 3D version -function convection_components!( - ::Dimension{3}, - c, - ∇c, - V, - ϕ, - setup, - cache; - bc_vectors, - get_jacobian = false, - newton_factor = false, - order4 = false, -) - order4 && error("order4 not implemented for 3D") - - (; grid, operators) = setup - (; indu, indv, indw) = grid - (; Cux, Cuy, Cuz, Cvx, Cvy, Cvz, Cwx, Cwy, Cwz) = operators - - (; Au_ux, Au_uy, Au_uz) = operators - (; Av_vx, Av_vy, Av_vz) = operators - (; Aw_wx, Aw_wy, Aw_wz) = operators - - (; Iu_ux, Iv_uy, Iw_uz) = operators - (; Iu_vx, Iv_vy, Iw_vz) = operators - (; Iu_wx, Iv_wy, Iw_wz) = operators - - (; yAu_ux, yAu_uy, yAu_uz) = bc_vectors - (; yAv_vx, yAv_vy, yAv_vz) = bc_vectors - (; yAw_wx, yAw_wy, yAw_wz) = bc_vectors - - (; yIu_ux, yIv_uy, yIw_uz) = bc_vectors - (; yIu_vx, yIv_vy, yIw_vz) = bc_vectors - (; yIu_wx, yIv_wy, yIw_wz) = bc_vectors - - (; u_ux, ū_ux, uū_ux, u_uy, v̄_uy, uv̄_uy, u_uz, w̄_uz, uw̄_uz) = cache - (; v_vx, ū_vx, vū_vx, v_vy, v̄_vy, vv̄_vy, v_vz, w̄_vz, vw̄_vz) = cache - (; w_wx, ū_wx, wū_wx, w_wy, v̄_wy, wv̄_wy, w_wz, w̄_wz, ww̄_wz) = cache - (; ∂uū∂x, ∂uv̄∂y, ∂uw̄∂z, ∂vū∂x, ∂vv̄∂y, ∂vw̄∂z, ∂wū∂x, ∂wv̄∂y, ∂ww̄∂z) = cache - (; Conv_ux_11, Conv_uy_11, Conv_uz_11, Conv_uy_12, Conv_uz_13) = cache - (; Conv_vx_21, Conv_vx_22, Conv_vy_22, Conv_vz_22, Conv_vz_23) = cache - (; Conv_wx_31, Conv_wy_32, Conv_wx_33, Conv_wy_33, Conv_wz_33) = cache - - cu = @view c[indu] - cv = @view c[indv] - cw = @view c[indw] - - uₕ = @view V[indu] - vₕ = @view V[indv] - wₕ = @view V[indw] - - ϕu = @view ϕ[indu] - ϕv = @view ϕ[indv] - ϕw = @view ϕ[indw] - - # Convection components - mul!(u_ux, Au_ux, uₕ) - mul!(ū_ux, Iu_ux, ϕu) - mul!(u_uy, Au_uy, uₕ) - mul!(v̄_uy, Iv_uy, ϕv) - mul!(u_uz, Au_uz, uₕ) - mul!(w̄_uz, Iw_uz, ϕw) - - mul!(v_vx, Av_vx, vₕ) - mul!(ū_vx, Iu_vx, ϕu) - mul!(v_vy, Av_vy, vₕ) - mul!(v̄_vy, Iv_vy, ϕv) - mul!(v_vz, Av_vz, vₕ) - mul!(w̄_vz, Iw_vz, ϕw) - - mul!(w_wx, Aw_wx, wₕ) - mul!(ū_wx, Iu_wx, ϕu) - mul!(w_wy, Aw_wy, wₕ) - mul!(v̄_wy, Iv_wy, ϕv) - mul!(w_wz, Aw_wz, wₕ) - mul!(w̄_wz, Iw_wz, ϕw) - - u_ux .+= yAu_ux - ū_ux .+= yIu_ux - @. uū_ux = u_ux * ū_ux - - u_uy .+= yAu_uy - v̄_uy .+= yIv_uy - @. uv̄_uy = u_uy * v̄_uy - - u_uz .+= yAu_uz - w̄_uz .+= yIw_uz - @. uw̄_uz = u_uz * w̄_uz - - v_vx .+= yAv_vx - ū_vx .+= yIu_vx - @. vū_vx = v_vx * ū_vx - - v_vy .+= yAv_vy - v̄_vy .+= yIv_vy - @. vv̄_vy = v_vy * v̄_vy - - v_vz .+= yAv_vz - w̄_vz .+= yIw_vz - @. vw̄_vz = v_vz * w̄_vz - - w_wx .+= yAw_wx - ū_wx .+= yIu_wx - @. wū_wx = w_wx * ū_wx - - w_wy .+= yAw_wy - v̄_wy .+= yIv_wy - @. wv̄_wy = w_wy * v̄_wy - - w_wz .+= yAw_wz - w̄_wz .+= yIw_wz - @. ww̄_wz = w_wz * w̄_wz - - mul!(∂uū∂x, Cux, uū_ux) - mul!(∂uv̄∂y, Cuy, uv̄_uy) - mul!(∂uw̄∂z, Cuz, uw̄_uz) - - mul!(∂vū∂x, Cvx, vū_vx) - mul!(∂vv̄∂y, Cvy, vv̄_vy) - mul!(∂vw̄∂z, Cvz, vw̄_vz) - - mul!(∂wū∂x, Cwx, wū_wx) - mul!(∂wv̄∂y, Cwy, wv̄_wy) - mul!(∂ww̄∂z, Cwz, ww̄_wz) - - @. cu = ∂uū∂x + ∂uv̄∂y + ∂uw̄∂z - @. cv = ∂vū∂x + ∂vv̄∂y + ∂vw̄∂z - @. cw = ∂wū∂x + ∂wv̄∂y + ∂ww̄∂z - - if get_jacobian - ## Convective terms, u-component - C1 = Cux * Diagonal(ū_ux) - C2 = Cux * Diagonal(u_ux) * newton_factor - Conv_ux_11 .= C1 * Au_ux .+ C2 * Iu_ux - - C1 = Cuy * Diagonal(v̄_uy) - C2 = Cuy * Diagonal(u_uy) * newton_factor - Conv_uy_11 .= C1 * Au_uy - Conv_uy_12 .= C2 * Iv_uy - - C1 = Cuz * Diagonal(w̄_uz) - C2 = Cuz * Diagonal(u_uz) * newton_factor - Conv_uz_11 .= C1 * Au_uz - Conv_uz_13 .= C2 * Iw_uz - - ## Convective terms, v-component - C1 = Cvx * Diagonal(ū_vx) - C2 = Cvx * Diagonal(v_vx) * newton_factor - Conv_vx_21 .= C2 * Iu_vx - Conv_vx_22 .= C1 * Av_vx - - C1 = Cvy * Diagonal(v̄_vy) - C2 = Cvy * Diagonal(v_vy) * newton_factor - Conv_vy_22 .= C1 * Av_vy .+ C2 * Iv_vy - - C1 = Cvz * Diagonal(w̄_vz) - C2 = Cvz * Diagonal(v_vz) * newton_factor - Conv_vz_23 .= C2 * Iw_vz - Conv_vz_22 .= C1 * Av_vz - - ## Convective terms, w-component - C1 = Cwx * Diagonal(ū_wx) - C2 = Cwx * Diagonal(w_wx) * newton_factor - Conv_wx_31 .= C2 * Iu_wx - Conv_wx_33 .= C1 * Aw_wx - - C1 = Cwy * Diagonal(v̄_wy) - C2 = Cwy * Diagonal(w_wy) * newton_factor - Conv_wy_32 .= C2 * Iv_wy - Conv_wy_33 .= C1 * Aw_wy - - C1 = Cwz * Diagonal(w̄_wz) - C2 = Cwz * Diagonal(w_wz) * newton_factor - Conv_wz_33 .= C1 * Aw_wz .+ C2 * Iw_wz - - ## Jacobian - ∇c[indu, indu] = Conv_ux_11 + Conv_uy_11 + Conv_uz_11 - ∇c[indu, indv] = Conv_uy_12 - ∇c[indu, indw] = Conv_uz_13 - ∇c[indv, indu] = Conv_vx_21 - ∇c[indv, indv] = Conv_vx_22 + Conv_vy_22 + Conv_vz_22 - ∇c[indv, indw] = Conv_vz_23 - ∇c[indw, indu] = Conv_wx_31 - ∇c[indw, indv] = Conv_wy_32 - ∇c[indw, indw] = Conv_wx_33 + Conv_wy_33 + Conv_wz_33 - end - - c, ∇c -end diff --git a/src/momentum/diffusion.jl b/src/momentum/diffusion.jl deleted file mode 100644 index afbf95adc..000000000 --- a/src/momentum/diffusion.jl +++ /dev/null @@ -1,235 +0,0 @@ -""" - diffusion!(model, V, setup; bc_vectors, get_jacobian = false) - -Evaluate diffusive terms `d` and optionally Jacobian `∇d = ∂d/∂V` using viscosity model `model`. - -Non-mutating/allocating/out-of-place version. - -See also [`diffusion!`](@ref). -""" -function diffusion end - -diffusion(m, V, setup; kwargs...) = diffusion(setup.grid.dimension, m, V, setup; kwargs...) - -function diffusion(::LaminarModel, V, setup; bc_vectors, get_jacobian = false) - (; Re) = setup - (; Diff) = setup.operators - (; yDiff) = bc_vectors - - d = 1 ./ Re .* (Diff * V .+ yDiff) - - if get_jacobian - ∇d = 1 / Re * Diff - else - ∇d = nothing - end - - d, ∇d -end - -# 2D version -function diffusion( - ::Dimension{2}, - model::Union{QRModel,SmagorinskyModel,MixingLengthModel}, - V, - setup; - bc_vectors, - get_jacobian = false, -) - (; Re) = setup - (; indu, indv) = setup.grid - (; Dux, Duy, Dvx, Dvy) = setup.operators - (; Su_ux, Su_uy, Su_vx, Sv_vx, Sv_vy, Sv_uy) = setup.operators - (; Aν_ux, Aν_uy, Aν_vx, Aν_vy) = setup.operators - (; yAν_ux, yAν_uy, yAν_vx, yAν_vy) = bc_vectors - - # Get components of strain tensor and its magnitude; - # The magnitude S_abs is evaluated at pressure points - S11, S12, S21, S22, S_abs, S_abs_u, S_abs_v = - strain_tensor(V, setup; bc_vectors, get_jacobian) - - # Turbulent viscosity at all pressure points - ν_t = turbulent_viscosity(model, setup, S_abs) - - # To compute the diffusion, we need ν_t at ux, uy, vx and vy locations - # This means we have to reverse the process of `strain_tensor`: go - # from pressure points back to the ux, uy, vx, vy locations - ν_t_ux = Aν_ux * ν_t + yAν_ux - ν_t_uy = Aν_uy * ν_t + yAν_uy - ν_t_vx = Aν_vx * ν_t + yAν_vx - ν_t_vy = Aν_vy * ν_t + yAν_vy - - # Now the total diffusive terms (laminar + turbulent) is as follows - # Note that the factor 2 is because - # Tau = 2*(ν+ν_t)*S(u), with S(u) = 1/2*(∇u + (∇u)^T) - - # Molecular viscosity - ν = 1 / Re - - du = Dux * (2 .* (ν .+ ν_t_ux) .* S11[:]) .+ Duy * (2 .* (ν .+ ν_t_uy) .* S12[:]) - dv = Dvx * (2 .* (ν .+ ν_t_vx) .* S21[:]) .+ Dvy * (2 .* (ν .+ ν_t_vy) .* S22[:]) - - if get_jacobian - # Freeze ν_t, i.e. we skip the derivative of ν_t wrt V in the Jacobian - Jacu1 = - Dux * 2 * spdiagm(ν .+ ν_t_ux) * Su_ux + - Duy * 2 * spdiagm(ν .+ ν_t_uy) * 1 / 2 * Su_uy - Jacu2 = Duy * 2 * spdiagm(ν .+ ν_t_uy) * 1 / 2 * Sv_uy - Jacv1 = Dvx * 2 * spdiagm(ν .+ ν_t_vx) * 1 / 2 * Su_vx - Jacv2 = - Dvx * 2 * spdiagm(ν .+ ν_t_vx) * 1 / 2 * Sv_vx + - Dvy * 2 * spdiagm(ν .+ ν_t_vy) * Sv_vy - Jacu = [Jacu1 Jacu2] - Jacv = [Jacv1 Jacv2] - - K = turbulent_K(model, setup) - - tmpu1 = - 2 * Dux * spdiagm(S11) * Aν_ux * S_abs_u + - 2 * Duy * spdiagm(S12) * Aν_uy * S_abs_u - tmpu2 = 2 * Duy * spdiagm(S12) * Aν_uy * S_abs_v - tmpv1 = 2 * Dvx * spdiagm(S21) * Aν_vx * S_abs_u - tmpv2 = - 2 * Dvx * spdiagm(S21) * Aν_vx * S_abs_v + - 2 * Dvy * spdiagm(S22) * Aν_vy * S_abs_v - Jacu += K * [tmpu1 tmpu2] - Jacv += K * [tmpv1 tmpv2] - - ∇d = [Jacu; Jacv] - else - ∇d = nothing - end - - d = [du; dv] - - d, ∇d -end - -# 3D version -function diffusion( - ::Dimension{3}, - model::Union{QRModel,SmagorinskyModel,MixingLengthModel}, - V, - setup; - bc_vectors, - get_jacobian = false, -) - error("Not implemented") -end - -""" - diffusion!(model, d, ∇d, V, setup; bc_vectors, get_jacobian = false) - -Evaluate diffusive terms `d` and optionally Jacobian `∇d = ∂d/∂V` using viscosity model `model`. -""" -function diffusion! end - -diffusion!(m, d, ∇d, V, setup; kwargs...) = - diffusion!(setup.grid.dimension, m, d, ∇d, V, setup; kwargs...) - -function diffusion!(::LaminarModel, d, ∇d, V, setup; bc_vectors, get_jacobian = false) - (; Re) = setup - (; Diff) = setup.operators - (; yDiff) = bc_vectors - - # d = 1 ./ Re .* (Diff * V .+ yDiff) - mul!(d, Diff, V) - @. d = 1 / Re * (d + yDiff) - - get_jacobian && (@. ∇d = 1 / Re * Diff) - - d, ∇d -end - -# 2D version -function diffusion!( - ::Dimension{2}, - model::Union{QRModel,SmagorinskyModel,MixingLengthModel}, - d, - ∇d, - V, - setup; - bc_vectors, - get_jacobian = false, -) - (; Re) = setup - (; indu, indv) = setup.grid - (; Dux, Duy, Dvx, Dvy) = setup.operators - (; Su_ux, Su_uy, Su_vx, Sv_vx, Sv_vy, Sv_uy) = setup.operators - (; Aν_ux, Aν_uy, Aν_vx, Aν_vy) = setup.operators - (; yAν_ux, yAν_uy, yAν_vx, yAν_vy) = bc_vectors - - du = @view d[indu] - dv = @view d[indv] - - # Get components of strain tensor and its magnitude; - # The magnitude S_abs is evaluated at pressure points - S11, S12, S21, S22, S_abs, S_abs_u, S_abs_v = - strain_tensor(V, setup; bc_vectors, get_jacobian) - - # Turbulent viscosity at all pressure points - ν_t = turbulent_viscosity(model, setup, S_abs) - - # To compute the diffusion, we need ν_t at ux, uy, vx and vy locations - # This means we have to reverse the process of `strain_tensor`: go - # from pressure points back to the ux, uy, vx, vy locations - ν_t_ux = Aν_ux * ν_t + yAν_ux - ν_t_uy = Aν_uy * ν_t + yAν_uy - ν_t_vx = Aν_vx * ν_t + yAν_vx - ν_t_vy = Aν_vy * ν_t + yAν_vy - - # Now the total diffusive terms (laminar + turbulent) is as follows - # Note that the factor 2 is because - # Tau = 2*(ν+ν_t)*S(u), with S(u) = 1/2*(∇u + (∇u)^T) - - # Molecular viscosity - ν = 1 / Re - - du .= Dux * (2 .* (ν .+ ν_t_ux) .* S11) .+ Duy * (2 .* (ν .+ ν_t_uy) .* S12) - dv .= Dvx * (2 .* (ν .+ ν_t_vx) .* S21) .+ Dvy * (2 .* (ν .+ ν_t_vy) .* S22) - - if get_jacobian - # Freeze ν_t, i.e. we skip the derivative of ν_t wrt V in the Jacobian - Jacu1 = - Dux * 2 * spdiagm(ν .+ ν_t_ux) * Su_ux + - Duy * 2 * spdiagm(ν .+ ν_t_uy) * 1 // 2 * Su_uy - Jacu2 = Duy * 2 * spdiagm(ν .+ ν_t_uy) * 1 // 2 * Sv_uy - Jacv1 = Dvx * 2 * spdiagm(ν .+ ν_t_vx) * 1 // 2 * Su_vx - Jacv2 = - Dvx * 2 * spdiagm(ν .+ ν_t_vx) * 1 // 2 * Sv_vx + - Dvy * 2 * spdiagm(ν .+ ν_t_vy) * Sv_vy - Jacu = [Jacu1 Jacu2] - Jacv = [Jacv1 Jacv2] - - K = turbulent_K(model, setup) - - tmpu1 = - 2 * Dux * spdiagm(S11) * Aν_ux * S_abs_u + - 2 * Duy * spdiagm(S12) * Aν_uy * S_abs_u - tmpu2 = 2 * Duy * spdiagm(S12) * Aν_uy * S_abs_v - tmpv1 = 2 * Dvx * spdiagm(S21) * Aν_vx * S_abs_u - tmpv2 = - 2 * Dvx * spdiagm(S21) * Aν_vx * S_abs_v + - 2 * Dvy * spdiagm(S22) * Aν_vy * S_abs_v - Jacu += K * [tmpu1 tmpu2] - Jacv += K * [tmpv1 tmpv2] - - ∇d .= [Jacu; Jacv] - end - - d, ∇d -end - -# 3D version -function diffusion!( - ::Dimension{3}, - model::Union{QRModel,SmagorinskyModel,MixingLengthModel}, - d, - ∇d, - V, - setup; - bc_vectors, - get_jacobian = false, -) - error("Not implemented") -end diff --git a/src/momentum/momentum.jl b/src/momentum/momentum.jl deleted file mode 100644 index ff8d67dde..000000000 --- a/src/momentum/momentum.jl +++ /dev/null @@ -1,177 +0,0 @@ -""" - momentum( - V, ϕ, p, t, setup; - bc_vectors = nothing, - get_jacobian = false, - nopressure = false, - newton_factor = false, - ) - -Calculate RHS of momentum equations and, optionally, Jacobian with respect to velocity field. - - - `V`: velocity field - - `ϕ`: convected field: e.g. ``\\frac{\\partial (\\phi_x V)}{\\partial x} + \\frac{\\partial (\\phi_y V)}{\\partial y}``; usually `ϕ = V` (so `ϕx = u`, `ϕy = v`) - - `p`: pressure - - `bc_vectors`: boundary condition vectors `y` - - `get_jacobian`: return `∇F = ∂F/∂V` - - `nopressure`: exclude pressure gradient; in this case input argument `p` is not used - - `newton_factor` - -Non-mutating/allocating/out-of-place version. - -See also [`momentum!`](@ref). -""" -function momentum( - V, - ϕ, - p, - t, - setup; - bc_vectors = nothing, - get_jacobian = false, - nopressure = false, - newton_factor = false, -) - (; - viscosity_model, - convection_model, - force, - closure_model, - boundary_conditions, - operators, - ) = setup - (; G) = operators - - # Unsteady BC - if isnothing(bc_vectors) || boundary_conditions.bc_unsteady - bc_vectors = get_bc_vectors(setup, t) - end - (; y_p) = bc_vectors - - # Convection - c, ∇c = - convection(convection_model, V, ϕ, setup; bc_vectors, get_jacobian, newton_factor) - - # Diffusion - d, ∇d = diffusion(viscosity_model, V, setup; bc_vectors, get_jacobian) - - # Body force - b = force - - # Residual in Finite Volume form, excluding the pressure contribution - F = @. -c + d + b - - # Closure model - isnothing(closure_model) || (F += closure_model(V)) - - # Nopressure = false is the most common situation, in which we return the entire - # right-hand side vector - if !nopressure - F = F .- (G * p .+ y_p) - end - - if get_jacobian - # Jacobian requested - # We return only the Jacobian with respect to V (not p) - ∇F = @. -∇c + ∇d - @assert isnothing(closure_model) "Jacobian of closure model currently not implemented" - else - ∇F = nothing - end - - F, ∇F -end - -""" - momentum!(F, ∇F, V, ϕ, p, t, setup, cache; get_jacobian = false, nopressure = false) - -Calculate rhs of momentum equations and, optionally, Jacobian with respect to velocity field. - - - `V`: velocity field - - `ϕ`: convected field: e.g. ``\\frac{\\partial (\\phi_x V)}{\\partial x} + \\frac{\\partial (\\phi_y V)}{\\partial y}``; usually `ϕ = V` (so `ϕx = u`, `ϕy = v`) - - `p`: pressure - - `bc_vectors`: boundary condition vectors `y` - - `get_jacobian`: return `∇F = ∂F/∂V` - - `nopressure`: exclude pressure gradient; in this case input argument `p` is not used - - `newton_factor` - -Mutating/non-allocating/in-place version. - -See also [`momentum`](@ref). -""" -function momentum!( - F, - ∇F, - V, - ϕ, - p, - t, - setup, - cache; - bc_vectors = nothing, - get_jacobian = false, - nopressure = false, - newton_factor = false, -) - (; - viscosity_model, - convection_model, - force, - closure_model, - boundary_conditions, - operators, - ) = setup - (; G) = setup.operators - - # Unsteady BC - if isnothing(bc_vectors) || boundary_conditions.bc_unsteady - bc_vectors = get_bc_vectors(setup, t) - end - (; y_p) = bc_vectors - - # Store intermediate results in temporary variables - (; c, ∇c, d, ∇d, b, Gp) = cache - - # Convection - convection!( - convection_model, - c, - ∇c, - V, - ϕ, - setup, - cache; - bc_vectors, - get_jacobian, - newton_factor, - ) - - # Diffusion - diffusion!(viscosity_model, d, ∇d, V, setup; bc_vectors, get_jacobian) - - # Body force - b = force - - # Residual in Finite Volume form, excluding the pressure contribution - @. F = -c + d + b - - # Closure model - isnothing(closure_model) || (F .+= closure_model(V)) - - # Nopressure = false is the most common situation, in which we return the entire - # right-hand side vector - if !nopressure - mul!(Gp, G, p) - Gp .+= y_p - @. F -= Gp - end - - if get_jacobian - # Jacobian requested - # We return only the Jacobian with respect to V (not p) - @. ∇F = -∇c + ∇d - @assert isnothing(closure_model) "Jacobian of closure model currently not implemented" - end - - F, ∇F -end diff --git a/src/momentum/momentumcache.jl b/src/momentum/momentumcache.jl deleted file mode 100644 index 441d51a07..000000000 --- a/src/momentum/momentumcache.jl +++ /dev/null @@ -1,242 +0,0 @@ -""" - MomentumCache(setup) - -Preallocation structure for terms in the momentum equations. -""" -MomentumCache(setup, V, p) = MomentumCache(setup.grid.dimension, setup, V, p) - -function MomentumCache(::Dimension{2}, setup, V, p) - (; grid, operators) = setup - (; Nu, Nv, NV) = grid - (; Iu_ux, Iv_uy, Iu_vx, Iv_vy) = operators - - T = eltype(V) - - Gp = similar(V) - c = similar(V) - c2 = similar(V) - c3 = similar(V) - d = similar(V) - b = similar(V) - - ∇c = spzeros(T, NV, NV) - ∇c2 = spzeros(T, NV, NV) - ∇c3 = spzeros(T, NV, NV) - ∇d = spzeros(T, NV, NV) - ∇b = spzeros(T, NV, NV) - - u_ux = similar(V, size(Iu_ux, 1)) - ū_ux = similar(V, size(Iu_ux, 1)) - uū_ux = similar(V, size(Iu_ux, 1)) - - u_uy = similar(V, size(Iv_uy, 1)) - v̄_uy = similar(V, size(Iv_uy, 1)) - uv̄_uy = similar(V, size(Iv_uy, 1)) - - v_vx = similar(V, size(Iu_vx, 1)) - ū_vx = similar(V, size(Iu_vx, 1)) - vū_vx = similar(V, size(Iu_vx, 1)) - - v_vy = similar(V, size(Iv_vy, 1)) - v̄_vy = similar(V, size(Iv_vy, 1)) - vv̄_vy = similar(V, size(Iv_vy, 1)) - - ∂uū∂x = similar(V, Nu) - ∂uv̄∂y = similar(V, Nu) - ∂vū∂x = similar(V, Nv) - ∂vv̄∂y = similar(V, Nv) - - Conv_ux_11 = spzeros(T, Nu, Nu) - Conv_uy_11 = spzeros(T, Nu, Nu) - Conv_uy_12 = spzeros(T, Nu, Nv) - - Conv_vx_21 = spzeros(T, Nv, Nu) - Conv_vx_22 = spzeros(T, Nv, Nv) - Conv_vy_22 = spzeros(T, Nv, Nv) - - (; - Gp, - c, - c2, - c3, - d, - b, - ∇c, - ∇c2, - ∇c3, - ∇d, - ∇b, - u_ux, - ū_ux, - uū_ux, - u_uy, - v̄_uy, - uv̄_uy, - v_vx, - ū_vx, - vū_vx, - v_vy, - v̄_vy, - vv̄_vy, - ∂uū∂x, - ∂uv̄∂y, - ∂vū∂x, - ∂vv̄∂y, - Conv_ux_11, - Conv_uy_11, - Conv_uy_12, - Conv_vx_21, - Conv_vx_22, - Conv_vy_22, - ) -end - -function MomentumCache(::Dimension{3}, setup, V, p) - (; grid, operators) = setup - (; Nu, Nv, Nw, NV) = grid - (; Iu_ux, Iv_uy, Iw_uz, Iu_vx, Iv_vy, Iw_vz, Iu_wx, Iv_wy, Iw_wz) = operators - - T = eltype(Iu_ux) - - Gp = similar(V, NV) - c = similar(V, NV) - c2 = similar(V, NV) - c3 = similar(V, NV) - d = similar(V, NV) - b = similar(V, NV) - - ∇c = spzeros(T, NV, NV) - ∇c2 = spzeros(T, NV, NV) - ∇c3 = spzeros(T, NV, NV) - ∇d = spzeros(T, NV, NV) - ∇b = spzeros(T, NV, NV) - - u_ux = similar(V, size(Iu_ux, 1)) - ū_ux = similar(V, size(Iu_ux, 1)) - uū_ux = similar(V, size(Iu_ux, 1)) - - u_uy = similar(V, size(Iv_uy, 1)) - v̄_uy = similar(V, size(Iv_uy, 1)) - uv̄_uy = similar(V, size(Iv_uy, 1)) - - u_uz = similar(V, size(Iw_uz, 1)) - w̄_uz = similar(V, size(Iw_uz, 1)) - uw̄_uz = similar(V, size(Iw_uz, 1)) - - v_vx = similar(V, size(Iu_vx, 1)) - ū_vx = similar(V, size(Iu_vx, 1)) - vū_vx = similar(V, size(Iu_vx, 1)) - - v_vy = similar(V, size(Iv_vy, 1)) - v̄_vy = similar(V, size(Iv_vy, 1)) - vv̄_vy = similar(V, size(Iv_vy, 1)) - - v_vz = similar(V, size(Iw_vz, 1)) - w̄_vz = similar(V, size(Iw_vz, 1)) - vw̄_vz = similar(V, size(Iw_vz, 1)) - - w_wx = similar(V, size(Iu_wx, 1)) - ū_wx = similar(V, size(Iu_wx, 1)) - wū_wx = similar(V, size(Iu_wx, 1)) - - w_wy = similar(V, size(Iv_wy, 1)) - v̄_wy = similar(V, size(Iv_wy, 1)) - wv̄_wy = similar(V, size(Iv_wy, 1)) - - w_wz = similar(V, size(Iw_wz, 1)) - w̄_wz = similar(V, size(Iw_wz, 1)) - ww̄_wz = similar(V, size(Iw_wz, 1)) - - ∂uū∂x = similar(V, Nu) - ∂uv̄∂y = similar(V, Nu) - ∂uw̄∂z = similar(V, Nu) - ∂vū∂x = similar(V, Nv) - ∂vv̄∂y = similar(V, Nv) - ∂vw̄∂z = similar(V, Nv) - ∂wū∂x = similar(V, Nw) - ∂wv̄∂y = similar(V, Nw) - ∂ww̄∂z = similar(V, Nw) - - Conv_ux_11 = spzeros(T, Nu, Nu) - Conv_uy_11 = spzeros(T, Nu, Nu) - Conv_uz_11 = spzeros(T, Nu, Nu) - Conv_uy_12 = spzeros(T, Nu, Nv) - Conv_uz_13 = spzeros(T, Nu, Nw) - - Conv_vx_21 = spzeros(T, Nv, Nu) - Conv_vx_22 = spzeros(T, Nv, Nv) - Conv_vy_22 = spzeros(T, Nv, Nv) - Conv_vz_22 = spzeros(T, Nv, Nv) - Conv_vz_23 = spzeros(T, Nv, Nw) - - Conv_wx_31 = spzeros(T, Nw, Nu) - Conv_wy_32 = spzeros(T, Nw, Nv) - Conv_wx_33 = spzeros(T, Nw, Nw) - Conv_wy_33 = spzeros(T, Nw, Nw) - Conv_wz_33 = spzeros(T, Nw, Nw) - - (; - Gp, - c, - c2, - c3, - d, - b, - ∇c, - ∇c2, - ∇c3, - ∇d, - ∇b, - u_ux, - ū_ux, - uū_ux, - u_uy, - v̄_uy, - uv̄_uy, - u_uz, - w̄_uz, - uw̄_uz, - v_vx, - ū_vx, - vū_vx, - v_vy, - v̄_vy, - vv̄_vy, - v_vz, - w̄_vz, - vw̄_vz, - w_wx, - ū_wx, - wū_wx, - w_wy, - v̄_wy, - wv̄_wy, - w_wz, - w̄_wz, - ww̄_wz, - ∂uū∂x, - ∂uv̄∂y, - ∂uw̄∂z, - ∂vū∂x, - ∂vv̄∂y, - ∂vw̄∂z, - ∂wū∂x, - ∂wv̄∂y, - ∂ww̄∂z, - Conv_ux_11, - Conv_uy_11, - Conv_uz_11, - Conv_uy_12, - Conv_uz_13, - Conv_vx_21, - Conv_vx_22, - Conv_vy_22, - Conv_vz_22, - Conv_vz_23, - Conv_wx_31, - Conv_wy_32, - Conv_wx_33, - Conv_wy_33, - Conv_wz_33, - ) -end diff --git a/src/momentum/strain_tensor.jl b/src/momentum/strain_tensor.jl deleted file mode 100644 index ddf462182..000000000 --- a/src/momentum/strain_tensor.jl +++ /dev/null @@ -1,163 +0,0 @@ -""" - strain_tensor(V, setup; bc_vectors, get_jacobian = false, get_S_abs = false) - -Evaluate rate of strain tensor `S(V)` and its magnitude. -""" -function strain_tensor end - -strain_tensor(V, setup; kwargs...) = - strain_tensor(setup.grid.dimension, V, setup; kwargs...) - -# 2D version -function strain_tensor( - ::Dimension{2}, - V, - setup; - bc_vectors, - get_jacobian = false, - get_S_abs = false, -) - (; grid, operators, boundary_conditions) = setup - (; Nx, Ny, Nu, Nv, Np, indu, indv) = grid - (; Nux_in, Nuy_in, Nvx_in, Nvy_in) = grid - (; x, y, xp, yp) = grid - (; Su_ux, Su_uy, Su_vx, Sv_vx, Sv_vy, Sv_uy) = operators - (; Cux_k, Cuy_k, Cvx_k, Cvy_k, Auy_k, Avx_k) = operators - (; ySu_ux, ySu_uy, ySu_vx, ySv_vx, ySv_vy, ySv_uy) = bc_vectors - (; yCux_k, yCuy_k, yCvx_k, yCvy_k, yAuy_k, yAvx_k) = bc_vectors - - T = eltype(V) - - uₕ = @view V[indu] - vₕ = @view V[indv] - - # These four components are approximated by - S11 = Su_ux * uₕ + ySu_ux - S12 = 1 // 2 .* (Su_uy * uₕ .+ ySu_uy .+ Sv_uy * vₕ .+ ySv_uy) - S21 = 1 // 2 .* (Su_vx * uₕ .+ ySu_vx .+ Sv_vx * vₕ .+ ySv_vx) - S22 = Sv_vy * vₕ + ySv_vy - - # Note: S11 and S22 at xp, yp locations (pressure locations) - # S12, S21 at vorticity locations (corners of pressure cells, (x, y)) - - # Option 1: get each S11, S12, S21, S22 at 4 locations (ux locations, at uy locations, at vx - # Locations and at vy locations); this gives 16 S fields. determine S_abs at each of these - # Locations, giving 4 S_abs fields, that can be used in computing - # Dux*(S_abs_ux .* (Su_ux*uₕ+ySu_ux)) etc. - - # Option 2: interpolate S11, S12, S21, S22 to pressure locations - # Determine S_abs at pressure location - # Then interpolate to ux, uy, vx, vy locations - - # We will use option 2; - # Within option 2, we can decide to interpolate S12, S21 etc (option 2a), or we can use - # Directly the operators that map from velocity field to the S locations, - # As used for example in ke_production (option 2b). - - if get_S_abs - # Option 2b - if boundary_conditions.u.x == (:periodic, :periodic) - # "cut-off" the double points in case of periodic BC - # For periodic boundary conditions S11(Npx+1, :) = S11(1, :) - # So S11 has size (Npx+1)*Npy; the last row are "ghost" points equal to the - # First points. we have S11 at positions ([xp[1] - 1/2*(hx[1]+hx[end]); xp], yp) - S11_p = reshape(S11, Nux_in + 1, Nuy_in) - S11_p = S11_p(2:(Nux_in+1), :) # B - - # S12 is defined on the corners: size Nux_in*(Nuy_in+1), positions (xin, y) - # Get S12 and S21 at all corner points - S12_temp = zeros(T, Nx + 1, Ny + 1) - S12_temp[1:Nx, :] = reshape(S12, Nx, Ny + 1) - S12_temp[Nx+1, :] = S12_temp[1, :] - elseif boundary_conditions.u.x[1] == :dirichlet && - boundary_conditions.u.x[2] == :pressure - S11_p = reshape(S11, Nux_in + 1, Nuy_in) - S11_p = S11_p[1:Nux_in, :] # Cut off last point - - # S12 is defined on the corners: size Nux_in*(Nuy_in+1), positions (xin, y) - # Get S12 and S21 at all corner points - S12_temp = zeros(T, Nx + 1, Ny + 1) - S12_temp[2:(Nx+1), :] = reshape(S12, Nx, Ny + 1) - S12_temp[1, :] = S12_temp[2, :] # Copy from x[2] to x[1]; one could do this more accurately in principle by using the BC - else - error("BC not implemented in strain_tensor.jl") - end - - if boundary_conditions.v.y[1] == :periodic && - boundary_conditions.v.y[2] == :periodic - # Similarly, S22(:, Npy+1) = S22(:, 1). positions (xp, [yp;yp[1]]) - S22_p = reshape(S22, Nvx_in, Nvy_in + 1) - S22_p = S22_p(:, 2:(Nvy_in+1)) # Why not 1:Nvy_in? - - # Similarly S21 is size (Nux_in+1)*Nuy_in, positions (x, yin) - S21_temp = zeros(T, Nx + 1, Ny + 1) - S21_temp[:, 1:Ny] = reshape(S21, Nx + 1, Ny) - S21_temp[:, Ny+1] = S21_temp[:, 1] - elseif boundary_conditions.v.y[1] == :pressure && - boundary_conditions.v.y[2] == :pressure - S22_p = reshape(S22, Nvx_in, Nvy_in + 1) - S22_p = S22_p(:, 2:Nvy_in) - - # This is nicely defined on all corners - S21_temp = reshape(S21, Nx + 1, Ny + 1) - else - error("BC not implemented in strain_tensor.jl") - end - - # Now interpolate S12 and S21 to pressure points - # S11 and S22 have already been trimmed down to this grid - - # S21 and S12 should be equal! - # FIXME: Find interpolation syntax - error("Interpolation not implemented") - # S12_p = interp2(y', x, S12_temp, yp', xp) - # S21_p = interp2(y', x, S21_temp, yp', xp) - - ## Invariants - q = @. 1 // 2 * (S11_p[:]^2 + S12_p[:]^2 + S21_p[:]^2 + S22_p[:]^2) - - # Absolute value of strain tensor - # With S as defined above, i.e. 1/2*(grad u + grad u^T) - # S_abs = sqrt(2*tr(S^2)) = sqrt(4*q) - S_abs = sqrt(4q) - else - # Option 2a - S11_p = Cux_k * uₕ + yCux_k - S12_p = - 1 // 2 .* ( - Cuy_k * (Auy_k * uₕ + yAuy_k) .+ yCuy_k .+ Cvx_k * (Avx_k * vₕ + yAvx_k) .+ - yCvx_k - ) - S21_p = S12_p - S22_p = Cvy_k * vₕ + yCvy_k - - S_abs = @. sqrt(2 * S11_p^2 + 2 * S22_p^2 + 2 * S12_p^2 + 2 * S21_p^2) - - # Jacobian of S_abs wrt u and v - if get_jacobian - ϵ = 100 * eps(T) - Sabs_inv = spdiagm(1 ./ (2 .* S_abs .+ ϵ)) - Jacu = - Sabs_inv * (4 * spdiagm(S11_p) * Cux_k + 4 * spdiagm(S12_p) * Cuy_k * Auy_k) - Jacv = - Sabs_inv * (4 * spdiagm(S12_p) * Cvx_k * Avx_k + 4 * spdiagm(S22_p) * Cvy_k) - else - Jacu = spzeros(T, Np, Nu) - Jacv = spzeros(T, Np, Nv) - end - end - - S11, S12, S21, S22, S_abs, Jacu, Jacv -end - -# 3D version -function strain_tensor( - ::Dimension{3}, - V, - setup; - bc_vectors, - get_jacobian = false, - get_S_abs = false, -) - error("Not implemented (3D)") -end diff --git a/src/momentum/turbulent_K.jl b/src/momentum/turbulent_K.jl deleted file mode 100644 index 394ff7573..000000000 --- a/src/momentum/turbulent_K.jl +++ /dev/null @@ -1,17 +0,0 @@ -""" - turbulent_K(model, setup) - -Compute the constant part of the turbulent viscosity. -""" -function turbulent_K end - -turbulent_K(model::SmagorinskyModel, setup) = model.C_s^2 * max_size(setup.grid)^2 - -function turbulent_K(::QRModel, setup) - (; α_reg) = setup.operators - Δ = max_size(setup.grid) - C_d = Δ^2 / 8 - C_d * 1 // 2 * (1 - α_reg / C_d)^2 -end - -turbulent_K(model::MixingLengthModel, setup) = model.lm^2 diff --git a/src/momentum/turbulent_viscosity.jl b/src/momentum/turbulent_viscosity.jl deleted file mode 100644 index 81e980729..000000000 --- a/src/momentum/turbulent_viscosity.jl +++ /dev/null @@ -1,8 +0,0 @@ -""" - turbulent_viscosity(model, setup, S_abs) - -Compute turbulent viscosity based on `S_abs`. -""" -function turbulent_viscosity end - -turbulent_viscosity(model, setup, S_abs) = turbulent_K(model, setup) * S_abs diff --git a/src/operators/operator_averaging.jl b/src/operators/operator_averaging.jl deleted file mode 100644 index d39f02c2f..000000000 --- a/src/operators/operator_averaging.jl +++ /dev/null @@ -1,411 +0,0 @@ -""" - operator_averaging(grid, boundary_conditions) - -Construct averaging operators. -""" -operator_averaging(grid, boundary_conditions) = - operator_averaging(grid.dimension, grid, boundary_conditions) - -# 2D version -function operator_averaging(::Dimension{2}, grid, boundary_conditions) - (; Nux_in, Nux_b, Nux_t, Nuy_in, Nuy_b, Nuy_t) = grid - (; Nvx_in, Nvx_b, Nvx_t, Nvy_in, Nvy_b, Nvy_t) = grid - (; hx, hy) = grid - (; order4) = grid - - T = eltype(hx) - - # Averaging weight: - weight = T(1 / 2) - - ## Averaging operators, u-component - - ## Au_ux: evaluate u at ux location - diag1 = fill(weight, Nux_t - 1) - A1D = spdiagm(Nux_t - 1, Nux_t, 0 => diag1, 1 => diag1) - - # Boundary conditions - Au_ux_bc = bc_general( - Nux_t, - Nux_in, - Nux_b, - boundary_conditions.u.x[1], - boundary_conditions.u.x[2], - hx[1], - hx[end], - ) - - # Extend to 2D - Au_ux = I(Nuy_in) ⊗ (A1D * Au_ux_bc.B1D) - Au_ux_bc = (; Au_ux_bc..., Bbc = I(Nuy_in) ⊗ (A1D * Au_ux_bc.Btemp)) - - ## Au_uy: evaluate u at uy location - diag1 = fill(weight, Nuy_t - 1) - A1D = spdiagm(Nuy_t - 1, Nuy_t, 0 => diag1, 1 => diag1) - - # Boundary conditions - Au_uy_bc = bc_general_stag( - Nuy_t, - Nuy_in, - Nuy_b, - boundary_conditions.u.y[1], - boundary_conditions.u.y[2], - hy[1], - hy[end], - ) - - # Extend to 2D - Au_uy = (A1D * Au_uy_bc.B1D) ⊗ I(Nux_in) - Au_uy_bc = (; Au_uy_bc..., Bbc = (A1D * Au_uy_bc.Btemp) ⊗ I(Nux_in)) - - ## Averaging operators, v-component - - ## Av_vx: evaluate v at vx location - diag1 = fill(weight, Nvx_t - 1) - A1D = spdiagm(Nvx_t - 1, Nvx_t, 0 => diag1, 1 => diag1) - - # Boundary conditions - Av_vx_bc = bc_general_stag( - Nvx_t, - Nvx_in, - Nvx_b, - boundary_conditions.v.x[1], - boundary_conditions.v.x[2], - hx[1], - hx[end], - ) - - # Extend to 2D - Av_vx = I(Nvy_in) ⊗ (A1D * Av_vx_bc.B1D) - Av_vx_bc = (; Av_vx_bc..., Bbc = I(Nvy_in) ⊗ (A1D * Av_vx_bc.Btemp)) - - ## Av_vy: evaluate v at vy location - diag1 = fill(weight, Nvy_t - 1) - A1D = spdiagm(Nvy_t - 1, Nvy_t, 0 => diag1, 1 => diag1) - - # Boundary conditions - Av_vy_bc = bc_general( - Nvy_t, - Nvy_in, - Nvy_b, - boundary_conditions.v.y[1], - boundary_conditions.v.y[2], - hy[1], - hy[end], - ) - - # Extend to 2D - Av_vy = (A1D * Av_vy_bc.B1D) ⊗ I(Nvx_in) - Av_vy_bc = (; Av_vy_bc..., Bbc = (A1D * Av_vy_bc.Btemp) ⊗ I(Nvx_in)) - - ## Fourth order - if order4 - ## Au_ux: evaluate u at ux location - diag1 = fill(weight, Nux_in + 3) - A1D3 = spdiagm(Nux_in + 3, Nux_t + 4, 0 => diag1, 3 => diag1) - - # Boundary conditions - Au_ux_bc3 = bc_av3( - Nux_t + 4, - Nux_in, - Nux_t + 4 - Nux_in, - boundary_conditions.u.x[1], - boundary_conditions.u.x[2], - hx[1], - hx[end], - ) - - # Extend to 2D - Au_ux3 = I(Nuy_in) ⊗ (A1D3 * Au_ux_bc3.B1D) - Au_ux_bc3 = (; Au_ux_bc3..., Bbc = I(Nuy_in) ⊗ (A1D3 * Au_ux_bc3.Btemp)) - - ## Au_uy: evaluate u at uy location - diag1 = fill(weight, Nuy_in + 3) - A1D3 = spdiagm(Nuy_in + 3, Nuy_t + 4, 0 => diag1, 3 => diag1) - - # Boundary conditions - Au_uy_bc3 = bc_av_stag3( - Nuy_t + 4, - Nuy_in, - Nuy_t + 4 - Nuy_in, - boundary_conditions.u.y[1], - boundary_conditions.u.y[2], - hy[1], - hy[end], - ) - - # Extend to 2D - Au_uy3 = (A1D3 * Au_uy_bc3.B1D) ⊗ I(Nux_in) - Au_uy_bc3 = (; Au_uy_bc3..., Bbc = (A1D3 * Au_uy_bc3.Btemp) ⊗ I(Nux_in)) - - ## Av_vx: evaluate v at vx location - diag1 = fill(weight, Nvx_in + 3) - A1D3 = spdiagm(Nvx_in + 3, Nvx_t + 4, 0 => diag1, 3 => diag1) - - # Boundary conditions - Av_vx_bc3 = bc_av_stag3( - Nvx_t + 4, - Nvx_in, - Nvx_t + 4 - Nvx_in, - boundary_conditions.v.x[1], - boundary_conditions.v.x[2], - hx[1], - hx[end], - ) - - # Extend to 2D - Av_vx3 = I(Nvy_in) ⊗ (A1D3 * Av_vx_bc3.B1D) - Av_vx_bc3 = (; Av_vx_bc3..., Bbc = I(Nvy_in) ⊗ (A1D3 * Av_vx_bc3.Btemp)) - - ## Av_vy: evaluate v at vy location - diag1 = fill(weight, Nvy_in + 3) - A1D3 = spdiagm(Nvy_in + 3, Nvy_t + 4, 0 => diag1, 3 => diag1) - - # Boundary conditions - Av_vy_bc3 = bc_av3( - Nvy_t + 4, - Nvy_in, - Nvy_t + 4 - Nvy_in, - boundary_conditions.v.y[1], - boundary_conditions.v.y[2], - hy[1], - hy[end], - ) - - # Extend to 2D - Av_vy3 = (A1D3 * Av_vy_bc3.B1D) ⊗ I(Nvx_in) - Av_vy_bc3 = (; Av_vy_bc3..., Bbc = (A1D3 * Av_vy_bc3.Btemp) ⊗ I(Nvx_in)) - end - - ## Group operators - operators = (; Au_ux, Au_uy, Av_vx, Av_vy, Au_ux_bc, Au_uy_bc, Av_vx_bc, Av_vy_bc) - - if order4 - operators = (; - operators..., - Au_ux3, - Au_uy3, - Av_vx3, - Av_vy3, - Au_ux_bc3, - Au_uy_bc3, - Av_vx_bc3, - Av_vy_bc3, - ) - end - - operators -end - -# 3D version -function operator_averaging(::Dimension{3}, grid, boundary_conditions) - (; Nux_in, Nux_b, Nux_t, Nuy_in, Nuy_b, Nuy_t, Nuz_in, Nuz_b, Nuz_t) = grid - (; Nvx_in, Nvx_b, Nvx_t, Nvy_in, Nvy_b, Nvy_t, Nvz_in, Nvz_b, Nvz_t) = grid - (; Nwx_in, Nwx_b, Nwx_t, Nwy_in, Nwy_b, Nwy_t, Nwz_in, Nwz_b, Nwz_t) = grid - (; hx, hy, hz) = grid - - T = eltype(hx) - - # Averaging weight: - weight = T(1 / 2) - - ## Averaging operators, u-component - - ## Au_ux: evaluate u at ux location - diag1 = fill(weight, Nux_t - 1) - A1D = spdiagm(Nux_t - 1, Nux_t, 0 => diag1, 1 => diag1) - - # Boundary conditions - Au_ux_bc = bc_general( - Nux_t, - Nux_in, - Nux_b, - boundary_conditions.u.x[1], - boundary_conditions.u.x[2], - hx[1], - hx[end], - ) - - # Extend to 3D - Au_ux = I(Nuz_in) ⊗ I(Nuy_in) ⊗ (A1D * Au_ux_bc.B1D) - Au_ux_bc = (; Au_ux_bc..., Bbc = I(Nuz_in) ⊗ I(Nuy_in) ⊗ (A1D * Au_ux_bc.Btemp)) - - ## Au_uy: evaluate u at uy location - diag1 = fill(weight, Nuy_t - 1) - A1D = spdiagm(Nuy_t - 1, Nuy_t, 0 => diag1, 1 => diag1) - - # Boundary conditions - Au_uy_bc = bc_general_stag( - Nuy_t, - Nuy_in, - Nuy_b, - boundary_conditions.u.y[1], - boundary_conditions.u.y[2], - hy[1], - hy[end], - ) - - # Extend to 3D - Au_uy = I(Nuz_in) ⊗ (A1D * Au_uy_bc.B1D) ⊗ I(Nux_in) - Au_uy_bc = (; Au_uy_bc..., Bbc = I(Nuz_in) ⊗ (A1D * Au_uy_bc.Btemp) ⊗ I(Nux_in)) - - ## Au_uz: evaluate u at uz location - diag1 = fill(weight, Nuz_t - 1) - A1D = spdiagm(Nuz_t - 1, Nuz_t, 0 => diag1, 1 => diag1) - - # Boundary conditions - Au_uz_bc = bc_general_stag( - Nuz_t, - Nuz_in, - Nuz_b, - boundary_conditions.u.z[1], - boundary_conditions.u.z[2], - hz[1], - hz[end], - ) - - # Extend to 3D - Au_uz = (A1D * Au_uz_bc.B1D) ⊗ I(Nuy_in) ⊗ I(Nux_in) - Au_uz_bc = (; Au_uz_bc..., Bbc = (A1D * Au_uz_bc.Btemp) ⊗ I(Nuy_in) ⊗ I(Nux_in)) - - ## Averaging operators, v-component - - ## Av_vx: evaluate v at vx location - diag1 = fill(weight, Nvx_t - 1) - A1D = spdiagm(Nvx_t - 1, Nvx_t, 0 => diag1, 1 => diag1) - - # Boundary conditions - Av_vx_bc = bc_general_stag( - Nvx_t, - Nvx_in, - Nvx_b, - boundary_conditions.v.x[1], - boundary_conditions.v.x[2], - hx[1], - hx[end], - ) - - # Extend to 3D - Av_vx = I(Nvz_in) ⊗ I(Nvy_in) ⊗ (A1D * Av_vx_bc.B1D) - Av_vx_bc = (; Av_vx_bc..., Bbc = I(Nvz_in) ⊗ I(Nvy_in) ⊗ (A1D * Av_vx_bc.Btemp)) - - ## Av_vy: evaluate v at vy location - diag1 = fill(weight, Nvy_t - 1) - A1D = spdiagm(Nvy_t - 1, Nvy_t, 0 => diag1, 1 => diag1) - - # Boundary conditions - Av_vy_bc = bc_general( - Nvy_t, - Nvy_in, - Nvy_b, - boundary_conditions.v.y[1], - boundary_conditions.v.y[2], - hy[1], - hy[end], - ) - - # Extend to 3D - Av_vy = I(Nvz_in) ⊗ (A1D * Av_vy_bc.B1D) ⊗ I(Nvx_in) - Av_vy_bc = (; Av_vy_bc..., Bbc = I(Nvz_in) ⊗ (A1D * Av_vy_bc.Btemp) ⊗ I(Nvx_in)) - - ## Av_vz: evalvate v at vz location - diag1 = fill(weight, Nvz_t - 1) - A1D = spdiagm(Nvz_t - 1, Nvz_t, 0 => diag1, 1 => diag1) - - # Boundary conditions - Av_vz_bc = bc_general_stag( - Nvz_t, - Nvz_in, - Nvz_b, - boundary_conditions.v.z[1], - boundary_conditions.v.z[2], - hz[1], - hz[end], - ) - - # Extend to 3D - Av_vz = (A1D * Av_vz_bc.B1D) ⊗ I(Nvy_in) ⊗ I(Nvx_in) - Av_vz_bc = (; Av_vz_bc..., Bbc = (A1D * Av_vz_bc.Btemp) ⊗ I(Nvy_in) ⊗ I(Nvx_in)) - - ## Averaging operators, w-component - - ## Aw_wx: evaluate w at wx location - diag1 = fill(weight, Nwx_t - 1) - A1D = spdiagm(Nwx_t - 1, Nwx_t, 0 => diag1, 1 => diag1) - - # Boundary conditions - Aw_wx_bc = bc_general_stag( - Nwx_t, - Nwx_in, - Nwx_b, - boundary_conditions.w.x[1], - boundary_conditions.w.x[2], - hx[1], - hx[end], - ) - - # Extend to 3D - Aw_wx = I(Nwz_in) ⊗ I(Nwy_in) ⊗ (A1D * Aw_wx_bc.B1D) - Aw_wx_bc = (; Aw_wx_bc..., Bbc = I(Nwz_in) ⊗ I(Nwy_in) ⊗ (A1D * Aw_wx_bc.Btemp)) - - ## Aw_wy: evaluate w at wy location - diag1 = fill(weight, Nwy_t - 1) - A1D = spdiagm(Nwy_t - 1, Nwy_t, 0 => diag1, 1 => diag1) - - # Boundary conditions - Aw_wy_bc = bc_general_stag( - Nwy_t, - Nwy_in, - Nwy_b, - boundary_conditions.w.y[1], - boundary_conditions.w.y[2], - hy[1], - hy[end], - ) - - # Extend to 3D - Aw_wy = I(Nwz_in) ⊗ (A1D * Aw_wy_bc.B1D) ⊗ I(Nwx_in) - Aw_wy_bc = (; Aw_wy_bc..., Bbc = I(Nwz_in) ⊗ (A1D * Aw_wy_bc.Btemp) ⊗ I(Nwx_in)) - - ## Aw_wz: evaluate w at wz location - diag1 = fill(weight, Nwz_t - 1) - A1D = spdiagm(Nwz_t - 1, Nwz_t, 0 => diag1, 1 => diag1) - - # Boundary conditions - Aw_wz_bc = bc_general( - Nwz_t, - Nwz_in, - Nwz_b, - boundary_conditions.w.z[1], - boundary_conditions.w.z[2], - hz[1], - hz[end], - ) - - # Extend to 3D - Aw_wz = (A1D * Aw_wz_bc.B1D) ⊗ I(Nwy_in) ⊗ I(Nwx_in) - Aw_wz_bc = - (; Aw_wz_bc..., Bbc = kron((A1D * Aw_wz_bc.Btemp), kron(I(Nwy_in), I(Nwx_in)))) - - ## Group operators - (; - Au_ux, - Au_uy, - Au_uz, - Av_vx, - Av_vy, - Av_vz, - Aw_wx, - Aw_wy, - Aw_wz, - Au_ux_bc, - Au_uy_bc, - Au_uz_bc, - Av_vx_bc, - Av_vy_bc, - Av_vz_bc, - Aw_wx_bc, - Aw_wy_bc, - Aw_wz_bc, - ) -end diff --git a/src/operators/operator_convection_diffusion.jl b/src/operators/operator_convection_diffusion.jl deleted file mode 100644 index 0fae2e851..000000000 --- a/src/operators/operator_convection_diffusion.jl +++ /dev/null @@ -1,1071 +0,0 @@ -""" - operator_convection_diffusion(grid, boundary_conditions) - -Construct convection and diffusion operators. -""" -operator_convection_diffusion(grid, boundary_conditions) = - operator_convection_diffusion(grid.dimension, grid, boundary_conditions) - -# 2D version -function operator_convection_diffusion(::Dimension{2}, grid, boundary_conditions) - (; Nx, Ny) = grid - (; Nux_in, Nux_b, Nux_t, Nuy_in, Nuy_b, Nuy_t) = grid - (; Nvx_in, Nvx_b, Nvx_t, Nvy_in, Nvy_b, Nvy_t) = grid - (; hx, hy, hxi, hyi, hxd, hyd) = grid - (; gxi, gyi, gxd, gyd) = grid - (; Buvy, Bvux) = grid - (; order4, α) = grid - - if order4 - (; hxi3, hyi3, gxi3, gyi3, hxd13, hxd3, hyd13, hyd3) = grid - (; gxd13, gxd3, gyd13, gyd3) = grid - (; Ωux, Ωuy, Ωvx, Ωvy) = grid - (; Ωux1, Ωux3, Ωuy1, Ωuy3, Ωvx1, Ωvx3, Ωvy1, Ωvy3) = grid - end - - T = eltype(hx) - - ## Convection (differencing) operator Cu - - # Calculates difference from pressure points to velocity points - diag1 = ones(T, Nux_t - 2) - D1D = spdiagm(Nux_t - 2, Nux_t - 1, 0 => -diag1, 1 => diag1) - Cux = I(Nuy_in) ⊗ D1D - if !order4 - Dux = Diagonal(hyi) ⊗ D1D - end - - # Calculates difference from corner points to velocity points - diag1 = ones(T, Nuy_t - 2) - D1D = spdiagm(Nuy_t - 2, Nuy_t - 1, 0 => -diag1, 1 => diag1) - Cuy = D1D ⊗ I(Nux_in) - if !order4 - Duy = D1D ⊗ Diagonal(gxi) - end - - # Cu = [Cux Cuy] - # Du = [Dux Duy] - - ## Convection (differencing) operator Cv - - # Calculates difference from pressure points to velocity points - diag1 = ones(T, Nvx_t - 2) - D1D = spdiagm(Nvx_t - 2, Nvx_t - 1, 0 => -diag1, 1 => diag1) - Cvx = I(Nvy_in) ⊗ D1D - if !order4 - Dvx = Diagonal(gyi) ⊗ D1D - end - - # Calculates difference from corner points to velocity points - diag1 = ones(T, Nvy_t - 2) - D1D = spdiagm(Nvy_t - 2, Nvy_t - 1, 0 => -diag1, 1 => diag1) - Cvy = D1D ⊗ I(Nvx_in) - if !order4 - Dvy = D1D ⊗ Diagonal(hxi) - end - - # Cv = [Cvx Cvy] - # Dv = [Dvx Dvy] - - if order4 - ## Fourth order operators - ## Convection (differencing) operator Cu - - # Calculates difference from pressure points to velocity points - diag1 = ones(T, Nux_t - 2) - D1D = spdiagm(Nux_t - 2, Nux_t + 1, 1 => -diag1, 2 => diag1) - Dux = Diagonal(hyi) ⊗ D1D - - # The "second order" Cux is unchanged - # The "second order" Dux changes, because we also use the "second - # Order" flux at "fourth order" ghost points (Dux should have the same - # Size as Dux3) - - # Calculates difference from pressure points to velocity points - diag1 = ones(T, Nux_t - 2) - D1D3 = spdiagm(Nux_t - 2, Nux_t + 1, 0 => -diag1, 3 => diag1) - Cux3 = I(Ny) ⊗ D1D3 - Dux3 = Diagonal(hyi3) ⊗ D1D3 - - # Calculates difference from corner points to velocity points - diag1 = ones(T, Nuy_t - 2) - D1D = spdiagm(Nuy_t - 2, Nuy_t + 1, 1 => -diag1, 2 => diag1) - Duy = D1D ⊗ Diagonal(gxi) - - # Calculates difference from corner points to velocity points - diag1 = ones(T, Nuy_t - 2) - D1D3 = spdiagm(Nuy_t - 2, Nuy_t + 1, 0 => -diag1, 3 => diag1) - - # Uncomment for new BC (functions/new) - if boundary_conditions.u.y[1] == :dirichlet - D1D3[1, 1] = 1 - D1D3[1, 2] = -2 - end - if boundary_conditions.u.y[2] == :dirichlet - D1D3[end, end-1] = 2 - D1D3[end, end] = -1 - end - Cuy3 = D1D3 ⊗ I(Nux_in) - Duy3 = D1D3 ⊗ Diagonal(gxi3) - - ## Convection (differencing) operator Cv - - # Calculates difference from pressure points to velocity points - diag1 = ones(T, Nvx_t - 2) - D1D = spdiagm(Nvx_t - 2, Nvx_t + 1, 1 => -diag1, 2 => diag1) - Dvx = Diagonal(gyi) ⊗ D1D - - # Calculates difference from pressure points to velocity points - diag1 = ones(T, Nvx_t - 2) - D1D3 = spdiagm(Nvx_t - 2, Nvx_t + 1, 0 => -diag1, 3 => diag1) - - # Uncomment for new BC (functions/new) - if boundary_conditions.v.x[1] == :dirichlet - D1D3[1, 1] = 1 - D1D3[1, 2] = -2 - end - if boundary_conditions.v.x[2] == :dirichlet - D1D3[end, end-1] = 2 - D1D3[end, end] = -1 - end - Cvx3 = I(Nvy_in) ⊗ D1D3 - Dvx3 = Diagonal(gyi3) ⊗ D1D3 - - # Calculates difference from corner points to velocity points - diag1 = ones(T, Nvy_t - 2) - D1D = spdiagm(Nvy_t - 2, Nvy_t + 1, 1 => -diag1, 2 => diag1) - Dvy = D1D ⊗ Diagonal(hxi) - - # Calculates difference from corner points to velocity points - diag1 = ones(T, Nvy_t - 2) - D1D3 = spdiagm(Nvy_t - 2, Nvy_t + 1, 0 => -diag1, 3 => diag1) - Cvy3 = D1D3 ⊗ I(Nvx_in) - Dvy3 = D1D3 ⊗ Diagonal(hxi3) - - ## Su_ux: evaluate ux - diag1 = 1 ./ hxd13 - S1D = spdiagm(Nux_in + 3, Nux_t + 4, 1 => -diag1, 2 => diag1) - - # Boundary conditions - Su_ux_bc = bc_diff3( - Nux_t + 4, - Nux_in, - Nux_t + 4 - Nux_in, - boundary_conditions.u.x[1], - boundary_conditions.u.x[2], - hx[1], - hx[end], - ) - - # Extend to 2D - Su_ux = Diagonal(Ωux1) * (I(Ny) ⊗ (S1D * Su_ux_bc.B1D)) - Su_ux_bc = (; Su_ux_bc..., Bbc = Diagonal(Ωux1) * (I(Ny) ⊗ (S1D * Su_ux_bc.Btemp))) - - diag1 = 1 ./ hxd3 - S1D3 = spdiagm(Nux_in + 3, Nux_t + 4, 0 => -diag1, 3 => diag1) - - # Boundary conditions - Su_ux_bc3 = bc_diff3( - Nux_t + 4, - Nux_in, - Nux_t + 4 - Nux_in, - boundary_conditions.u.x[1], - boundary_conditions.u.x[2], - hx[1], - hx[end], - ) - - # Extend to 2D - Su_ux3 = Diagonal(Ωux3) * (I(Nuy_in) ⊗ (S1D3 * Su_ux_bc3.B1D)) - Su_ux_bc3 = - (; Su_ux_bc3..., Bbc = Diagonal(Ωux3) * (I(Nuy_in) ⊗ (S1D3 * Su_ux_bc3.Btemp))) - - ## Su_uy: evaluate uy - diag1 = 1 ./ gyd13 - S1D = spdiagm(Nuy_in + 3, Nuy_t + 4, 1 => -diag1, 2 => diag1) - - # Boundary conditions - Su_uy_bc = bc_diff_stag3( - Nuy_t + 4, - Nuy_in, - Nuy_t + 4 - Nuy_in, - boundary_conditions.u.y[1], - boundary_conditions.u.y[2], - hy[1], - hy[end], - ) - - # Extend to 2D - Su_uy = Diagonal(Ωuy1) * ((S1D * Su_uy_bc.B1D) ⊗ I(Nux_in)) - Su_uy_bc = - (; Su_uy_bc..., Bbc = Diagonal(Ωuy1) * ((S1D * Su_uy_bc.Btemp) ⊗ I(Nux_in))) - - diag1 = 1 ./ gyd3 - S1D3 = spdiagm(Nuy_in + 3, Nuy_t + 4, 0 => -diag1, 3 => diag1) - - # Boundary conditions - Su_uy_bc3 = bc_diff_stag3( - Nuy_t + 4, - Nuy_in, - Nuy_t + 4 - Nuy_in, - boundary_conditions.u.y[1], - boundary_conditions.u.y[2], - hy[1], - hy[end], - ) - - # Extend to 2D - Su_uy3 = Diagonal(Ωuy3) * ((S1D3 * Su_uy_bc3.B1D) ⊗ I(Nux_in)) - Su_uy_bc3 = - (; Su_uy_bc3..., Bbc = Diagonal(Ωuy3) * ((S1D3 * Su_uy_bc3.Btemp) ⊗ I(Nux_in))) - - ## Sv_vx: evaluate vx - diag1 = 1 ./ gxd13 - S1D = spdiagm(Nvx_in + 3, Nvx_t + 4, 1 => -diag1, 2 => diag1) - - # Boundary conditions - Sv_vx_bc = bc_diff_stag3( - Nvx_t + 4, - Nvx_in, - Nvx_t + 4 - Nvx_in, - boundary_conditions.v.x[1], - boundary_conditions.v.x[2], - hx[1], - hx[end], - ) - - # Extend to 2D - Sv_vx = Diagonal(Ωvx1) * (I(Nvy_in) ⊗ (S1D * Sv_vx_bc.B1D)) - Sv_vx_bc = - (; Sv_vx_bc..., Bbc = Diagonal(Ωvx1) * (I(Nvy_in) ⊗ (S1D * Sv_vx_bc.Btemp))) - - diag1 = 1 ./ gxd3 - S1D3 = spdiagm(Nvx_in + 3, Nvx_t + 4, 0 => -diag1, 3 => diag1) - - # Boundary conditions - Sv_vx_bc3 = bc_diff_stag3( - Nvx_t + 4, - Nvx_in, - Nvx_t + 4 - Nvx_in, - boundary_conditions.v.x[1], - boundary_conditions.v.x[2], - hx[1], - hx[end], - ) - # Extend to 2D - Sv_vx3 = Diagonal(Ωvx3) * (I(Nvy_in) ⊗ (S1D3 * Sv_vx_bc3.B1D)) - Sv_vx_bc3 = - (; Sv_vx_bc3..., Bbc = Diagonal(Ωvx3) * (I(Nvy_in) ⊗ (S1D3 * Sv_vx_bc3.Btemp))) - - ## Sv_vy: evaluate vy - diag1 = 1 ./ hyd13 - S1D = spdiagm(Nvy_in + 3, Nvy_t + 4, 1 => -diag1, 2 => diag1) - - # Boundary conditions - Sv_vy_bc = bc_diff3( - Nvy_t + 4, - Nvy_in, - Nvy_t + 4 - Nvy_in, - boundary_conditions.v.y[1], - boundary_conditions.v.y[2], - hy[1], - hy[end], - ) - - # Extend to 2D - Sv_vy = Diagonal(Ωvy1) * ((S1D * Sv_vy_bc.B1D) ⊗ I(Nvx_in)) - Sv_vy_bc = - (; Sv_vy_bc..., Bbc = Diagonal(Ωvy1) * ((S1D * Sv_vy_bc.Btemp) ⊗ I(Nvx_in))) - - diag1 = 1 ./ hyd3 - S1D3 = spdiagm(Nvy_in + 3, Nvy_t + 4, 0 => -diag1, 3 => diag1) - - # Boundary conditions - Sv_vy_bc3 = bc_diff3( - Nvy_t + 4, - Nvy_in, - Nvy_t + 4 - Nvy_in, - boundary_conditions.v.y[1], - boundary_conditions.v.y[2], - hy[1], - hy[end], - ) - # Extend to 2D - Sv_vy3 = Diagonal(Ωvy3) * ((S1D3 * Sv_vy_bc3.B1D) ⊗ I(Nvx_in)) - Sv_vy_bc3 = - (; Sv_vy_bc3..., Bbc = Diagonal(Ωvy3) * ((S1D3 * Sv_vy_bc3.Btemp) ⊗ I(Nvx_in))) - else - ## Diffusion operator (stress tensor), u-component: similar to averaging, but with mesh sizes - - ## Su_ux: evaluate ux - diag1 = 1 ./ hxd - S1D = spdiagm(Nux_t - 1, Nux_t, 0 => -diag1, 1 => diag1) - - # Boundary conditions - Su_ux_bc = bc_general( - Nux_t, - Nux_in, - Nux_b, - boundary_conditions.u.x[1], - boundary_conditions.u.x[2], - hx[1], - hx[end], - ) - - # Extend to 2D - Su_ux = I(Ny) ⊗ (S1D * Su_ux_bc.B1D) - Su_ux_bc = (; Su_ux_bc..., Bbc = I(Ny) ⊗ (S1D * Su_ux_bc.Btemp)) - - ## Su_uy: evaluate uy - diag1 = 1 ./ gyd - S1D = spdiagm(Nuy_t - 1, Nuy_t, 0 => -diag1, 1 => diag1) - - # Boundary conditions - Su_uy_bc = bc_diff_stag( - Nuy_t, - Nuy_in, - Nuy_b, - boundary_conditions.u.y[1], - boundary_conditions.u.y[2], - hy[1], - hy[end], - ) - - # Extend to 2D - Su_uy = (S1D * Su_uy_bc.B1D) ⊗ I(Nux_in) - Su_uy_bc = (; Su_uy_bc..., Bbc = (S1D * Su_uy_bc.Btemp) ⊗ I(Nux_in)) - - ## Sv_uy: evaluate vx at uy; same as Iv_uy except for mesh sizes and -diag diag - diag1 = 1 ./ gxd - S1D = spdiagm(Nvx_t - 1, Nvx_t, 0 => -diag1, 1 => diag1) - - # The restriction is essentially 1D so it can be directly applied to I1D - S1D = Bvux * S1D - S2D = I(Nuy_t - 1) ⊗ S1D - - # Boundary conditions low/up - Nb = Nuy_in + 1 - Nvy_in - Sv_uy_bc_lu = bc_general( - Nuy_in + 1, - Nvy_in, - Nb, - boundary_conditions.v.y[1], - boundary_conditions.v.y[2], - hy[1], - hy[end], - ) - Sv_uy_bc_lu = (; Sv_uy_bc_lu..., B2D = Sv_uy_bc_lu.B1D ⊗ I(Nvx_in)) - Sv_uy_bc_lu = (; Sv_uy_bc_lu..., Bbc = Sv_uy_bc_lu.Btemp ⊗ I(Nvx_in)) - - # Boundary conditions left/right - Sv_uy_bc_lr = bc_general_stag( - Nvx_t, - Nvx_in, - Nvx_b, - boundary_conditions.v.x[1], - boundary_conditions.v.x[2], - hx[1], - hx[end], - ) - - # Take I2D into left/right operators for convenience - Sv_uy_bc_lr = (; Sv_uy_bc_lr..., B2D = S2D * (I(Nuy_t - 1) ⊗ Sv_uy_bc_lr.B1D)) - Sv_uy_bc_lr = (; Sv_uy_bc_lr..., Bbc = S2D * (I(Nuy_t - 1) ⊗ Sv_uy_bc_lr.Btemp)) - - # Resulting operator: - Sv_uy = Sv_uy_bc_lr.B2D * Sv_uy_bc_lu.B2D - - ## Diffusion operator (stress tensor), v-component: similar to averaging! - - ## Su_vx: evaluate uy at vx. Same as Iu_vx except for mesh sizes and -diag diag - diag1 = 1 ./ gyd - S1D = spdiagm(Nuy_t - 1, Nuy_t, 0 => -diag1, 1 => diag1) - S1D = Buvy * S1D - S2D = S1D ⊗ I(Nvx_t - 1) - - # Boundary conditions low/up - Su_vx_bc_lu = bc_general_stag( - Nuy_t, - Nuy_in, - Nuy_b, - boundary_conditions.u.y[1], - boundary_conditions.u.y[2], - hy[1], - hy[end], - ) - Su_vx_bc_lu = (; Su_vx_bc_lu..., B2D = S2D * (Su_vx_bc_lu.B1D ⊗ I(Nvx_t - 1))) - Su_vx_bc_lu = (; Su_vx_bc_lu..., Bbc = S2D * (Su_vx_bc_lu.Btemp ⊗ I(Nvx_t - 1))) - - # Boundary conditions left/right - Nb = Nvx_in + 1 - Nux_in - Su_vx_bc_lr = bc_general( - Nvx_in + 1, - Nux_in, - Nb, - boundary_conditions.u.x[1], - boundary_conditions.u.x[2], - hx[1], - hx[end], - ) - - Su_vx_bc_lr = (; Su_vx_bc_lr..., B2D = I(Nuy_in) ⊗ Su_vx_bc_lr.B1D) - Su_vx_bc_lr = (; Su_vx_bc_lr..., Bbc = I(Nuy_in) ⊗ Su_vx_bc_lr.Btemp) - - # Resulting operator: - Su_vx = Su_vx_bc_lu.B2D * Su_vx_bc_lr.B2D - - ## Sv_vx: evaluate vx - diag1 = 1 ./ gxd - S1D = spdiagm(Nvx_t - 1, Nvx_t, 0 => -diag1, 1 => diag1) - - # Boundary conditions - Sv_vx_bc = bc_diff_stag( - Nvx_t, - Nvx_in, - Nvx_b, - boundary_conditions.v.x[1], - boundary_conditions.v.x[2], - hx[1], - hx[end], - ) - - # Extend to 2D - Sv_vx = I(Nvy_in) ⊗ (S1D * Sv_vx_bc.B1D) - - Sv_vx_bc = (; Sv_vx_bc..., Bbc = I(Nvy_in) ⊗ (S1D * Sv_vx_bc.Btemp)) - - ## Sv_vy: evaluate vy - diag1 = 1 ./ hyd - S1D = spdiagm(Nvy_t - 1, Nvy_t, 0 => -diag1, 1 => diag1) - - # Boundary conditions - Sv_vy_bc = bc_general( - Nvy_t, - Nvy_in, - Nvy_b, - boundary_conditions.v.y[1], - boundary_conditions.v.y[2], - hy[1], - hy[end], - ) - - # Extend to 2D - Sv_vy = (S1D * Sv_vy_bc.B1D) ⊗ I(Nx) - Sv_vy_bc = (; Sv_vy_bc..., Bbc = (S1D * Sv_vy_bc.Btemp) ⊗ I(Nx)) - end - - ## Assemble operators - if order4 - Diffux_div = (α * Dux - Dux3) * Diagonal(1 ./ Ωux) - Diffuy_div = (α * Duy - Duy3) * Diagonal(1 ./ Ωuy) - Diffvx_div = (α * Dvx - Dvx3) * Diagonal(1 ./ Ωvx) - Diffvy_div = (α * Dvy - Dvy3) * Diagonal(1 ./ Ωvy) - Diffu = Diffux_div * (α * Su_ux - Su_ux3) + Diffuy_div * (α * Su_uy - Su_uy3) - Diffv = Diffvx_div * (α * Sv_vx - Sv_vx3) + Diffvy_div * (α * Sv_vy - Sv_vy3) - else - Diffu = Dux * Su_ux + Duy * Su_uy - Diffv = Dvx * Sv_vx + Dvy * Sv_vy - end - Diff = blockdiag(Diffu, Diffv) - - ## Group operators - operators = (; - Cux, - Cuy, - Cvx, - Cvy, - Su_ux, - Su_uy, - Sv_vx, - Sv_vy, - Su_ux_bc, - Su_uy_bc, - Sv_vx_bc, - Sv_vy_bc, - Dux, - Duy, - Dvx, - Dvy, - Diff, - Sv_uy, - Su_vx, - ) - - if order4 - operators = (; - operators..., - Cux3, - Cuy3, - Cvx3, - Cvy3, - Su_ux_bc3, - Su_uy_bc3, - Sv_vx_bc3, - Sv_vy_bc3, - Diffux_div, - Diffuy_div, - Diffvx_div, - Diffvy_div, - ) - else - operators = (; operators..., Su_vx_bc_lr, Su_vx_bc_lu, Sv_uy_bc_lr, Sv_uy_bc_lu) - end - - operators -end - -# 3D version -function operator_convection_diffusion(::Dimension{3}, grid, boundary_conditions) - (; Nx, Ny, Nz) = grid - (; Nux_in, Nux_b, Nux_t, Nuy_in, Nuy_b, Nuy_t, Nuz_in, Nuz_b, Nuz_t) = grid - (; Nvx_in, Nvx_b, Nvx_t, Nvy_in, Nvy_b, Nvy_t, Nvz_in, Nvz_b, Nvz_t) = grid - (; Nwx_in, Nwx_b, Nwx_t, Nwy_in, Nwy_b, Nwy_t, Nwz_in, Nwz_b, Nwz_t) = grid - (; hx, hy, hz, hxi, hyi, hzi, hxd, hyd, hzd) = grid - (; gxi, gyi, gzi, gxd, gyd, gzd) = grid - (; Bvux, Bwux, Buvy, Bwvy, Buwz, Bvwz) = grid - - T = eltype(hx) - - ## Convection (differencing) operator Cu - - # Calculates difference from pressure points to velocity points - diag1 = ones(T, Nux_t - 2) - M1D = spdiagm(Nux_t - 2, Nux_t - 1, 0 => -diag1, 1 => diag1) - Cux = I(Nz) ⊗ I(Nuy_in) ⊗ M1D - Dux = Diagonal(hzi) ⊗ Diagonal(hyi) ⊗ M1D - - # Calculates difference from corner points to velocity points - diag1 = ones(T, Nuy_t - 2) - M1D = spdiagm(Nuy_t - 2, Nuy_t - 1, 0 => -diag1, 1 => diag1) - Cuy = I(Nz) ⊗ M1D ⊗ I(Nux_in) - Duy = Diagonal(hzi) ⊗ M1D ⊗ Diagonal(gxi) - - # Calculates difference from corner points to velocity points - diag1 = ones(T, Nuz_t - 2) - M1D = spdiagm(Nuz_t - 2, Nuz_t - 1, 0 => -diag1, 1 => diag1) - Cuz = M1D ⊗ I(Ny) ⊗ I(Nux_in) - Duz = M1D ⊗ Diagonal(hyi) ⊗ Diagonal(gxi) - - # Cu = [Cux Cuy Cuz] - # Du = [Dux Duy Duz] - - ## Convection (differencing) operator Cv - - # Calculates difference from pressure points to velocity points - diag1 = ones(T, Nvx_t - 2) - M1D = spdiagm(Nvx_t - 2, Nvx_t - 1, 0 => -diag1, 1 => diag1) - Cvx = I(Nz) ⊗ I(Nvy_in) ⊗ M1D - Dvx = Diagonal(hzi) ⊗ Diagonal(gyi) ⊗ M1D - - # Calculates difference from corner points to velocity points - diag1 = ones(T, Nvy_t - 2) - M1D = spdiagm(Nvy_t - 2, Nvy_t - 1, 0 => -diag1, 1 => diag1) - Cvy = I(Nz) ⊗ M1D ⊗ I(Nx) - Dvy = Diagonal(hzi) ⊗ M1D ⊗ Diagonal(hxi) - - # Calculates difference from corner points to velocity points - diag1 = ones(T, Nvz_t - 2) - M1D = spdiagm(Nvz_t - 2, Nvz_t - 1, 0 => -diag1, 1 => diag1) - Cvz = M1D ⊗ I(Nvy_in) ⊗ I(Nx) - Dvz = M1D ⊗ Diagonal(gyi) ⊗ Diagonal(hxi) - - # Cv = [Cvx Cvy Cvz] - # Dv = [Dvx Dvy Dvz] - - ## Convection (differencing) operator Cw - - # Calculates difference from pressure points to velocity points - diag1 = ones(T, Nwx_t - 2) - M1D = spdiagm(Nwx_t - 2, Nwx_t - 1, 0 => -diag1, 1 => diag1) - Cwx = I(Nwz_in) ⊗ I(Ny) ⊗ M1D - Dwx = Diagonal(gzi) ⊗ Diagonal(hyi) ⊗ M1D - - # Calculates difference from corner points to velocity points - diag1 = ones(T, Nwy_t - 2) - M1D = spdiagm(Nwy_t - 2, Nwy_t - 1, 0 => -diag1, 1 => diag1) - Cwy = I(Nwz_in) ⊗ M1D ⊗ I(Nx) - Dwy = Diagonal(gzi) ⊗ M1D ⊗ Diagonal(hxi) - - # Calculates difference from corner points to velocity points - diag1 = ones(T, Nwz_t - 2) - M1D = spdiagm(Nwz_t - 2, Nwz_t - 1, 0 => -diag1, 1 => diag1) - Cwz = M1D ⊗ I(Ny) ⊗ I(Nx) - Dwz = M1D ⊗ Diagonal(hyi) ⊗ Diagonal(hxi) - - # Cw = [Cwx Cwy Cwz] - # Dw = [Dwx Dwy Dwz] - - ## Diffusion operator (stress tensor), u-component: similar to averaging, but with mesh sizes - - ## Su_ux: evaluate ux - diag1 = 1 ./ hxd - S1D = spdiagm(Nux_t - 1, Nux_t, 0 => -diag1, 1 => diag1) - - # Boundary conditions - Su_ux_bc = bc_general( - Nux_t, - Nux_in, - Nux_b, - boundary_conditions.u.x[1], - boundary_conditions.u.x[2], - hx[1], - hx[end], - ) - - # Extend to 3D - Su_ux = I(Nuz_in) ⊗ I(Nuy_in) ⊗ (S1D * Su_ux_bc.B1D) - Su_ux_bc = (; Su_ux_bc..., Bbc = I(Nuz_in) ⊗ I(Nuy_in) ⊗ (S1D * Su_ux_bc.Btemp)) - - ## Su_uy: evaluate uy - diag1 = 1 ./ gyd - S1D = spdiagm(Nuy_t - 1, Nuy_t, 0 => -diag1, 1 => diag1) - - # Boundary conditions - Su_uy_bc = bc_general_stag_diff( - Nuy_t, - Nuy_in, - Nuy_b, - boundary_conditions.u.y[1], - boundary_conditions.u.y[2], - hy[1], - hy[end], - ) - - # Extend to 3D - Su_uy = I(Nuz_in) ⊗ (S1D * Su_uy_bc.B1D) ⊗ I(Nux_in) - Su_uy_bc = (; Su_uy_bc..., Bbc = I(Nuz_in) ⊗ (S1D * Su_uy_bc.Btemp) ⊗ I(Nux_in)) - - ## Su_uz: evaluate uz - diag1 = 1 ./ gzd - S1D = spdiagm(Nuz_t - 1, Nuz_t, 0 => -diag1, 1 => diag1) - - # Boundary conditions - Su_uz_bc = bc_general_stag_diff( - Nuz_t, - Nuz_in, - Nuz_b, - boundary_conditions.u.z[1], - boundary_conditions.u.z[2], - hz[1], - hz[end], - ) - - # Extend to 3D - Su_uz = (S1D * Su_uz_bc.B1D) ⊗ I(Nuy_in) ⊗ I(Nux_in) - Su_uz_bc = (; Su_uz_bc..., Bbc = (S1D * Su_uz_bc.Btemp) ⊗ I(Nuy_in) ⊗ I(Nux_in)) - - ## Sv_uy: evaluate vx at uy; same as Iv_uy except for mesh sizes and -diag diag - diag1 = 1 ./ gxd - S1D = spdiagm(Nvx_t - 1, Nvx_t, 0 => -diag1, 1 => diag1) - - # The restriction is essentially 1D so it can be directly applied to I1D - S1D = Bvux * S1D - S3D = I(Nz) ⊗ I(Nuy_t - 1) ⊗ S1D - - # Boundary conditions left/right - Sv_uy_bc_lr = bc_general_stag( - Nvx_t, - Nvx_in, - Nvx_b, - boundary_conditions.v.x[1], - boundary_conditions.v.x[2], - hx[1], - hx[end], - ) - - # Take I3D into left/right operators for convenience - Sv_uy_bc_lr = (; Sv_uy_bc_lr..., B3D = S3D * (I(Nz) ⊗ I(Nuy_t - 1) ⊗ Sv_uy_bc_lr.B1D)) - Sv_uy_bc_lr = (; Sv_uy_bc_lr..., Bbc = S3D * (I(Nz) ⊗ I(Nuy_t - 1) ⊗ Sv_uy_bc_lr.Btemp)) - - # Boundary conditions low/up - Nb = Nuy_in + 1 - Nvy_in - Sv_uy_bc_lu = bc_general( - Nuy_in + 1, - Nvy_in, - Nb, - boundary_conditions.v.y[1], - boundary_conditions.v.y[2], - hy[1], - hy[end], - ) - Sv_uy_bc_lu = (; Sv_uy_bc_lu..., B3D = I(Nz) ⊗ Sv_uy_bc_lu.B1D ⊗ I(Nvx_in)) - Sv_uy_bc_lu = (; Sv_uy_bc_lu..., Bbc = I(Nz) ⊗ Sv_uy_bc_lu.Btemp ⊗ I(Nvx_in)) - - # Resulting operator: - Sv_uy = Sv_uy_bc_lr.B3D * Sv_uy_bc_lu.B3D - - ## Sw_uz: evaluate wx at uz; same as Iw_uz except for mesh sizes and -diag diag - diag1 = 1 ./ gxd - S1D = spdiagm(Nwx_t - 1, Nwx_t, 0 => -diag1, 1 => diag1) - - # The restriction is essentially 1D so it can be directly applied to I1D - S1D = Bwux * S1D - S3D = I(Nz + 1) ⊗ I(Ny) ⊗ S1D - - # Boundary conditions left/right - Sw_uz_bc_lr = bc_general_stag( - Nwx_t, - Nwx_in, - Nwx_b, - boundary_conditions.w.x[1], - boundary_conditions.w.x[2], - hx[1], - hx[end], - ) - - # Take I3D into left/right operators for convenience - Sw_uz_bc_lr = (; Sw_uz_bc_lr..., B3D = S3D * (I(Nz + 1) ⊗ I(Ny) ⊗ Sw_uz_bc_lr.B1D)) - Sw_uz_bc_lr = (; Sw_uz_bc_lr..., Bbc = S3D * (I(Nz + 1) ⊗ I(Ny) ⊗ Sw_uz_bc_lr.Btemp)) - - # Boundary conditions back/front - Nb = Nuz_in + 1 - Nwz_in - Sw_uz_bc_bf = bc_general( - Nuz_in + 1, - Nwz_in, - Nb, - boundary_conditions.w.z[1], - boundary_conditions.w.z[2], - hz[1], - hz[end], - ) - Sw_uz_bc_bf = (; Sw_uz_bc_bf..., B3D = Sw_uz_bc_bf.B1D ⊗ I(Ny) ⊗ I(Nx)) - Sw_uz_bc_bf = (; Sw_uz_bc_bf..., Bbc = Sw_uz_bc_bf.Btemp ⊗ I(Ny) ⊗ I(Nx)) - - # Resulting operator: - Sw_uz = Sw_uz_bc_lr.B3D * Sw_uz_bc_bf.B3D - - ## Diffusion operator (stress tensor), v-component: similar to averaging! - - ## Su_vx: evaluate uy at vx. Same as Iu_vx except for mesh sizes and -diag diag - diag1 = 1 ./ gyd - S1D = spdiagm(Nuy_t - 1, Nuy_t, 0 => -diag1, 1 => diag1) - S1D = Buvy * S1D - S3D = I(Nz) ⊗ S1D ⊗ I(Nx + 1) - - # Boundary conditions left/right - Nb = Nvx_in + 1 - Nux_in - Su_vx_bc_lr = bc_general( - Nvx_in + 1, - Nux_in, - Nb, - boundary_conditions.u.x[1], - boundary_conditions.u.x[2], - hx[1], - hx[end], - ) - Su_vx_bc_lr = (; Su_vx_bc_lr..., B3D = I(Nz) ⊗ I(Ny) ⊗ Su_vx_bc_lr.B1D) - Su_vx_bc_lr = (; Su_vx_bc_lr..., Bbc = I(Nz) ⊗ I(Ny) ⊗ Su_vx_bc_lr.Btemp) - - # Boundary conditions low/up - Su_vx_bc_lu = bc_general_stag( - Nuy_t, - Nuy_in, - Nuy_b, - boundary_conditions.u.y[1], - boundary_conditions.u.y[2], - hy[1], - hy[end], - ) - Su_vx_bc_lu = (; Su_vx_bc_lu..., B3D = S3D * (I(Nz) ⊗ Su_vx_bc_lu.B1D ⊗ I(Nx + 1))) - Su_vx_bc_lu = (; Su_vx_bc_lu..., Bbc = S3D * (I(Nz) ⊗ Su_vx_bc_lu.Btemp ⊗ I(Nx + 1))) - - # Resulting operator: - Su_vx = Su_vx_bc_lu.B3D * Su_vx_bc_lr.B3D - - ## Sv_vx: evaluate vx - diag1 = 1 ./ gxd - S1D = spdiagm(Nvx_t - 1, Nvx_t, 0 => -diag1, 1 => diag1) - - # Boundary conditions - Sv_vx_bc = bc_general_stag_diff( - Nvx_t, - Nvx_in, - Nvx_b, - boundary_conditions.v.x[1], - boundary_conditions.v.x[2], - hx[1], - hx[end], - ) - - # Extend to 3D - Sv_vx = I(Nvz_in) ⊗ I(Nvy_in) ⊗ (S1D * Sv_vx_bc.B1D) - Sv_vx_bc = (; Sv_vx_bc..., Bbc = I(Nvz_in) ⊗ I(Nvy_in) ⊗ (S1D * Sv_vx_bc.Btemp)) - - ## Sv_vy: evaluate vy - diag1 = 1 ./ hyd - S1D = spdiagm(Nvy_t - 1, Nvy_t, 0 => -diag1, 1 => diag1) - - # Boundary conditions - Sv_vy_bc = bc_general( - Nvy_t, - Nvy_in, - Nvy_b, - boundary_conditions.v.y[1], - boundary_conditions.v.y[2], - hy[1], - hy[end], - ) - - # Extend to 3D - Sv_vy = I(Nvz_in) ⊗ (S1D * Sv_vy_bc.B1D) ⊗ I(Nx) - Sv_vy_bc = (; Sv_vy_bc..., Bbc = I(Nvz_in) ⊗ (S1D * Sv_vy_bc.Btemp) ⊗ I(Nx)) - - ## Sv_vz: evaluate v at vz location - diag1 = 1 ./ gzd - S1D = spdiagm(Nvz_t - 1, Nvz_t, 0 => -diag1, 1 => diag1) - - # Boundary conditions - Sv_vz_bc = bc_general_stag_diff( - Nvz_t, - Nvz_in, - Nvz_b, - boundary_conditions.v.z[1], - boundary_conditions.v.z[2], - hz[1], - hz[end], - ) - - # Extend to 3D - Sv_vz = (S1D * Sv_vz_bc.B1D) ⊗ I(Nvy_in) ⊗ I(Nvx_in) - Sv_vz_bc = (; Sv_vz_bc..., Bbc = (S1D * Sv_vz_bc.Btemp) ⊗ I(Nvy_in) ⊗ I(Nvx_in)) - - ## Sw_vz: evaluate wy at vz location - diag1 = 1 ./ gyd - S1D = spdiagm(Nwy_t - 1, Nwy_t, 0 => -diag1, 1 => diag1) - S1D = Bwvy * S1D - S3D = I(Nz + 1) ⊗ S1D ⊗ I(Nx) - - # Boundary conditions low/up - Sw_vz_bc_lu = bc_general_stag( - Nwy_t, - Nwy_in, - Nwy_b, - boundary_conditions.w.y[1], - boundary_conditions.w.y[2], - hy[1], - hy[end], - ) - Sw_vz_bc_lu = (; Sw_vz_bc_lu..., B3D = S3D * (I(Nz + 1) ⊗ Sw_vz_bc_lu.B1D ⊗ I(Nx))) - Sw_vz_bc_lu = (; Sw_vz_bc_lu..., Bbc = S3D * (I(Nz + 1) ⊗ Sw_vz_bc_lu.Btemp ⊗ I(Nx))) - - # Boundary conditions back/front - Nb = Nvz_in + 1 - Nwz_in - Sw_vz_bc_bf = bc_general( - Nvz_in + 1, - Nwz_in, - Nb, - boundary_conditions.w.z[1], - boundary_conditions.w.z[2], - hz[1], - hz[end], - ) - Sw_vz_bc_bf = (; Sw_vz_bc_bf..., B3D = Sw_vz_bc_bf.B1D ⊗ I(Ny) ⊗ I(Nx)) - Sw_vz_bc_bf = (; Sw_vz_bc_bf..., Bbc = Sw_vz_bc_bf.Btemp ⊗ I(Ny) ⊗ I(Nx)) - - # Resulting operator: - Sw_vz = Sw_vz_bc_lu.B3D * Sw_vz_bc_bf.B3D - - ## Diffusion operator (stress tensor), w-component: similar to averaging, but with mesh sizes - - ## Sw_wx: evaluate w at wx location - diag1 = 1 ./ gxd - S1D = spdiagm(Nwx_t - 1, Nwx_t, 0 => -diag1, 1 => diag1) - - # Boundary conditions - Sw_wx_bc = bc_general_stag_diff( - Nwx_t, - Nwx_in, - Nwx_b, - boundary_conditions.w.x[1], - boundary_conditions.w.x[2], - hx[1], - hx[end], - ) - - # Extend to 3D - Sw_wx = I(Nwz_in) ⊗ I(Nwy_in) ⊗ (S1D * Sw_wx_bc.B1D) - Sw_wx_bc = (; Sw_wx_bc..., Bbc = I(Nwz_in) ⊗ I(Nwy_in) ⊗ (S1D * Sw_wx_bc.Btemp)) - - ## Sw_wy: evaluate w at wy location - diag1 = 1 ./ gyd - S1D = spdiagm(Nwy_t - 1, Nwy_t, 0 => -diag1, 1 => diag1) - - # Boundary conditions - Sw_wy_bc = bc_general_stag_diff( - Nwy_t, - Nwy_in, - Nwy_b, - boundary_conditions.w.y[1], - boundary_conditions.w.y[2], - hy[1], - hy[end], - ) - - # Extend to 3D - Sw_wy = I(Nwz_in) ⊗ (S1D * Sw_wy_bc.B1D) ⊗ I(Nwx_in) - Sw_wy_bc = (; Sw_wy_bc..., Bbc = I(Nwz_in) ⊗ (S1D * Sw_wy_bc.Btemp) ⊗ I(Nwx_in)) - - ## Sw_wz: evaluate w at wz location - diag1 = 1 ./ hzd - S1D = spdiagm(Nwz_t - 1, Nwz_t, 0 => -diag1, 1 => diag1) - - # Boundary conditions - Sw_wz_bc = bc_general( - Nwz_t, - Nwz_in, - Nwz_b, - boundary_conditions.w.z[1], - boundary_conditions.w.z[2], - hz[1], - hz[end], - ) - - # Extend to 3D - Sw_wz = (S1D * Sw_wz_bc.B1D) ⊗ I(Nwy_in) ⊗ I(Nwx_in) - Sw_wz_bc = (; Sw_wz_bc..., Bbc = (S1D * Sw_wz_bc.Btemp) ⊗ I(Nwy_in) ⊗ I(Nwx_in)) - - ## Su_wx: evaluate uz at wx - diag1 = 1 ./ gzd - S1D = spdiagm(Nuz_t - 1, Nuz_t, 0 => -diag1, 1 => diag1) - - # The restriction is essentially 1D so it can be directly applied to I1D - S1D = Buwz * S1D - S3D = S1D ⊗ I(Ny) ⊗ I(Nx + 1) - - # Boundary conditions back/front - Su_wx_bc_bf = bc_general_stag( - Nuz_t, - Nuz_in, - Nuz_b, - boundary_conditions.u.z[1], - boundary_conditions.u.z[2], - hz[1], - hz[end], - ) - - # Take I3D into left/right operators for convenience - Su_wx_bc_bf = (; Su_wx_bc_bf..., B3D = S3D * (Su_wx_bc_bf.B1D ⊗ I(Ny) ⊗ I(Nx + 1))) - Su_wx_bc_bf = (; Su_wx_bc_bf..., Bbc = S3D * (Su_wx_bc_bf.Btemp ⊗ I(Ny) ⊗ I(Nx + 1))) - - # Boundary conditions left/right - Nb = Nwx_in + 1 - Nux_in - Su_wx_bc_lr = bc_general( - Nwx_in + 1, - Nux_in, - Nb, - boundary_conditions.u.x[1], - boundary_conditions.u.x[2], - hx[1], - hx[end], - ) - Su_wx_bc_lr = (; Su_wx_bc_lr..., B3D = I(Nz) ⊗ I(Ny) ⊗ Su_wx_bc_lr.B1D) - Su_wx_bc_lr = (; Su_wx_bc_lr..., Bbc = I(Nz) ⊗ I(Ny) ⊗ Su_wx_bc_lr.Btemp) - - # Resulting operator: - Su_wx = Su_wx_bc_bf.B3D * Su_wx_bc_lr.B3D - - ## Sv_wy: evaluate vz at wy - diag1 = 1 ./ gzd - S1D = spdiagm(Nvz_t - 1, Nvz_t, 0 => -diag1, 1 => diag1) - - # The restriction is essentially 1D so it can be directly applied to I1D - S1D = Bvwz * S1D - S3D = S1D ⊗ I(Ny + 1) ⊗ I(Nx) - - # Boundary conditions back/front - Sv_wy_bc_bf = bc_general_stag( - Nvz_t, - Nvz_in, - Nvz_b, - boundary_conditions.v.z[1], - boundary_conditions.v.z[2], - hz[1], - hz[end], - ) - - # Take I3D into left/right operators for convenience - Sv_wy_bc_bf = (; Sv_wy_bc_bf..., B3D = S3D * (Sv_wy_bc_bf.B1D ⊗ I(Ny + 1) ⊗ I(Nx))) - Sv_wy_bc_bf = (; Sv_wy_bc_bf..., Bbc = S3D * (Sv_wy_bc_bf.Btemp ⊗ I(Ny + 1) ⊗ I(Nx))) - - # Boundary conditions low/up - Nb = Nwy_in + 1 - Nvy_in - Sv_wy_bc_lu = bc_general( - Nwy_in + 1, - Nvy_in, - Nb, - boundary_conditions.v.y[1], - boundary_conditions.v.y[2], - hy[1], - hy[end], - ) - Sv_wy_bc_lu = (; Sv_wy_bc_lu..., B3D = I(Nz) ⊗ Sv_wy_bc_lu.B1D ⊗ I(Nx)) - Sv_wy_bc_lu = (; Sv_wy_bc_lu..., Bbc = I(Nz) ⊗ Sv_wy_bc_lu.Btemp ⊗ I(Nx)) - - # Resulting operator: - Sv_wy = Sv_wy_bc_bf.B3D * Sv_wy_bc_lu.B3D - - ## Assemble operators - Diffu = Dux * Su_ux + Duy * Su_uy + Duz * Su_uz - Diffv = Dvx * Sv_vx + Dvy * Sv_vy + Dvz * Sv_vz - Diffw = Dwx * Sw_wx + Dwy * Sw_wy + Dwz * Sw_wz - Diff = blockdiag(Diffu, Diffv, Diffw) - - ## Group opearators - (; - Cux, - Cuy, - Cuz, - Cvx, - Cvy, - Cvz, - Cwx, - Cwy, - Cwz, - Su_ux, - Su_uy, - Su_uz, - Sv_vx, - Sv_vy, - Sv_vz, - Sw_wx, - Sw_wy, - Sw_wz, - Su_ux_bc, - Su_uy_bc, - Su_uz_bc, - Sv_vx_bc, - Sv_vy_bc, - Sv_vz_bc, - Sw_wx_bc, - Sw_wy_bc, - Sw_wz_bc, - Sv_uy_bc_lr, - Sv_uy_bc_lu, - Sw_uz_bc_lr, - Sw_uz_bc_bf, - Su_vx_bc_lr, - Su_vx_bc_lu, - Sw_vz_bc_lu, - Sw_vz_bc_bf, - Su_wx_bc_lr, - Su_wx_bc_bf, - Sv_wy_bc_lu, - Sv_wy_bc_bf, - Dux, - Duy, - Duz, - Dvx, - Dvy, - Dvz, - Dwx, - Dwy, - Dwz, - Diff, - Sv_uy, - Su_vx, - Sw_uz, - Su_wx, - Sw_vz, - Sv_wy, - ) -end diff --git a/src/operators/operator_divergence.jl b/src/operators/operator_divergence.jl deleted file mode 100644 index ae1ddf7fc..000000000 --- a/src/operators/operator_divergence.jl +++ /dev/null @@ -1,293 +0,0 @@ -""" - operator_divergence(grid, boundary_conditions) - -Construct divergence and gradient operator. -""" -operator_divergence(grid, boundary_conditions) = - operator_divergence(grid.dimension, grid, boundary_conditions) - -# 2D version -function operator_divergence(::Dimension{2}, grid, boundary_conditions) - (; Npx, Npy) = grid - (; Nux_in, Nux_b, Nux_t, Nuy_in) = grid - (; Nvx_in, Nvy_in, Nvy_b, Nvy_t) = grid - (; hx, hy) = grid - (; Ω) = grid - (; order4, α) = grid - - bc = boundary_conditions - - if order4 - (; hxi3, hyi3) = grid - end - - T = eltype(hx) - - ## Divergence operator M - - # Note that the divergence matrix M is not square - mat_hx = Diagonal(hx) - mat_hy = Diagonal(hy) - - # For fourth order: mat_hx3 is defined in operator_interpolation - - ## Mx - # Building blocks consisting of diagonal matrices where the diagonal is - # equal to constant per block (hy(block)) and changing for next block to - # hy(block+1) - diag1 = ones(T, Nux_t - 1) - M1D = spdiagm(Nux_t - 1, Nux_t, 0 => -diag1, 1 => diag1) - - # We only need derivative at inner pressure points, so we map the resulting - # boundary matrix (restrict) - diagpos = 0 - bc.u.x[2] == :pressure && bc.u.x[1] == :pressure && (diagpos = 1) - bc.u.x[2] != :pressure && bc.u.x[1] == :pressure && (diagpos = 1) - bc.u.x[2] == :pressure && bc.u.x[1] != :pressure && (diagpos = 0) - bc.u.x[2] == :periodic && bc.u.x[1] == :periodic && (diagpos = 1) - - BMx = spdiagm(Npx, Nux_t - 1, diagpos => ones(T, Npx)) - M1D = BMx * M1D - - # Extension to 2D to be used in post-processing files - Bup = I(Nuy_in) ⊗ BMx - - # Boundary conditions - Mx_bc = bc_general(Nux_t, Nux_in, Nux_b, bc.u.x[1], bc.u.x[2], hx[1], hx[end]) - Mx_bc = (; Mx_bc..., Bbc = mat_hy ⊗ (M1D * Mx_bc.Btemp)) - - # Extend to 2D - Mx = mat_hy ⊗ (M1D * Mx_bc.B1D) - - if order4 - mat_hy3 = Diagonal(hyi3) - diag1 = ones(T, Nux_t - 1) - M1D3 = spdiagm(Nux_t - 1, Nux_t + 2, 0 => -diag1, 3 => diag1) - M1D3 = BMx * M1D3 - Mx_bc3 = bc_div2( - Nux_t + 2, - Nux_in, - Nux_t + 2 - Nux_in, - bc.u.x[1], - bc.u.x[2], - hx[1], - hx[end], - ) - Mx3 = mat_hy3 ⊗ (M1D3 * Mx_bc3.B1D) - Mx_bc3 = (; Mx_bc3..., Bbc = mat_hy3 ⊗ (M1D3 * Mx_bc3.Btemp)) - end - - ## My (same as Mx but reversing indices and kron arguments) - diag1 = ones(T, Nvy_t - 1) - M1D = spdiagm(Nvy_t - 1, Nvy_t, 0 => -diag1, 1 => diag1) - - # We only need derivative at inner pressure points, so we map the resulting - # boundary matrix (restriction) - diagpos = 0 - bc.v.y[2] == :pressure && bc.v.y[1] == :pressure && (diagpos = 1) - bc.v.y[2] != :pressure && bc.v.y[1] == :pressure && (diagpos = 1) - bc.v.y[2] == :pressure && bc.v.y[1] != :pressure && (diagpos = 0) - bc.v.y[2] == :periodic && bc.v.y[1] == :periodic && (diagpos = 1) - - BMy = spdiagm(Npy, Nvy_t - 1, diagpos => ones(T, Npy)) - M1D = BMy * M1D - - # Extension to 2D to be used in post-processing files - Bvp = BMy ⊗ I(Nvx_in) - - # Boundary conditions - My_bc = bc_general(Nvy_t, Nvy_in, Nvy_b, bc.v.y[1], bc.v.y[2], hy[1], hy[end]) - My_bc = (; My_bc..., Bbc = (M1D * My_bc.Btemp) ⊗ mat_hx) - - # Extend to 2D - My = (M1D * My_bc.B1D) ⊗ mat_hx - - if order4 - mat_hx3 = Diagonal(hxi3) - diag1 = ones(T, Nvy_t - 1) - M1D3 = spdiagm(Nvy_t - 1, Nvy_t + 2, 0 => -diag1, 3 => diag1) - M1D3 = BMy * M1D3 - My_bc3 = bc_div2( - Nvy_t + 2, - Nvy_in, - Nvy_t + 2 - Nvy_in, - bc.v.y[1], - bc.v.y[2], - hy[1], - hy[end], - ) - My3 = (M1D3 * My_bc3.B1D) ⊗ mat_hx3 - My_bc3 = (; My_bc3..., Bbc = (M1D3 * My_bc3.Btemp) ⊗ mat_hx3) - end - - ## Resulting divergence matrix - if order4 - Mx = α * Mx - Mx3 - My = α * My - My3 - end - M = [Mx My] - - ## Gradient operator G - - # Like in the continuous case, grad = -div^T - # Note that this also holds for outflow boundary conditions, if the stress - # on the ouflow boundary is properly taken into account in y_p (often this - # stress will be zero) - G = -sparse(M') - - ## Pressure matrix for pressure correction method; - # Also used to make initial data divergence free or compute additional poisson solve - # Note that the matrix for the pressure is constant in time. - # Only the right hand side vector changes, so the pressure matrix can be set up outside the time-stepping-loop. - - # Laplace = div grad - A = M * Diagonal(1 ./ Ω) * G - - # Check if all the row sums of the pressure matrix are zero, which - # should be the case if there are no pressure boundary conditions - if all(≠(:pressure), (bc.u.x..., bc.v.y...)) - if any(!≈(0; atol = sqrt(eps(T))), sum(A; dims = 2)) - @warn "Pressure matrix: not all rowsums are zero!" - end - end - - ## Group operators - operators = (; M, Mx_bc, My_bc, G, Bup, Bvp, A) - - if order4 - operators = (; operators..., Mx3, My3, Mx_bc3, My_bc3) - end - - operators -end - -# 3D version -function operator_divergence(::Dimension{3}, grid, boundary_conditions) - (; Nux_in, Nux_b, Nux_t, Nuy_in, Nuz_in) = grid - (; Nvx_in, Nvy_in, Nvy_b, Nvy_t, Nvz_in) = grid - (; Nwx_in, Nwy_in, Nwz_in, Nwz_b, Nwz_t) = grid - (; Npx, Npy, Npz) = grid - (; hx, hy, hz) = grid - (; Ω) = grid - - T = eltype(hx) - - bc = boundary_conditions - - ## Divergence operator M - - # Note that the divergence matrix M is not square - mat_hx = Diagonal(hx) - mat_hy = Diagonal(hy) - mat_hz = Diagonal(hz) - - ## Mx - # Building blocks consisting of diagonal matrices where the diagonal is - # Equal to constant per block (hy(block)) and changing for next block to - # Hy(block+1) - diag1 = ones(T, Nux_t - 1) - M1D = spdiagm(Nux_t - 1, Nux_t, 0 => -diag1, 1 => diag1) - - # We only need derivative at inner pressure points, so we map the resulting - # Boundary matrix (restrict) - diagpos = 0 - bc.u.x[2] == :pressure && bc.u.x[1] == :pressure && (diagpos = 1) - bc.u.x[2] != :pressure && bc.u.x[1] == :pressure && (diagpos = 1) - bc.u.x[2] == :pressure && bc.u.x[1] != :pressure && (diagpos = 0) - bc.u.x[2] == :periodic && bc.u.x[1] == :periodic && (diagpos = 1) - BMx = spdiagm(Npx, Nux_t - 1, diagpos => ones(T, Npx)) - M1D = BMx * M1D - - # Extension to 3D to be used in post-processing files - Bup = I(Nuz_in) ⊗ I(Nuy_in) ⊗ BMx - - # Boundary conditions - Mx_bc = bc_general(Nux_t, Nux_in, Nux_b, bc.u.x[1], bc.u.x[2], hx[1], hx[end]) - Mx_bc = (; Mx_bc..., Bbc = mat_hz ⊗ mat_hy ⊗ (M1D * Mx_bc.Btemp)) - - # Extend to 3D - Mx = mat_hz ⊗ mat_hy ⊗ (M1D * Mx_bc.B1D) - - ## My - # Same as Mx but reversing indices and kron arguments - diag1 = ones(T, Nvy_t - 1) - M1D = spdiagm(Nvy_t - 1, Nvy_t, 0 => -diag1, 1 => diag1) - - # We only need derivative at inner pressure points, so we map the resulting - # Boundary matrix (restriction) - diagpos = 0 - bc.v.y[2] == :pressure && bc.v.y[1] == :pressure && (diagpos = 1) - bc.v.y[2] != :pressure && bc.v.y[1] == :pressure && (diagpos = 1) - bc.v.y[2] == :pressure && bc.v.y[1] != :pressure && (diagpos = 0) - bc.v.y[2] == :periodic && bc.v.y[1] == :periodic && (diagpos = 1) - BMy = spdiagm(Npy, Nvy_t - 1, diagpos => ones(T, Npy)) - M1D = BMy * M1D - - # Extension to 3D to be used in post-processing files - Bvp = I(Nvz_in) ⊗ BMy ⊗ I(Nvx_in) - - # Boundary conditions - My_bc = bc_general(Nvy_t, Nvy_in, Nvy_b, bc.v.y[1], bc.v.y[2], hy[1], hy[end]) - My_bc = (; My_bc..., Bbc = mat_hz ⊗ (M1D * My_bc.Btemp) ⊗ mat_hx) - - # Extend to 3D - My = mat_hz ⊗ (M1D * My_bc.B1D) ⊗ mat_hx - - ## Mz - # Same as Mx but reversing indices and kron arguments - diag1 = ones(T, Nwz_t - 1) - M1D = spdiagm(Nwz_t - 1, Nwz_t, 0 => -diag1, 1 => diag1) - - # We only need derivative at inner pressure points, so we map the resulting - # Boundary matrix (restriction) - diagpos = 0 - bc.w.z[1] == :pressure && bc.w.z[2] == :pressure && (diagpos = 1) - bc.w.z[1] == :pressure && bc.w.z[2] != :pressure && (diagpos = 1) - bc.w.z[1] != :pressure && bc.w.z[2] == :pressure && (diagpos = 0) - bc.w.z[1] == :periodic && bc.w.z[2] == :periodic && (diagpos = 1) - - BMz = spdiagm(Npz, Nwz_t - 1, diagpos => ones(T, Npz)) - M1D = BMz * M1D - - # Extension to 3D to be used in post-processing files - Bwp = BMz ⊗ I(Nwy_in) ⊗ I(Nwx_in) - - # Boundary conditions - Mz_bc = bc_general(Nwz_t, Nwz_in, Nwz_b, bc.w.z[1], bc.w.z[2], hz[1], hz[end]) - Mz_bc = (; Mz_bc..., Bbc = (M1D * Mz_bc.Btemp) ⊗ mat_hy ⊗ mat_hx) - - # Extend to 3D - Mz = (M1D * Mz_bc.B1D) ⊗ mat_hy ⊗ mat_hx - - ## Resulting divergence matrix - M = [Mx My Mz] - - ## Gradient operator G - - # Like in the continuous case, grad = -div^T - # Note that this also holds for outflow boundary conditions, if the stress - # on the ouflow boundary is properly taken into account in y_p (often this - # stress will be zero) - G = -sparse(M') - - ## Pressure matrix for pressure correction method; - # Also used to make initial data divergence free or compute additional poisson solve - # if !is_steady(problem) - # Note that the matrix for the pressure is constant in time. - # Only the right hand side vector changes, so the pressure matrix can be set up - # outside the time-stepping-loop. - - # Laplace = div grad - A = M * Diagonal(1 ./ Ω) * G - - # Check if all the row sums of the pressure matrix are zero, which - # should be the case if there are no pressure boundary conditions - if all(≠(:pressure), (bc.u.x..., bc.v.y..., bc.w.z...)) - if any(!≈(0; atol = sqrt(eps(T))), sum(A; dims = 2)) - @warn "Pressure matrix: not all rowsums are zero!" - end - end - - ## Group operators - (; M, Mx_bc, My_bc, Mz_bc, G, Bup, Bvp, Bwp, A) -end diff --git a/src/operators/operator_filter.jl b/src/operators/operator_filter.jl deleted file mode 100644 index a048f13a5..000000000 --- a/src/operators/operator_filter.jl +++ /dev/null @@ -1,125 +0,0 @@ -""" - create_top_hat_p([T = Float64], N, M) - -`N` fine points and `M` coarse points in each dimension. -""" -create_top_hat_p(N, M) = create_top_hat_p(Float64, N, M) -function create_top_hat_p(T, N, M) - s = N ÷ M - @assert s * M == N - - i = 1:M - j = reshape(1:M, 1, :) - ij = @. i + M * (j - 1) - ij = repeat(ij, 1, 1, s, s) - - k = reshape(1:s, 1, 1, :) - l = reshape(1:s, 1, 1, 1, :) - - ijkl = @. s * (i - 1) + k + s * N * (j - 1) + N * (l - 1) - - z = fill(T(1) / s^2, N^2) - - sparse(ij[:], ijkl[:], z) -end - -""" - create_top_hat_u([T = Float64], N, M) - -`N` fine points and `M` coarse points in each dimension. -""" -create_top_hat_u(N, M) = create_top_hat_u(Float64, N, M) -function create_top_hat_u(T, N, M) - s = N ÷ M - @assert s * M == N - - i = 1:M - j = reshape(1:M, 1, :) - ij = @. i + M * (j - 1) - ij = repeat(ij, 1, 1, 1, s) - - k = fill(1, 1, 1, 1) - l = reshape(1:s, 1, 1, 1, :) - - ijkl = @. s * (i - 1) + k + s * N * (j - 1) + N * (l - 1) - - z = fill(T(1) / s, N * M) - - sparse(ij[:], ijkl[:], z, M^2, N^2) -end - -""" -create_top_hat_v([T = Float64], N, M) - -`N` fine points and `M` coarse points in each dimension. -""" -create_top_hat_v(N, M) = create_top_hat_v(Float64, N, M) -function create_top_hat_v(T, N, M) - s = N ÷ M - @assert s * M == N - - i = 1:M - j = reshape(1:M, 1, :) - ij = @. i + M * (j - 1) - ij = repeat(ij, 1, 1, s, 1) - - k = reshape(1:s, 1, 1, :, 1) - l = fill(1, 1, 1, 1, 1) - - ijkl = @. s * (i - 1) + k + s * N * (j - 1) + N * (l - 1) - - z = fill(T(1) / s, N * M) - - sparse(ij[:], ijkl[:], z, M^2, N^2) -end - -""" - create_top_hat_velocity(N, M) - -`N` fine points and `M` coarse points in each dimension. -""" -function create_top_hat_velocity(N, M) - Wu = create_top_hat_u(N, M) - Wv = create_top_hat_v(N, M) - blockdiag(Wu, Wv) -end - -""" - operator_filter(grid, boundary_conditions) - -Construct filtering operator. -""" -operator_filter(grid, boundary_conditions, s) = - operator_filter(grid.dimension, grid, boundary_conditions, s) - -# 2D version -function operator_filter(::Dimension{2}, grid, boundary_conditions, s) - (; Nx, Ny, hx, hy, x) = grid - T = eltype(x) - N = Nx - M = N ÷ s - - @assert s * M == N - - # Requirements - Δx = first(hx) - Δy = first(hy) - @assert all(≈(Δx), hx) && all(≈(Δy), hy) "Filter assumes uniform grid" - @assert all( - ==((:periodic, :periodic)), - (boundary_conditions.u.x, boundary_conditions.v.y), - ) "Filter assumes periodic boundary conditions" - @assert Nx == Ny - - Kp = create_top_hat_p(T, N, M) - Ku = create_top_hat_u(T, N, M) - Kv = create_top_hat_v(T, N, M) - KV = blockdiag(Ku, Kv) - - (; KV, Kp) -end - -# 3D version -function operator_filter(::Dimension{3}, grid, boundary_conditions) - error("Not implemented") -end diff --git a/src/operators/operator_interpolation.jl b/src/operators/operator_interpolation.jl deleted file mode 100644 index 9608e9734..000000000 --- a/src/operators/operator_interpolation.jl +++ /dev/null @@ -1,857 +0,0 @@ -""" - operator_interpolation(grid, boundary_conditions) - -Construct interpolation operators. -""" -operator_interpolation(grid, boundary_conditions) = - operator_interpolation(grid.dimension, grid, boundary_conditions) - -# 2D version -function operator_interpolation(::Dimension{2}, grid, boundary_conditions) - (; Nx, Ny) = grid - (; Nux_in, Nux_b, Nux_t, Nuy_in, Nuy_b, Nuy_t) = grid - (; Nvx_in, Nvx_b, Nvx_t, Nvy_in, Nvy_b, Nvy_t) = grid - (; hx, hy, hxi, hyi) = grid - (; Buvy, Bvux) = grid - (; order4, β) = grid - - if order4 - (; hxi3, hyi3, hx3, hy3) = grid - end - - T = eltype(hx) - - weight = T(1 / 2) - - mat_hx = Diagonal(hxi) - mat_hy = Diagonal(hyi) - - # Periodic boundary conditions - if boundary_conditions.u.x == (:periodic, :periodic) - mat_hx2 = spdiagm(Nx + 2, Nx + 2, [hx[end]; hx; hx[1]]) - else - mat_hx2 = spdiagm(Nx + 2, Nx + 2, [hx[1]; hx; hx[end]]) - end - - if boundary_conditions.v.y == (:periodic, :periodic) - mat_hy2 = spdiagm(Ny + 2, Ny + 2, [hy[end]; hy; hy[1]]) - else - mat_hy2 = spdiagm(Ny + 2, Ny + 2, [hy[1]; hy; hy[end]]) - end - - ## Interpolation operators, u-component - if order4 - mat_hx3 = Diagonal(hxi3) - mat_hy3 = Diagonal(hyi3) - - weight1 = 1 // 2 * β - weight2 = 1 // 2 * (1 - β) - - # Periodic boundary conditions - if boundary_conditions.u.x == (:periodic, :periodic) - mat_hx2 = spdiagm(Nx + 4, Nx + 4, [hx[end-1]; hx[end]; hx; hx[1]; hx[2]]) - mat_hx4 = spdiagm(Nx + 4, Nx + 4, [hx3[end-1]; hx3[end]; hxi3; hx3[1]; hx3[2]]) - else - mat_hx2 = spdiagm(Nx + 4, Nx + 4, [hx[2]; hx[1]; hx; hx[end]; hx[end-1]]) - mat_hx4 = spdiagm( - Nx + 4, - Nx + 4, - [ - hx[1] + hx[2] + hx[3] - 2 * hx[1] + hx[2] - hxi3 - 2 * hx[end] + hx[end-1] - hx[end] + hx[end-1] + hx[end-2] - ], - ) - end - - if boundary_conditions.v.y == (:periodic, :periodic) - mat_hy2 = spdiagm(Ny + 4, Ny + 4, [hy[end-1]; hy[end]; hy; hy[1]; hy[2]]) - mat_hy4 = spdiagm(Ny + 4, Ny + 4, [hy3[end-1]; hy3[end]; hyi3; hy3[1]; hy3[2]]) - else - mat_hy2 = spdiagm(Ny + 4, Ny + 4, [hy[2]; hy[1]; hy; hy[end]; hy[end-1]]) - mat_hy4 = spdiagm( - Ny + 4, - Ny + 4, - [ - hy[1] + hy[2] + hy[3] - 2 * hy[1] + hy[2] - hyi3 - 2 * hy[end] + hy[end-1] - hy[end] + hy[end-1] + hy[end-2] - ], - ) - end - - ## Iu_ux - diag1 = fill(weight1, Nux_t - 1) - diag2 = fill(weight2, Nux_t - 1) - I1D = spdiagm(Nux_t - 1, Nux_t + 2, 0 => diag2, 1 => diag1, 2 => diag1, 3 => diag2) - - # Boundary conditions - Iu_ux_bc = bc_int2( - Nux_t + 2, - Nux_in, - Nux_t + 2 - Nux_in, - boundary_conditions.u.x[1], - boundary_conditions.u.x[2], - hx[1], - hx[end], - ) - - # Extend to 2D - Iu_ux = mat_hy ⊗ (I1D * Iu_ux_bc.B1D) - Iu_ux_bc = (; Iu_ux_bc..., Bbc = mat_hy ⊗ (I1D * Iu_ux_bc.Btemp)) - - ## Iu_ux3 - diag1 = fill(weight1, Nux_in + 3) - diag2 = fill(weight2, Nux_in + 3) - I1D3 = - spdiagm(Nux_in + 3, Nux_t + 4, 0 => diag2, 1 => diag1, 2 => diag1, 3 => diag2) - - # Boundary conditions - Iu_ux_bc3 = bc_int3( - Nux_t + 4, - Nux_in, - Nux_t + 4 - Nux_in, - boundary_conditions.u.x[1], - boundary_conditions.u.x[2], - hx[1], - hx[end], - ) - - # Extend to 2D - Iu_ux3 = mat_hy3 ⊗ (I1D3 * Iu_ux_bc3.B1D) - Iu_ux_bc3 = (; Iu_ux_bc3..., Bbc = mat_hy3 ⊗ (I1D3 * Iu_ux_bc3.Btemp)) - - ## Iv_uy - diag1 = fill(weight1, Nvx_t - 1) - diag2 = fill(weight2, Nvx_t - 1) - I1D = spdiagm(Nvx_t - 1, Nvx_t + 2, 0 => diag2, 1 => diag1, 2 => diag1, 3 => diag2) - any(==(:pressure), boundary_conditions.u.x) && - @warn "Possible interpolation bug (see https://github.com/bsanderse/INS2D/commit/b8de84dbe151d6de32928563ca8fa5785cce6318)" - - # Restrict to u-points - # The restriction is essentially 1D so it can be directly applied to I1D - I1D = Bvux * I1D * mat_hx2 - I2D = I(Nuy_t - 1) ⊗ I1D - - # Boundary conditions low/up - Nb = Nuy_in + 1 - Nvy_in - Iv_uy_bc_lu = bc_general( - Nuy_in + 1, - Nvy_in, - Nb, - boundary_conditions.v.y[1], - boundary_conditions.v.y[2], - hy[1], - hy[end], - ) - Iv_uy_bc_lu = (; Iv_uy_bc_lu..., B2D = Iv_uy_bc_lu.B1D ⊗ I(Nvx_in)) - Iv_uy_bc_lu = (; Iv_uy_bc_lu..., Bbc = Iv_uy_bc_lu.Btemp ⊗ I(Nvx_in)) - - # Boundary conditions left/right - Iv_uy_bc_lr = bc_int_mixed_stag2( - Nvx_t + 2, - Nvx_in, - Nvx_t + 2 - Nvx_in, - boundary_conditions.v.x[1], - boundary_conditions.v.x[2], - hx[1], - hx[end], - ) - - # Take I2D into left/right operators for convenience - Iv_uy_bc_lr = (; Iv_uy_bc_lr..., B2D = I2D * (I(Nuy_t - 1) ⊗ Iv_uy_bc_lr.B1D)) - Iv_uy_bc_lr = (; Iv_uy_bc_lr..., Bbc = I2D * (I(Nuy_t - 1) ⊗ Iv_uy_bc_lr.Btemp)) - - # Resulting operator: - Iv_uy = Iv_uy_bc_lr.B2D * Iv_uy_bc_lu.B2D - - ## Iv_uy3 - diag1 = fill(weight1, Nvx_t - 1) - diag2 = fill(weight2, Nvx_t - 1) - I1D = spdiagm(Nvx_t - 1, Nvx_t + 2, 0 => diag2, 1 => diag1, 2 => diag1, 3 => diag2) - - # Restrict to u-points - # The restriction is essentially 1D so it can be directly applied to I1D - I1D = Bvux * I1D * mat_hx4 - I2D = I(Nuy_t + 1) ⊗ I1D - - # Boundary conditions low/up - Nb = Nuy_in + 3 - Nvy_in - Iv_uy_bc_lu3 = bc_int_mixed2( - Nuy_in + 3, - Nvy_in, - Nb, - boundary_conditions.v.y[1], - boundary_conditions.v.y[2], - hy[1], - hy[end], - ) - Iv_uy_bc_lu3 = (; Iv_uy_bc_lu3..., B2D = Iv_uy_bc_lu3.B1D ⊗ I(Nvx_in)) - Iv_uy_bc_lu3 = (; Iv_uy_bc_lu3..., Bbc = Iv_uy_bc_lu3.Btemp ⊗ I(Nvx_in)) - - # Boundary conditions left/right - Iv_uy_bc_lr3 = bc_int_mixed_stag3( - Nvx_t + 2, - Nvx_in, - Nvx_t + 2 - Nvx_in, - boundary_conditions.v.x[1], - boundary_conditions.v.x[2], - hx[1], - hx[end], - ) - - # Take I2D into left/right operators for convenience - Iv_uy_bc_lr3 = (; Iv_uy_bc_lr3..., B2D = I2D * (I(Nuy_t + 1) ⊗ Iv_uy_bc_lr3.B1D)) - Iv_uy_bc_lr3 = (; Iv_uy_bc_lr3..., Bbc = I2D * (I(Nuy_t + 1) ⊗ Iv_uy_bc_lr3.Btemp)) - - # Resulting operator: - Iv_uy3 = Iv_uy_bc_lr3.B2D * Iv_uy_bc_lu3.B2D - - ## Iu_vx - diag1 = fill(weight1, Nuy_t - 1) - diag2 = fill(weight2, Nuy_t - 1) - I1D = spdiagm(Nuy_t - 1, Nuy_t + 2, 0 => diag2, 1 => diag1, 2 => diag1, 3 => diag2) - any(==(:pressure), boundary_conditions.v.y) && - @warn "Possible interpolation bug (see https://github.com/bsanderse/INS2D/commit/b8de84dbe151d6de32928563ca8fa5785cce6318)" - - # Restrict to v-points - I1D = Buvy * I1D * mat_hy2 - I2D = I1D ⊗ I(Nvx_t - 1) - - # Boundary conditions low/up - Iu_vx_bc_lu = bc_int_mixed_stag2( - Nuy_t + 2, - Nuy_in, - Nuy_t + 2 - Nuy_in, - boundary_conditions.u.y[1], - boundary_conditions.u.y[2], - hy[1], - hy[end], - ) - Iu_vx_bc_lu = (; Iu_vx_bc_lu..., B2D = I2D * (Iu_vx_bc_lu.B1D ⊗ I(Nvx_t - 1))) - Iu_vx_bc_lu = (; Iu_vx_bc_lu..., Bbc = I2D * (Iu_vx_bc_lu.Btemp ⊗ I(Nvx_t - 1))) - - # Boundary conditions left/right - Nb = Nvx_in + 1 - Nux_in - Iu_vx_bc_lr = bc_general( - Nvx_in + 1, - Nux_in, - Nb, - boundary_conditions.u.x[1], - boundary_conditions.u.x[2], - hx[1], - hx[end], - ) - - Iu_vx_bc_lr = (; Iu_vx_bc_lr..., B2D = I(Nuy_in) ⊗ Iu_vx_bc_lr.B1D) - Iu_vx_bc_lr = (; Iu_vx_bc_lr..., Bbc = I(Nuy_in) ⊗ Iu_vx_bc_lr.Btemp) - - # Resulting operator: - Iu_vx = Iu_vx_bc_lu.B2D * Iu_vx_bc_lr.B2D - - ## Iu_vx3 - diag1 = fill(weight1, Nuy_t - 1) - diag2 = fill(weight2, Nuy_t - 1) - I1D = spdiagm(Nuy_t - 1, Nuy_t + 2, 0 => diag2, 1 => diag1, 2 => diag1, 3 => diag2) - - # Restrict to v-points - I1D = Buvy * I1D * mat_hy4 - I2D = I1D ⊗ I(Nvx_t + 1) - - # Boundary conditions low/up - Iu_vx_bc_lu3 = bc_int_mixed_stag3( - Nuy_t + 2, - Nuy_in, - Nuy_t + 2 - Nuy_in, - boundary_conditions.u.y[1], - boundary_conditions.u.y[2], - hy[1], - hy[end], - ) - Iu_vx_bc_lu3 = (; Iu_vx_bc_lu3..., B2D = I2D * (Iu_vx_bc_lu3.B1D ⊗ I(Nvx_t + 1))) - Iu_vx_bc_lu3 = (; Iu_vx_bc_lu3..., Bbc = I2D * (Iu_vx_bc_lu3.Btemp ⊗ I(Nvx_t + 1))) - - # Boundary conditions left/right - Nb = Nvx_in + 3 - Nux_in - Iu_vx_bc_lr3 = bc_int_mixed2( - Nvx_in + 3, - Nux_in, - Nb, - boundary_conditions.u.x[1], - boundary_conditions.u.x[2], - hx[1], - hx[end], - ) - - Iu_vx_bc_lr3 = (; Iu_vx_bc_lr3..., B2D = I(Nuy_in) ⊗ Iu_vx_bc_lr3.B1D) - Iu_vx_bc_lr3 = (; Iu_vx_bc_lr3..., Bbc = I(Nuy_in) ⊗ Iu_vx_bc_lr3.Btemp) - - # Resulting operator: - Iu_vx3 = Iu_vx_bc_lu3.B2D * Iu_vx_bc_lr3.B2D - - ## Iv_vy - diag1 = fill(weight1, Nvy_t - 1) - diag2 = fill(weight2, Nvy_t - 1) - I1D = spdiagm(Nvy_t - 1, Nvy_t + 2, 0 => diag2, 1 => diag1, 2 => diag1, 3 => diag2) - - # Boundary conditions - Iv_vy_bc = bc_int2( - Nvy_t + 2, - Nvy_in, - Nvy_t + 2 - Nvy_in, - boundary_conditions.v.y[1], - boundary_conditions.v.y[2], - hy[1], - hy[end], - ) - - # Extend to 2D - Iv_vy = (I1D * Iv_vy_bc.B1D) ⊗ mat_hx - Iv_vy_bc = (; Iv_vy_bc..., Bbc = (I1D * Iv_vy_bc.Btemp) ⊗ mat_hx) - - ## Iv_vy3 - diag1 = fill(weight1, Nvy_in + 3) - diag2 = fill(weight2, Nvy_in + 3) - I1D3 = - spdiagm(Nvy_in + 3, Nvy_t + 4, 0 => diag2, 1 => diag1, 2 => diag1, 3 => diag2) - - # Boundary conditions - Iv_vy_bc3 = bc_int3( - Nvy_t + 4, - Nvy_in, - Nvy_t + 4 - Nvy_in, - boundary_conditions.v.y[1], - boundary_conditions.v.y[2], - hy[1], - hy[end], - ) - - # Extend to 2D - Iv_vy3 = (I1D3 * Iv_vy_bc3.B1D) ⊗ mat_hx3 - Iv_vy_bc3 = (; Iv_vy_bc3..., Bbc = (I1D3 * Iv_vy_bc3.Btemp) ⊗ mat_hx3) - else - ## Iu_ux - diag1 = fill(weight, Nux_t - 1) - I1D = spdiagm(Nux_t - 1, Nux_t, 0 => diag1, 1 => diag1) - - # Boundary conditions - Iu_ux_bc = bc_general( - Nux_t, - Nux_in, - Nux_b, - boundary_conditions.u.x[1], - boundary_conditions.u.x[2], - hx[1], - hx[end], - ) - - # Extend to 2D - Iu_ux = mat_hy ⊗ (I1D * Iu_ux_bc.B1D) - Iu_ux_bc = (; Iu_ux_bc..., Bbc = mat_hy ⊗ (I1D * Iu_ux_bc.Btemp)) - - ## Iv_uy - diag1 = fill(weight, Nvx_t - 1) - I1D = spdiagm(Nvx_t - 1, Nvx_t, 0 => diag1, 1 => diag1) - boundary_conditions.u.x[1] == :pressure && (I1D[1, :] ./= 2) - boundary_conditions.u.x[2] == :pressure && (I1D[end, :] ./= 2) - - # The restriction is essentially 1D so it can be directly applied to I1D - I1D = Bvux * I1D * mat_hx2 - I2D = I(Nuy_t - 1) ⊗ I1D - - # Boundary conditions low/up - Nb = Nuy_in + 1 - Nvy_in - Iv_uy_bc_lu = bc_general( - Nuy_in + 1, - Nvy_in, - Nb, - boundary_conditions.v.y[1], - boundary_conditions.v.y[2], - hy[1], - hy[end], - ) - Iv_uy_bc_lu = (; Iv_uy_bc_lu..., B2D = Iv_uy_bc_lu.B1D ⊗ I(Nvx_in)) - Iv_uy_bc_lu = (; Iv_uy_bc_lu..., Bbc = Iv_uy_bc_lu.Btemp ⊗ I(Nvx_in)) - - # Boundary conditions left/right - Iv_uy_bc_lr = bc_general_stag( - Nvx_t, - Nvx_in, - Nvx_b, - boundary_conditions.v.x[1], - boundary_conditions.v.x[2], - hx[1], - hx[end], - ) - - # Take I2D into left/right operators for convenience - Iv_uy_bc_lr = (; Iv_uy_bc_lr..., B2D = I2D * (I(Nuy_t - 1) ⊗ Iv_uy_bc_lr.B1D)) - Iv_uy_bc_lr = (; Iv_uy_bc_lr..., Bbc = I2D * (I(Nuy_t - 1) ⊗ Iv_uy_bc_lr.Btemp)) - - # Resulting operator: - Iv_uy = Iv_uy_bc_lr.B2D * Iv_uy_bc_lu.B2D - - ## Interpolation operators, v-component - - ## Iu_vx - diag1 = fill(weight, Nuy_t - 1) - I1D = spdiagm(Nuy_t - 1, Nuy_t, 0 => diag1, 1 => diag1) - I1D = Buvy * I1D * mat_hy2 - I2D = I1D ⊗ I(Nvx_t - 1) - boundary_conditions.v.y[1] == :pressure && (I1D[1, :] ./= 2) - boundary_conditions.v.y[2] == :pressure && (I1D[end, :] ./= 2) - - # Boundary conditions low/up - Iu_vx_bc_lu = bc_general_stag( - Nuy_t, - Nuy_in, - Nuy_b, - boundary_conditions.u.y[1], - boundary_conditions.u.y[2], - hy[1], - hy[end], - ) - Iu_vx_bc_lu = (; Iu_vx_bc_lu..., B2D = I2D * (Iu_vx_bc_lu.B1D ⊗ I(Nvx_t - 1))) - Iu_vx_bc_lu = (; Iu_vx_bc_lu..., Bbc = I2D * (Iu_vx_bc_lu.Btemp ⊗ I(Nvx_t - 1))) - - # Boundary conditions left/right - Nb = Nvx_in + 1 - Nux_in - Iu_vx_bc_lr = bc_general( - Nvx_in + 1, - Nux_in, - Nb, - boundary_conditions.u.x[1], - boundary_conditions.u.x[2], - hx[1], - hx[end], - ) - - Iu_vx_bc_lr = (; Iu_vx_bc_lr..., B2D = I(Nuy_in) ⊗ Iu_vx_bc_lr.B1D) - Iu_vx_bc_lr = (; Iu_vx_bc_lr..., Bbc = I(Nuy_in) ⊗ Iu_vx_bc_lr.Btemp) - - # Resulting operator: - Iu_vx = Iu_vx_bc_lu.B2D * Iu_vx_bc_lr.B2D - - ## Iv_vy - diag1 = fill(weight, Nvy_t - 1) - I1D = spdiagm(Nvy_t - 1, Nvy_t, 0 => diag1, 1 => diag1) - - # Boundary conditions - Iv_vy_bc = bc_general( - Nvy_t, - Nvy_in, - Nvy_b, - boundary_conditions.v.y[1], - boundary_conditions.v.y[2], - hy[1], - hy[end], - ) - - # Extend to 2D - Iv_vy = (I1D * Iv_vy_bc.B1D) ⊗ mat_hx - Iv_vy_bc = (; Iv_vy_bc..., Bbc = (I1D * Iv_vy_bc.Btemp) ⊗ mat_hx) - end - - ## Group operators - operators = (; - Iu_ux, - Iv_uy, - Iu_vx, - Iv_vy, - Iu_ux_bc, - Iv_vy_bc, - Iv_uy_bc_lr, - Iv_uy_bc_lu, - Iu_vx_bc_lr, - Iu_vx_bc_lu, - ) - - if order4 - operators = (; - operators..., - Iu_ux3, - Iv_uy3, - Iu_vx3, - Iv_vy3, - Iu_ux_bc3, - Iv_uy_bc_lr3, - Iv_uy_bc_lu3, - Iu_vx_bc_lr3, - Iu_vx_bc_lu3, - Iv_vy_bc3, - ) - end - - operators -end - -# 3D version -function operator_interpolation(::Dimension{3}, grid, boundary_conditions) - (; Nx, Ny, Nz) = grid - (; Nux_in, Nux_b, Nux_t, Nuy_in, Nuy_b, Nuy_t, Nuz_in, Nuz_b, Nuz_t) = grid - (; Nvx_in, Nvx_b, Nvx_t, Nvy_in, Nvy_b, Nvy_t, Nvz_in, Nvz_b, Nvz_t) = grid - (; Nwx_in, Nwx_b, Nwx_t, Nwy_in, Nwy_b, Nwy_t, Nwz_in, Nwz_b, Nwz_t) = grid - (; hx, hy, hz, hxi, hyi, hzi) = grid - (; Buvy, Buwz, Bvux, Bvwz, Bwux, Bwvy) = grid - - T = eltype(hx) - - weight = T(1 / 2) - - mat_hx = spdiagm(Nx, Nx, hxi) - mat_hy = spdiagm(Ny, Ny, hyi) - mat_hz = spdiagm(Nz, Nz, hzi) - - # Periodic boundary conditions - if boundary_conditions.u.x == (:periodic, :periodic) - mat_hx2 = spdiagm(Nx + 2, Nx + 2, [hx[end]; hx; hx[1]]) - else - mat_hx2 = spdiagm(Nx + 2, Nx + 2, [hx[1]; hx; hx[end]]) - end - - if boundary_conditions.v.y == (:periodic, :periodic) - mat_hy2 = spdiagm(Ny + 2, Ny + 2, [hy[end]; hy; hy[1]]) - else - mat_hy2 = spdiagm(Ny + 2, Ny + 2, [hy[1]; hy; hy[end]]) - end - - if boundary_conditions.w.z == (:periodic, :periodic) - mat_hz2 = spdiagm(Nz + 2, Nz + 2, [hz[end]; hz; hz[1]]) - else - mat_hz2 = spdiagm(Nz + 2, Nz + 2, [hz[1]; hz; hz[end]]) - end - - ## Interpolation operators, u-component - - ## Iu_ux - diag1 = fill(weight, Nux_t - 1) - I1D = spdiagm(Nux_t - 1, Nux_t, 0 => diag1, 1 => diag1) - - # Boundary conditions - Iu_ux_bc = bc_general( - Nux_t, - Nux_in, - Nux_b, - boundary_conditions.u.x[1], - boundary_conditions.u.x[2], - hx[1], - hx[end], - ) - - # Extend to 3D - Iu_ux = mat_hz ⊗ mat_hy ⊗ (I1D * Iu_ux_bc.B1D) - Iu_ux_bc = (; Iu_ux_bc..., Bbc = mat_hz ⊗ mat_hy ⊗ (I1D * Iu_ux_bc.Btemp)) - - ## Iv_uy - diag1 = fill(weight, Nvx_t - 1) - I1D = spdiagm(Nvx_t - 1, Nvx_t, 0 => diag1, 1 => diag1) - boundary_conditions.u.x[1] == :pressure && (I1D[1, :] ./= 2) - boundary_conditions.u.x[2] == :pressure && (I1D[end, :] ./= 2) - - # The restriction is essentially 1D so it can be directly applied to I1D - I1D = Bvux * I1D * mat_hx2 - I3D = kron(mat_hz, kron(I(Nuy_t - 1), I1D)) - - # Boundary conditions low/up - Nb = Nuy_in + 1 - Nvy_in - Iv_uy_bc_lu = bc_general( - Nuy_in + 1, - Nvy_in, - Nb, - boundary_conditions.v.y[1], - boundary_conditions.v.y[2], - hy[1], - hy[end], - ) - Iv_uy_bc_lu = (; Iv_uy_bc_lu..., B3D = I(Nz) ⊗ Iv_uy_bc_lu.B1D ⊗ I(Nvx_in)) - Iv_uy_bc_lu = (; Iv_uy_bc_lu..., Bbc = I(Nz) ⊗ Iv_uy_bc_lu.Btemp ⊗ I(Nvx_in)) - - # Boundary conditions left/right - Iv_uy_bc_lr = bc_general_stag( - Nvx_t, - Nvx_in, - Nvx_b, - boundary_conditions.v.x[1], - boundary_conditions.v.x[2], - hx[1], - hx[end], - ) - - # Take I2D into left/right operators for convenience - Iv_uy_bc_lr = (; Iv_uy_bc_lr..., B3D = I3D * (I(Nz) ⊗ I(Nuy_t - 1) ⊗ Iv_uy_bc_lr.B1D)) - Iv_uy_bc_lr = (; Iv_uy_bc_lr..., Bbc = I3D * (I(Nz) ⊗ I(Nuy_t - 1) ⊗ Iv_uy_bc_lr.Btemp)) - - # Resulting operator: - Iv_uy = Iv_uy_bc_lr.B3D * Iv_uy_bc_lu.B3D - - ## Iw_uz - diag1 = fill(weight, Nwx_t - 1) - I1D = spdiagm(Nwx_t - 1, Nwx_t, 0 => diag1, 1 => diag1) - boundary_conditions.u.x[1] == :pressure && (I1D[1, :] ./= 2) - boundary_conditions.u.x[2] == :pressure && (I1D[end, :] ./= 2) - - I1D = Bwux * I1D * mat_hx2 - I3D = I(Nz + 1) ⊗ mat_hy ⊗ I1D - - # Boundary conditions left/right - Iw_uz_bc_lr = bc_general_stag( - Nwx_t, - Nwx_in, - Nwx_b, - boundary_conditions.w.x[1], - boundary_conditions.w.x[2], - hx[1], - hx[end], - ) - - # Take I3D into left/right operators for convenience - Iw_uz_bc_lr = (; Iw_uz_bc_lr..., B3D = I3D * (I(Nz + 1) ⊗ I(Ny) ⊗ Iw_uz_bc_lr.B1D)) - Iw_uz_bc_lr = (; Iw_uz_bc_lr..., Bbc = I3D * (I(Nz + 1) ⊗ I(Ny) ⊗ Iw_uz_bc_lr.Btemp)) - - # Boundary conditions back/front - Nb = Nuz_in + 1 - Nwz_in - Iw_uz_bc_bf = bc_general( - Nuz_in + 1, - Nwz_in, - Nb, - boundary_conditions.w.z[1], - boundary_conditions.w.z[2], - hz[1], - hz[end], - ) - Iw_uz_bc_bf = (; Iw_uz_bc_bf..., B3D = Iw_uz_bc_bf.B1D ⊗ I(Ny) ⊗ I(Nx)) - Iw_uz_bc_bf = (; Iw_uz_bc_bf..., Bbc = Iw_uz_bc_bf.Btemp ⊗ I(Ny) ⊗ I(Nx)) - - # Resulting operator: - Iw_uz = Iw_uz_bc_lr.B3D * Iw_uz_bc_bf.B3D - - ## Interpolation operators, v-component - - ## Iu_vx - diag1 = fill(weight, Nuy_t - 1) - I1D = spdiagm(Nuy_t - 1, Nuy_t, 0 => diag1, 1 => diag1) - boundary_conditions.v.y[1] == :pressure && (I1D[1, :] ./= 2) - boundary_conditions.v.y[2] == :pressure && (I1D[end, :] ./= 2) - - I1D = Buvy * I1D * mat_hy2 - I3D = mat_hz ⊗ I1D ⊗ I(Nx + 1) - - # Boundary conditions low/up - Iu_vx_bc_lu = bc_general_stag( - Nuy_t, - Nuy_in, - Nuy_b, - boundary_conditions.u.y[1], - boundary_conditions.u.y[2], - hy[1], - hy[end], - ) - Iu_vx_bc_lu = (; Iu_vx_bc_lu..., B3D = I3D * (I(Nz) ⊗ Iu_vx_bc_lu.B1D ⊗ I(Nx + 1))) - Iu_vx_bc_lu = (; Iu_vx_bc_lu..., Bbc = I3D * (I(Nz) ⊗ Iu_vx_bc_lu.Btemp ⊗ I(Nx + 1))) - - # Boundary conditions left/right - Nb = Nvx_in + 1 - Nux_in - Iu_vx_bc_lr = bc_general( - Nvx_in + 1, - Nux_in, - Nb, - boundary_conditions.u.x[1], - boundary_conditions.u.x[2], - hx[1], - hx[end], - ) - Iu_vx_bc_lr = (; Iu_vx_bc_lr..., B3D = I(Nz) ⊗ I(Ny) ⊗ Iu_vx_bc_lr.B1D) - Iu_vx_bc_lr = (; Iu_vx_bc_lr..., Bbc = I(Nz) ⊗ I(Ny) ⊗ Iu_vx_bc_lr.Btemp) - - # Resulting operator: - Iu_vx = Iu_vx_bc_lu.B3D * Iu_vx_bc_lr.B3D - - ## Iv_vy - diag1 = fill(weight, Nvy_t - 1) - I1D = spdiagm(Nvy_t - 1, Nvy_t, 0 => diag1, 1 => diag1) - - # Boundary conditions - Iv_vy_bc = bc_general( - Nvy_t, - Nvy_in, - Nvy_b, - boundary_conditions.v.y[1], - boundary_conditions.v.y[2], - hy[1], - hy[end], - ) - - # Extend to 3D - Iv_vy = mat_hz ⊗ (I1D * Iv_vy_bc.B1D) ⊗ mat_hx - Iv_vy_bc = (; Iv_vy_bc..., Bbc = mat_hz ⊗ (I1D * Iv_vy_bc.Btemp) ⊗ mat_hx) - - ## Iw_vz - diag1 = fill(weight, Nwy_t - 1) - I1D = spdiagm(Nwy_t - 1, Nwy_t, 0 => diag1, 1 => diag1) - boundary_conditions.v.y[1] == :pressure && (I1D[1, :] ./= 2) - boundary_conditions.v.y[2] == :pressure && (I1D[end, :] ./= 2) - - I1D = Bwvy * I1D * mat_hy2 - I3D = I(Nz + 1) ⊗ I1D ⊗ mat_hx - - # Boundary conditions low/up - Iw_vz_bc_lu = bc_general_stag( - Nwy_t, - Nwy_in, - Nwy_b, - boundary_conditions.w.y[1], - boundary_conditions.w.y[2], - hy[1], - hy[end], - ) - Iw_vz_bc_lu = (; Iw_vz_bc_lu..., B3D = I3D * (I(Nz + 1) ⊗ Iw_vz_bc_lu.B1D ⊗ I(Nx))) - Iw_vz_bc_lu = (; Iw_vz_bc_lu..., Bbc = I3D * (I(Nz + 1) ⊗ Iw_vz_bc_lu.Btemp ⊗ I(Nx))) - - # Boundary conditions left/right - Nb = Nvz_in + 1 - Nwz_in - Iw_vz_bc_bf = bc_general( - Nvz_in + 1, - Nwz_in, - Nb, - boundary_conditions.w.z[1], - boundary_conditions.w.z[2], - hz[1], - hz[end], - ) - Iw_vz_bc_bf = (; Iw_vz_bc_bf..., B3D = Iw_vz_bc_bf.B1D ⊗ I(Ny) ⊗ I(Nx)) - Iw_vz_bc_bf = (; Iw_vz_bc_bf..., Bbc = Iw_vz_bc_bf.Btemp ⊗ I(Ny) ⊗ I(Nx)) - - # Resulting operator: - Iw_vz = Iw_vz_bc_lu.B3D * Iw_vz_bc_bf.B3D - - ## Interpolation operators, w-component - - ## Iu_wx - diag1 = fill(weight, Nuz_t - 1) - I1D = spdiagm(Nuz_t - 1, Nuz_t, 0 => diag1, 1 => diag1) - boundary_conditions.w.z[1] == :pressure && (I1D[1, :] ./= 2) - boundary_conditions.w.z[2] == :pressure && (I1D[end, :] ./= 2) - - I1D = Buwz * I1D * mat_hz2 - I3D = I1D ⊗ mat_hy ⊗ I(Nx + 1) - - # Boundary conditions back/front - Iu_wx_bc_bf = bc_general_stag( - Nuz_t, - Nuz_in, - Nuz_b, - boundary_conditions.u.z[1], - boundary_conditions.u.z[2], - hz[1], - hz[end], - ) - Iu_wx_bc_bf = (; Iu_wx_bc_bf..., B3D = I3D * (Iu_wx_bc_bf.B1D ⊗ I(Ny) ⊗ I(Nx + 1))) - Iu_wx_bc_bf = (; Iu_wx_bc_bf..., Bbc = I3D * (Iu_wx_bc_bf.Btemp ⊗ I(Ny) ⊗ I(Nx + 1))) - - # Boundary conditions left/right - Nb = Nwx_in + 1 - Nux_in - Iu_wx_bc_lr = bc_general( - Nwx_in + 1, - Nux_in, - Nb, - boundary_conditions.u.x[1], - boundary_conditions.u.x[2], - hx[1], - hx[end], - ) - Iu_wx_bc_lr = (; Iu_wx_bc_lr..., B3D = I(Nz) ⊗ I(Ny) ⊗ Iu_wx_bc_lr.B1D) - Iu_wx_bc_lr = (; Iu_wx_bc_lr..., Bbc = I(Nz) ⊗ I(Ny) ⊗ Iu_wx_bc_lr.Btemp) - - # Resulting operator: - Iu_wx = Iu_wx_bc_bf.B3D * Iu_wx_bc_lr.B3D - - ## Iv_wy - diag1 = fill(weight, Nvz_t - 1) - I1D = spdiagm(Nvz_t - 1, Nvz_t, 0 => diag1, 1 => diag1) - boundary_conditions.w.z[1] == :pressure && (I1D[1, :] ./= 2) - boundary_conditions.w.z[2] == :pressure && (I1D[end, :] ./= 2) - - I1D = Bvwz * I1D * mat_hz2 - I3D = I1D ⊗ I(Ny + 1) ⊗ mat_hx - - # Boundary conditions back/front - Iv_wy_bc_bf = bc_general_stag( - Nvz_t, - Nvz_in, - Nvz_b, - boundary_conditions.v.z[1], - boundary_conditions.v.z[2], - hz[1], - hz[end], - ) - Iv_wy_bc_bf = (; Iv_wy_bc_bf..., B3D = I3D * (Iv_wy_bc_bf.B1D ⊗ I(Ny + 1) ⊗ I(Nx))) - Iv_wy_bc_bf = (; Iv_wy_bc_bf..., Bbc = I3D * (Iv_wy_bc_bf.Btemp ⊗ I(Ny + 1) ⊗ I(Nx))) - - # Boundary conditions low/up - Nb = Nwy_in + 1 - Nvy_in - Iv_wy_bc_lu = bc_general( - Nwy_in + 1, - Nvy_in, - Nb, - boundary_conditions.v.y[1], - boundary_conditions.v.y[2], - hy[1], - hy[end], - ) - Iv_wy_bc_lu = (; Iv_wy_bc_lu..., B3D = I(Nz) ⊗ Iv_wy_bc_lu.B1D ⊗ I(Nx)) - Iv_wy_bc_lu = (; Iv_wy_bc_lu..., Bbc = I(Nz) ⊗ Iv_wy_bc_lu.Btemp ⊗ I(Nx)) - - # Resulting operator: - Iv_wy = Iv_wy_bc_bf.B3D * Iv_wy_bc_lu.B3D - - ## Iw_wz - diag1 = fill(weight, Nwz_t - 1) - I1D = spdiagm(Nwz_t - 1, Nwz_t, 0 => diag1, 1 => diag1) - - # Boundary conditions - Iw_wz_bc = bc_general( - Nwz_t, - Nwz_in, - Nwz_b, - boundary_conditions.w.z[1], - boundary_conditions.w.z[2], - hz[1], - hz[end], - ) - - # Extend to 3D - Iw_wz = (I1D * Iw_wz_bc.B1D) ⊗ mat_hy ⊗ mat_hx - Iw_wz_bc = (; Iw_wz_bc..., Bbc = (I1D * Iw_wz_bc.Btemp) ⊗ mat_hy ⊗ mat_hx) - - ## Group operators - (; - Iu_ux, - Iv_uy, - Iw_uz, - Iu_vx, - Iv_vy, - Iw_vz, - Iu_wx, - Iv_wy, - Iw_wz, - Iu_ux_bc, - Iv_vy_bc, - Iw_wz_bc, - Iv_uy_bc_lr, - Iv_uy_bc_lu, - Iu_vx_bc_lr, - Iu_vx_bc_lu, - Iw_uz_bc_lr, - Iw_uz_bc_bf, - Iw_vz_bc_lu, - Iw_vz_bc_bf, - Iu_wx_bc_lr, - Iu_wx_bc_bf, - Iv_wy_bc_lu, - Iv_wy_bc_bf, - ) -end diff --git a/src/operators/operator_postprocessing.jl b/src/operators/operator_postprocessing.jl deleted file mode 100644 index 8ee8869af..000000000 --- a/src/operators/operator_postprocessing.jl +++ /dev/null @@ -1,173 +0,0 @@ -""" - operator_postprocessing(grid, boundary_conditions) - -Construct postprocessing operators such as vorticity. -""" -operator_postprocessing(grid, boundary_conditions) = - operator_postprocessing(grid.dimension, grid, boundary_conditions) - -# 2D version -function operator_postprocessing(::Dimension{2}, grid, boundary_conditions) - (; Nx, Ny, gx, gy, gxd, gyd) = grid - - T = eltype(gx) - - if all(==(:periodic), (boundary_conditions.u.x[1], boundary_conditions.v.y[1])) - # For entirely periodic BC, covering entire mesh - - # dv/dx, like Sv_vx - diag = 1 ./ gxd - ∂x = spdiagm( - Nx + 1, - Nx, - -Nx => diag[[1]], - -1 => -diag[1:(end-1)], - 0 => diag[1:(end-1)], - Nx - 1 => -diag[[end - 1]], - ) - weight = T(1) - repeat_x = spdiagm(Nx + 1, Nx, -Nx => [weight], 0 => fill(weight, Nx)) - - # du/dy, like Su_uy - diag = 1 ./ gyd - ∂y = spdiagm( - Ny + 1, - Ny, - -Ny => diag[[1]], - -1 => -diag[1:(end-1)], - 0 => diag[1:(end-1)], - Ny - 1 => -diag[[end - 1]], - ) - weight = T(1) - repeat_y = spdiagm(Ny + 1, Ny, -Ny => [weight], 0 => fill(weight, Ny)) - else - # dv/dx, like Sv_vx - diag = 1 ./ gy[2:(end-1)] - ∂y = spdiagm(Ny - 1, Ny, 0 => -diag, 1 => diag) - repeat_y = I(Ny - 1) - - # du/dy, like Su_uy - diag = 1 ./ gx[2:(end-1)] - ∂x = spdiagm(Nx - 1, Nx, 0 => -diag, 1 => diag) - repeat_x = I(Nx - 1) - end - - # Extend to 2D - Wu_uy = ∂y ⊗ repeat_x - Wv_vx = repeat_y ⊗ ∂x - - (; Wv_vx, Wu_uy) -end - -# 3D version -function operator_postprocessing(::Dimension{3}, grid, boundary_conditions) - (; Nx, Ny, Nz, gx, gy, gz, gxd, gyd, gzd) = grid - - T = eltype(gx) - - if all( - ==(:periodic), - ( - boundary_conditions.u.x[1], - boundary_conditions.v.y[1], - boundary_conditions.w.z[1], - ), - ) - # For entirely periodic BC, covering entire mesh - - diag = 1 ./ gxd - ∂x = spdiagm( - Nx + 1, - Nx, - -Nx => diag[[1]], - -1 => -diag[1:(end-1)], - 0 => diag[1:(end-1)], - Nx - 1 => -diag[[end - 1]], - ) - # FIXME: nonuniform weights: 1/gi / (1/gi + 1/gj) ? - weight = T(1 // 2) - average_x = spdiagm( - Nx + 1, - Nx, - -Nx => [weight], - -1 => fill(weight, Nx), - 0 => fill(weight, Nx), - Nx - 1 => [weight], - ) - weight = T(1) - repeat_x = spdiagm(Nx + 1, Nx, -Nx => [weight], 0 => fill(weight, Nx)) - - diag = 1 ./ gyd - ∂y = spdiagm( - Ny + 1, - Ny, - -Ny => diag[[1]], - -1 => -diag[1:(end-1)], - 0 => diag[1:(end-1)], - Ny - 1 => -diag[[end - 1]], - ) - weight = T(1 // 2) - average_y = spdiagm( - Ny + 1, - Ny, - -Ny => [weight], - -1 => fill(weight, Ny), - 0 => fill(weight, Ny), - Ny - 1 => [weight], - ) - weight = T(1) - repeat_y = spdiagm(Ny + 1, Ny, -Ny => [weight], 0 => fill(weight, Ny)) - - diag = 1 ./ gzd - ∂z = spdiagm( - Nz + 1, - Nz, - -Nz => diag[[1]], - -1 => -diag[1:(end-1)], - 0 => diag[1:(end-1)], - Nz - 1 => -diag[[end - 1]], - ) - weight = T(1 // 2) - average_z = spdiagm( - Nz + 1, - Nz, - -Nz => [weight], - -1 => fill(weight, Nz), - 0 => fill(weight, Nz), - Nz - 1 => [weight], - ) - weight = T(1) - repeat_z = spdiagm(Nz + 1, Nz, -Nz => [weight], 0 => fill(weight, Nz)) - else - diag = 1 ./ gx[2:(end-1)] - ∂x = spdiagm(Nx - 1, Nx, 0 => -diag, 1 => diag) - diag = fill(T(1 // 2), Nx - 1) - average_x = spdiagm(Nx - 1, Nx, 0 => diag, 1 => diag) - # average_x = sparse(I, Nx - 1, Nx) - repeat_x = I(Nx - 1) - - diag = 1 ./ gy[2:(end-1)] - ∂y = spdiagm(Ny - 1, Ny, 0 => -diag, 1 => diag) - diag = fill(T(1 // 2), Ny - 1) - average_y = spdiagm(Ny - 1, Ny, 0 => diag, 1 => diag) - # average_y = sparse(I, Ny - 1, Ny) - repeat_y = I(Ny - 1) - - diag = 1 ./ gz[2:(end-1)] - ∂z = spdiagm(Nz - 1, Nz, 0 => -diag, 1 => diag) - diag = fill(T(1 / 2), Nz - 1) - average_z = spdiagm(Nz - 1, Nz, 0 => diag, 1 => diag) - # average_z = sparse(I, Nz - 1, Nz) - repeat_z = I(Nz - 1) - end - - # Extend to 3D - Wu_uy = average_z ⊗ ∂y ⊗ repeat_x - Wu_uz = ∂z ⊗ average_y ⊗ repeat_x - Wv_vx = average_z ⊗ repeat_y ⊗ ∂x - Wv_vz = ∂z ⊗ repeat_y ⊗ average_x - Ww_wy = repeat_z ⊗ ∂y ⊗ average_x - Ww_wx = repeat_z ⊗ average_y ⊗ ∂x - - (; Wu_uy, Wu_uz, Wv_vx, Wv_vz, Ww_wy, Ww_wx) -end diff --git a/src/operators/operator_regularization.jl b/src/operators/operator_regularization.jl deleted file mode 100644 index ba309a553..000000000 --- a/src/operators/operator_regularization.jl +++ /dev/null @@ -1,49 +0,0 @@ -""" - operator_regularization(grid, operators) - -Build regularization matrices. -""" -operator_regularization(grid, boundary_conditions) = - operator_regularization(grid.dimension, grid, boundary_conditions) - -# 2D version -function operator_regularization(::Dimension{2}, grid, operators) - (; Ω, indu, indv) = grid - (; Dux, Duy, Dvx, Dvy) = operators - (; Su_ux, Su_uy, Sv_vx, Sv_vy) = operators - - Δ = max_size(grid) - α_reg = Δ^2 / 16 - - Ωu⁻¹ = 1 ./ Ω[indu] - Ωv⁻¹ = 1 ./ Ω[indv] - - # Diffusive matrices in finite-difference setting, without viscosity - Diffu_f = Diagonal(Ωu⁻¹) * (Dux * Su_ux + Duy * Su_uy) - Diffv_f = Diagonal(Ωv⁻¹) * (Dvx * Sv_vx + Dvy * Sv_vy) - - (; Diffu_f, Diffv_f, α_reg) -end - -# 3D version -function operator_regularization(::Dimension{3}, grid, operators) - (; Ω, indu, indv, indw) = grid - (; Dux, Duy, Duz, Dvx, Dvy, Dvz, Dwx, Dwy, Dwz) = operators - (; Su_ux, Su_uy, Su_uz) = operators - (; Sv_vx, Sv_vy, Sv_vz) = operators - (; Sw_wx, Sw_wy, Sw_wz) = operators - - Δ = max_size(grid) - α_reg = Δ^2 / 16 - - Ωu⁻¹ = 1 ./ Ω[indu] - Ωv⁻¹ = 1 ./ Ω[indv] - Ωw⁻¹ = 1 ./ Ω[indw] - - # Diffusive matrices in finite-difference setting, without viscosity - Diffu_f = Diagonal(Ωu⁻¹) * (Dux * Su_ux + Duy * Su_uy + Duz * Su_uz) - Diffv_f = Diagonal(Ωv⁻¹) * (Dvx * Sv_vx + Dvy * Sv_vy + Dvz * Sv_vz) - Diffw_f = Diagonal(Ωw⁻¹) * (Dwx * Sw_wx + Dwy * Sw_wy + Dwz * Sw_wz) - - (; Diffu_f, Diffv_f, Diffw_f, α_reg) -end diff --git a/src/operators/operator_turbulent_diffusion.jl b/src/operators/operator_turbulent_diffusion.jl deleted file mode 100644 index fd8423ccf..000000000 --- a/src/operators/operator_turbulent_diffusion.jl +++ /dev/null @@ -1,309 +0,0 @@ -""" - operator_turbulent_diffusion(grid, boundary_conditions) - -Average (turbulent) viscosity to cell faces: from `ν` at `xp`, `yp` to `ν` at `ux`, `uy`, -`vx`, `vy` locations. -""" -operator_turbulent_diffusion(grid, boundary_conditions) = - operator_turbulent_diffusion(grid.dimension, grid, boundary_conditions) - -# 2D version -function operator_turbulent_diffusion(::Dimension{2}, grid, boundary_conditions) - (; Npx, Npy) = grid - (; Nux_in, Nuy_in, Nvx_in, Nvy_in) = grid - (; hx, hy, gxd, gyd) = grid - (; Buvy, Bvux, Bkux, Bkvy) = grid - - T = eltype(hx) - - # Averaging weight: - weight = T(1 / 2) - - ## Nu to ux positions - - A1D = I(Npx + 2) - A1D = Bkux * A1D - - # Boundary conditions for ν; mapping from Npx (k) points to Npx+2 points - Aν_ux_bc = bc_general_stag( - Npx + 2, - Npx, - 2, - boundary_conditions.ν.x[1], - boundary_conditions.ν.x[2], - hx[1], - hx[end], - ) - # Then map back to Nux_in+1 points (ux-faces) with Bkux - - # Extend to 2D - Aν_ux = I(Nuy_in) ⊗ (A1D * Aν_ux_bc.B1D) - Aν_ux_bc = (; Aν_ux_bc..., Bbc = I(Nuy_in) ⊗ (A1D * Aν_ux_bc.Btemp)) - - ## Nu to uy positions - - # Average in x-direction - diag1 = fill(weight, Npx + 1) - A1D = spdiagm(Npx + 1, Npx + 2, 0 => diag1, 1 => diag1) - # Then map to Nux_in points (like Iv_uy) with Bvux - A1D = Bvux * A1D - - # Calculate average in y-direction; no boundary conditions - diag1 = fill(weight, Npy + 1) - A1Dy = spdiagm(Npy + 1, Npy + 2, 0 => diag1, 1 => diag1) - A2Dy = A1Dy ⊗ I(Nux_in) - - # Boundary conditions for ν in x-direction; - # Mapping from Npx (ν) points to Npx+2 points - Aν_uy_bc_lr = bc_general_stag( - Npx + 2, - Npx, - 2, - boundary_conditions.ν.x[1], - boundary_conditions.ν.x[2], - hx[1], - hx[end], - ) - - # Extend BC to 2D - A2D = I(Npy + 2) ⊗ (A1D * Aν_uy_bc_lr.B1D) - - # Apply BC in y-direction - Aν_uy_bc_lu = bc_general_stag( - Npy + 2, - Npy, - 2, - boundary_conditions.ν.y[1], - boundary_conditions.ν.y[2], - hy[1], - hy[end], - ) - - A2Dx = A2D * (Aν_uy_bc_lu.B1D ⊗ I(Npx)) - - Aν_uy = A2Dy * A2Dx - - Aν_uy_bc_lr = (; Aν_uy_bc_lr..., B2D = A2Dy * (I(Npy + 2) ⊗ (A1D * Aν_uy_bc_lr.Btemp))) - Aν_uy_bc_lu = (; Aν_uy_bc_lu..., B2D = A2Dy * A2D * (Aν_uy_bc_lu.Btemp ⊗ I(Npx))) - - ## Nu to vx positions - diag1 = fill(weight, Npy + 1) - A1D = spdiagm(Npy + 1, Npy + 2, 0 => diag1, 1 => diag1) - - # Map to Nvy_in points (like Iu_vx) with Buvy - A1D = Buvy * A1D - - # Calculate average in x-direction; no boundary conditions - diag1 = fill(weight, Npx + 1) - A1Dx = spdiagm(Npx + 1, Npx + 2, 0 => diag1, 1 => diag1) - A2Dx = kron(I(Nvy_in), A1Dx) - - # Boundary conditions for ν in y-direction; - # Mapping from Npy (ν) points to Npy+2 points - Aν_vx_bc_lu = bc_general_stag( - Npy + 2, - Npy, - 2, - boundary_conditions.ν.y[1], - boundary_conditions.ν.y[2], - hy[1], - hy[end], - ) - - # Extend BC to 2D - A2D = (A1D * Aν_vx_bc_lu.B1D) ⊗ I(Npx + 2) - - # Apply boundary conditions also in x-direction: - Aν_vx_bc_lr = bc_general_stag( - Npx + 2, - Npx, - 2, - boundary_conditions.ν.x[1], - boundary_conditions.ν.x[2], - hx[1], - hx[end], - ) - - A2Dy = A2D * (I(Npy) ⊗ Aν_vx_bc_lr.B1D) - - Aν_vx = A2Dx * A2Dy - - Aν_vx_bc_lu = (; Aν_vx_bc_lu..., B2D = A2Dx * ((A1D * Aν_vx_bc_lu.Btemp) ⊗ I(Npx + 2))) - Aν_vx_bc_lr = (; Aν_vx_bc_lr..., B2D = A2Dx * A2D * (I(Npy) ⊗ Aν_vx_bc_lr.Btemp)) - - ## Nu to vy positions - A1D = I(Npy + 2) - # Then map back to Nvy_in+1 points (vy-faces) with Bkvy - A1D = Bkvy * A1D - - # Boundary conditions for ν; mapping from Npy (ν) points to Npy+2 (vy faces) points - Aν_vy_bc = bc_general_stag( - Npy + 2, - Npy, - 2, - boundary_conditions.ν.y[1], - boundary_conditions.ν.y[2], - hy[1], - hy[end], - ) - - # Extend to 2D - Aν_vy = (A1D * Aν_vy_bc.B1D) ⊗ I(Nvx_in) - Aν_vy_bc = (; Aν_vy_bc..., Bbc = ((A1D * Aν_vy_bc.Btemp) ⊗ I(Nvx_in))) - - # So ν at vy is given by: - # (Aν_vy * k + yAν_vy) - - ## Get derivatives u_x, u_y, v_x and v_y at cell centers - # Differencing velocity to ν-points - - ## Du/dx - - # Differencing matrix - diag1 = 1 ./ hx - C1D = spdiagm(Npx, Npx + 1, 0 => -diag1, 1 => diag1) - - # Boundary conditions - Cux_k_bc = bc_general( - Npx + 1, - Nux_in, - Npx + 1 - Nux_in, - boundary_conditions.u.x[1], - boundary_conditions.u.x[2], - hx[1], - hx[end], - ) - - Cux_k = I(Npy) ⊗ (C1D * Cux_k_bc.B1D) - Cux_k_bc = (; Cux_k_bc..., Bbc = I(Npy) ⊗ (C1D * Cux_k_bc.Btemp)) - - # Cux_k*uₕ+yCux_k; - - ## Du/dy - - # Average to k-positions (in x-dir) - weight = T(1 / 2) - diag1 = fill(weight, Npx) - A1D = spdiagm(Npx, Npx + 1, 0 => diag1, 1 => diag1) - - # Boundary conditions - Auy_k_bc = bc_general( - Npx + 1, - Nux_in, - Npx + 1 - Nux_in, - boundary_conditions.u.x[1], - boundary_conditions.u.x[2], - hx[1], - hx[end], - ) - - Auy_k = I(Npy) ⊗ (A1D * Auy_k_bc.B1D) - Auy_k_bc = (; Auy_k_bc..., Bbc = I(Npy) ⊗ (A1D * Auy_k_bc.Btemp)) - - # Take differences - gydnew = gyd[1:end-1] + gyd[2:end] # Differencing over 2*Δy - diag2 = 1 ./ gydnew - C1D = spdiagm(Npy, Npy + 2, 0 => -diag2, 2 => diag2) - - Cuy_k_bc = bc_general_stag( - Npy + 2, - Npy, - 2, - boundary_conditions.u.y[1], - boundary_conditions.u.y[2], - hy[1], - hy[end], - ) - - Cuy_k = (C1D * Cuy_k_bc.B1D) ⊗ I(Npx) - Cuy_k_bc = (; Cuy_k_bc..., Bbc = (C1D * Cuy_k_bc.Btemp) ⊗ I(Npx)) - - ## Dv/dx - - # Average to k-positions (in y-dir) - weight = T(1 / 2) - diag1 = fill(weight, Npy) - A1D = spdiagm(Npy, Npy + 1, 0 => diag1, 1 => diag1) - - # Boundary conditions - Avx_k_bc = bc_general( - Npy + 1, - Nvy_in, - Npy + 1 - Nvy_in, - boundary_conditions.v.y[1], - boundary_conditions.v.y[2], - hy[1], - hy[end], - ) - Avx_k = (A1D * Avx_k_bc.B1D) ⊗ I(Npx) - Avx_k_bc = (; Avx_k_bc..., Bbc = (A1D * Avx_k_bc.Btemp) ⊗ I(Npx)) - - # Take differences - gxdnew = gxd[1:end-1] + gxd[2:end] # Differencing over 2*Δx - diag2 = 1 ./ gxdnew - C1D = spdiagm(Npx, Npx + 2, 0 => -diag2, 2 => diag2) - - Cvx_k_bc = bc_general_stag( - Npx + 2, - Npx, - Npx + 2 - Npx, - boundary_conditions.v.x[1], - boundary_conditions.v.x[2], - hx[1], - hx[end], - ) - - Cvx_k = I(Npy) ⊗ (C1D * Cvx_k_bc.B1D) - Cvx_k_bc = (; Cvx_k_bc..., Bbc = I(Npy) ⊗ (C1D * Cvx_k_bc.Btemp)) - - ## Dv/dy - - # Differencing matrix - diag1 = 1 ./ hy - C1D = spdiagm(Npy, Npy + 1, 0 => -diag1, 1 => diag1) - - # Boundary conditions - Cvy_k_bc = bc_general( - Npy + 1, - Nvy_in, - Npy + 1 - Nvy_in, - boundary_conditions.v.y[1], - boundary_conditions.v.y[2], - hy[1], - hy[end], - ) - - Cvy_k = (C1D * Cvy_k_bc.B1D) ⊗ I(Npx) - Cvy_k_bc = (; Cvy_k_bc..., Bbc = (C1D * Cvy_k_bc.Btemp) ⊗ I(Npx)) - - ## Group operators - (; - Aν_ux, - Aν_ux_bc, - Aν_uy, - Aν_uy_bc_lr, - Aν_uy_bc_lu, - Aν_vx, - Aν_vx_bc_lr, - Aν_vx_bc_lu, - Aν_vy, - Aν_vy_bc, - Cux_k, - Cux_k_bc, - Cuy_k, - Cuy_k_bc, - Cvx_k, - Cvx_k_bc, - Cvy_k, - Cvy_k_bc, - Auy_k, - Auy_k_bc, - Avx_k, - Avx_k_bc, - ) -end - -# TODO: 3D implementation -function operator_turbulent_diffusion(::Dimension{3}, grid, boundary_conditions) - error("Not implemented (3D)") -end diff --git a/src/operators/operator_viscosity.jl b/src/operators/operator_viscosity.jl deleted file mode 100644 index d066c54db..000000000 --- a/src/operators/operator_viscosity.jl +++ /dev/null @@ -1,16 +0,0 @@ -""" - operator_viscosity(viscosity_model, grid, boundary_conditions) - -Classical turbulence modelling via the diffusive term -""" -function operator_viscosity end - -operator_viscosity(::LaminarModel, grid, boundary_conditions) = (;) - -function operator_viscosity( - ::Union{QRModel,SmagorinskyModel,MixingLengthModel}, - grid, - boundary_conditions, -) - operator_turbulent_diffusion(grid, boundary_conditions) -end diff --git a/src/operators/operators.jl b/src/operators/operators.jl deleted file mode 100644 index 887646356..000000000 --- a/src/operators/operators.jl +++ /dev/null @@ -1,33 +0,0 @@ -""" - Operators(grid, boundary_conditions, viscosity_model) - -Build operators. -""" -function Operators(grid, boundary_conditions) - # Averaging operators - op_ave = operator_averaging(grid, boundary_conditions) - - # Interpolation operators - op_int = operator_interpolation(grid, boundary_conditions) - - # Divergence (u, v) -> p and gradient p -> (u, v) operator - op_div = operator_divergence(grid, boundary_conditions) - - # Convection operators on u- and v- centered volumes - op_con = operator_convection_diffusion(grid, boundary_conditions) - - # Regularization modelling - this changes the convective term - op_reg = operator_regularization(grid, op_con) - - # # Classical turbulence modelling via the diffusive term - # op_vis = operator_viscosity(viscosity_model, grid, boundary_conditions) - - # Classical turbulence modelling via the diffusive term - # Note: We build turbulent diffusion operator even for laminar model - op_vis = operator_turbulent_diffusion(grid, boundary_conditions) - - # Post-processing - op_pos = operator_postprocessing(grid, boundary_conditions) - - (; op_ave..., op_int..., op_div..., op_con..., op_reg..., op_vis..., op_pos...) -end diff --git a/src/postprocess/get_streamfunction.jl b/src/postprocess/get_streamfunction.jl deleted file mode 100644 index 434f81c71..000000000 --- a/src/postprocess/get_streamfunction.jl +++ /dev/null @@ -1,110 +0,0 @@ -""" - get_streamfunction(setup, V, t) - -Compute streamfunction ``\\psi`` from a Poisson equation ``\\nabla^2 \\psi = -\\omega``. -""" -function get_streamfunction end - -get_streamfunction(setup, V, t) = get_streamfunction(setup.grid.dimension, setup, V, t) - -# 2D version -function get_streamfunction(::Dimension{2}, setup, V, t) - (; grid, operators, boundary_conditions) = setup - (; u_bc, v_bc) = boundary_conditions - (; indu, indv, Nux_in, Nvx_in, Nx, Ny) = grid - (; hx, hy, x, y, xp, yp) = grid - (; Wv_vx, Wu_uy) = operators - - uₕ = @view V[indu] - vₕ = @view V[indv] - - # Boundary values by integrating around domain - # Start with ψ = 0 at lower left corner - - # u = d ψ / dy; integrate low->up - if boundary_conditions.u.x[1] == :dirichlet - # u1 = interp1(y, uLe, yp); - u1 = u_bc.(x[1], yp, t) - elseif boundary_conditions.u.x[1] ∈ (:pressure, :periodic) - u1 = uₕ[1:Nux_in:end] - end - ψLe = cumsum(hy .* u1) - ψUpLe = ψLe[end] - ψLe = ψLe[1:(end-1)] - - # v = -d ψ / dx; integrate left->right - if boundary_conditions.v.y[2] == :dirichlet - v1 = v_bc.(xp, y[end], t) - elseif boundary_conditions.v.y[2] == :pressure - v1 = vₕ[(end-Nvx_in+1):end] - elseif boundary_conditions.v.y[2] == :periodic - v1 = vₕ[1:Nvx_in] - end - ψUp = ψUpLe .- cumsum(hx .* v1) - ψUpRi = ψUp[end] - ψUp = ψUp[1:(end-1)] - - # u = d ψ / dy; integrate up->lo - if boundary_conditions.u.x[2] == :dirichlet - u2 = u_bc.(x[end], yp, t) - elseif boundary_conditions.u.x[2] == :pressure - u2 = uₕ[Nux_in:Nux_in:end] - elseif boundary_conditions.u.x[2] == :periodic - u2 = uₕ[1:Nux_in:end] - end - ψRi = ψUpRi .- cumsum(hy[end:-1:1] .* u2[end:-1:1]) - ψLoRi = ψRi[end] - ψRi = ψRi[(end-1):-1:1] - - # v = -d ψ / dx; integrate right->left - if boundary_conditions.v.y[1] == :dirichlet - v2 = v_bc.(xp, y[1], t) - elseif boundary_conditions.v.y[1] ∈ (:pressure, :periodic) - v2 = vₕ[1:Nvx_in] - end - ψLo = ψLoRi .+ cumsum(hx[end:-1:1] .* v2[end:-1:1]) - ψLoLe = ψLo[end] - ψLo = ψLo[(end-1):-1:1] - - abs(ψLoLe) > 1e-12 && @warn "Contour integration of ψ not consistent" abs(ψLoLe) - - # Solve del^2 ψ = -ω - # Only dirichlet boundary conditions because we calculate streamfunction at - # Inner points only - - # X-direction - diag = 1 ./ hx - Q1D = spdiagm(Nx, Nx + 1, 0 => -diag, 1 => diag) - Qx_bc = bc_general(Nx + 1, Nx - 1, 2, :dirichlet, :dirichlet, hx[1], hx[end]) - - # Extend to 2D - Q2Dx = I(Ny - 1) ⊗ (Q1D * Qx_bc.B1D) - yQx = (I(Ny - 1) ⊗ (Q1D * Qx_bc.Btemp)) * (ψLe ⊗ Qx_bc.ybc1 + ψRi ⊗ Qx_bc.ybc2) - - # Y-direction - diag = 1 ./ hy - Q1D = spdiagm(Ny, Ny + 1, 0 => -diag, 1 => diag) - Qy_bc = bc_general(Ny + 1, Ny - 1, 2, :dirichlet, :dirichlet, hy[1], hy[end]) - - # Extend to 2D - Q2Dy = (Q1D * Qy_bc.B1D) ⊗ I(Nx - 1) - yQy = ((Q1D * Qy_bc.Btemp) ⊗ I(Nx - 1)) * (Qy_bc.ybc1 ⊗ ψLo + Qy_bc.ybc2 ⊗ ψUp) - - # FIXME: Dimension error in periodic case - # @show size(Wv_vx) size(Q2Dx) size(Wu_uy) size(Q2Dy) - Aψ = Wv_vx * Q2Dx + Wu_uy * Q2Dy - yAψ = Wv_vx * yQx + Wu_uy * yQy - - ω = get_vorticity(setup, V, t) - ω_flat = reshape(ω, length(ω)) - - # Solve streamfunction from Poisson equation - ψ = -Aψ \ (ω_flat + yAψ) - - reshape(ψ, size(ω)...) -end - -# 3D version -function get_streamfunction(::Dimension{3}, setup, V, t) - error("Not implemented") -end diff --git a/src/postprocess/get_velocity.jl b/src/postprocess/get_velocity.jl deleted file mode 100644 index 094ef5961..000000000 --- a/src/postprocess/get_velocity.jl +++ /dev/null @@ -1,49 +0,0 @@ -""" - get_velocity(setup, V, t) - -Get velocity values at pressure points. Interpolate velocities to pressure positions using -`BMx` and `BMy` (and `BMz`), constructed in operator_divergence.jl. -""" -function get_velocity end - -get_velocity(setup, V, t) = get_velocity(setup.grid.dimension, setup, V, t) - -# 2D version -function get_velocity(::Dimension{2}, setup, V, t) - (; grid, operators) = setup - (; Npx, Npy, indu, indv) = grid - (; Au_ux, Av_vy, Bup, Bvp) = operators - - # Evaluate boundary conditions at current time - bc_vectors = get_bc_vectors(setup, t) - (; yAu_ux, yAv_vy) = bc_vectors - - uh = @view V[indu] - vh = @view V[indv] - - up = reshape(Bup * (Au_ux * uh + yAu_ux), Npx, Npy) - vp = reshape(Bvp * (Av_vy * vh + yAv_vy), Npx, Npy) - - up, vp -end - -# 3D version -function get_velocity(::Dimension{3}, setup, V, t) - (; grid, operators) = setup - (; Au_ux, Av_vy, Aw_wz, Bup, Bvp, Bwp) = operators - (; Npx, Npy, Npz, indu, indv, indw) = grid - - # Evaluate boundary conditions at current time - bc_vectors = get_bc_vectors(setup, t) - (; yAu_ux, yAv_vy, yAw_wz) = bc_vectors - - uh = @view V[indu] - vh = @view V[indv] - wh = @view V[indw] - - up = reshape(Bup * (Au_ux * uh + yAu_ux), Npx, Npy, Npz) - vp = reshape(Bvp * (Av_vy * vh + yAv_vy), Npx, Npy, Npz) - wp = reshape(Bwp * (Aw_wz * wh + yAw_wz), Npx, Npy, Npz) - - up, vp, wp -end diff --git a/src/postprocess/get_vorticity.jl b/src/postprocess/get_vorticity.jl deleted file mode 100644 index 72ff1d195..000000000 --- a/src/postprocess/get_vorticity.jl +++ /dev/null @@ -1,156 +0,0 @@ -""" - get_vorticity(setup, V, t) - -Get vorticity from velocity field. -""" -function get_vorticity end - -get_vorticity(setup, V, t) = get_vorticity(setup.grid.dimension, setup, V, t) - -# 2D version -function get_vorticity(::Dimension{2}, setup, V, t) - (; grid, boundary_conditions) = setup - (; Nx, Ny) = grid - - if all(==(:periodic), (boundary_conditions.u.x[1], boundary_conditions.v.y[1])) - Nωx = Nx + 1 - Nωy = Ny + 1 - else - Nωx = Nx - 1 - Nωy = Ny - 1 - end - - ω = similar(V, Nωx, Nωy) - - vorticity!(ω, setup, V, t) -end - -# 3D version -function get_vorticity(::Dimension{3}, setup, V, t) - (; grid, boundary_conditions) = setup - (; Nx, Ny, Nz) = grid - - if all( - ==(:periodic), - ( - boundary_conditions.u.x[1], - boundary_conditions.v.y[1], - boundary_conditions.w.z[1], - ), - ) - Nωx = Nx + 1 - Nωy = Ny + 1 - Nωz = Nz + 1 - else - Nωx = Nx - 1 - Nωy = Ny - 1 - Nωz = Nz - 1 - end - - ω = similar(V, Nωx, Nωy, Nωz) - - vorticity!(ω, setup, V, t) -end - -""" - vorticity!(ω, V, t, setup) - -Compute vorticity values at pressure midpoints. -This should be consistent with `operator_postprocessing.jl`. -""" -function vorticity! end - -vorticity!(ω, setup, V, t) = vorticity!(setup.grid.dimension, ω, setup, V, t) - -# 2D version -function vorticity!(::Dimension{2}, ω, setup, V, t) - (; grid, operators, boundary_conditions) = setup - (; indu, indv, Nux_in, Nvy_in, Nx, Ny) = grid - (; Wv_vx, Wu_uy) = operators - - T = eltype(V) - - uₕ = @view V[indu] - vₕ = @view V[indv] - ω_flat = reshape(ω, :) - - if boundary_conditions.u.x[1] == :periodic && boundary_conditions.v.y[1] == :periodic - uₕ_in = uₕ - vₕ_in = vₕ - else - # Velocity at inner points - diagpos = 0 - boundary_conditions.u.x[1] == :pressure && (diagpos = 1) - boundary_conditions.u.x[1] == :periodic && (diagpos = 1) - B1D = spdiagm(Nx - 1, Nux_in, diagpos => ones(T, Nx - 1)) - B2D = I(Ny) ⊗ B1D - uₕ_in = B2D * uₕ - - diagpos = 0 - boundary_conditions.v.y[1] == :pressure && (diagpos = 1) - boundary_conditions.v.y[1] == :periodic && (diagpos = 1) - B1D = spdiagm(Ny - 1, Nvy_in, diagpos => ones(T, Ny - 1)) - B2D = B1D ⊗ I(Nx) - vₕ_in = B2D * vₕ - end - - # ω_flat .= Wv_vx * vₕ_in - Wu_uy * uₕ_in - mul!(ω_flat, Wv_vx, vₕ_in) # a = b * c - mul!(ω_flat, Wu_uy, uₕ_in, -1, 1) # a = -b * c + a - - ω -end - -# 3D version -function vorticity!(::Dimension{3}, ω, setup, V, t) - (; grid, operators, boundary_conditions) = setup - (; indu, indv, indw, Nux_in, Nvy_in, Nwz_in, Nx, Ny, Nz) = grid - (; Wu_uy, Wu_uz, Wv_vx, Wv_vz, Ww_wx, Ww_wy) = operators - - T = eltype(V) - - uₕ = @view V[indu] - vₕ = @view V[indv] - wₕ = @view V[indw] - ω_flat = reshape(ω, :) - - if boundary_conditions.u.x[1] == :periodic && - boundary_conditions.v.y[1] == :periodic && - boundary_conditions.w.z[1] == :periodic - uₕ_in = uₕ - vₕ_in = vₕ - wₕ_in = wₕ - else - # Velocity at inner points - diagpos = 0 - boundary_conditions.u.x[1] == :pressure && (diagpos = 1) - boundary_conditions.u.x == (:periodic, :periodic) && (diagpos = 1) - B1D = spdiagm(Nx - 1, Nux_in, diagpos => ones(T, Nx - 1)) - B2D = I(Nz) ⊗ I(Ny) ⊗ B1D - uₕ_in = B2D * uₕ - - diagpos = 0 - boundary_conditions.v.y[1] == :pressure && (diagpos = 1) - boundary_conditions.v.y == (:periodic, :periodic) && (diagpos = 1) - B1D = spdiagm(Ny - 1, Nvy_in, diagpos => ones(T, Ny - 1)) - B2D = I(Nz) ⊗ B1D ⊗ I(Nx) - vₕ_in = B2D * vₕ - - diagpos = 0 - boundary_conditions.w.z[1] == :pressure && (diagpos = 1) - boundary_conditions.w.z == (:periodic, :periodic) && (diagpos = 1) - B1D = spdiagm(Nz - 1, Nwz_in, diagpos => ones(T, Nz - 1)) - B2D = B1D ⊗ I(Ny) ⊗ I(Nx) - wₕ_in = B2D * wₕ - end - - # ωx_flat .= Ww_wy * wₕ_in - Wy_yz * vₕ_in - # ωy_flat .= Wu_uz * uₕ_in - Ww_wx * wₕ_in - # ωz_flat .= Wv_vx * vₕ_in - Wu_uy * uₕ_in - - ω_flat .= .√( - (Ww_wy * wₕ_in .- Wv_vz * vₕ_in) .^ 2 .+ (Wu_uz * uₕ_in .- Ww_wx * wₕ_in) .^ 2 .+ (Wv_vx * vₕ_in .- Wu_uy * uₕ_in) .^ 2, - ) - - ω -end diff --git a/src/postprocess/plot_pressure.jl b/src/postprocess/plot_pressure.jl index 1a4bbbb56..1a417bad3 100644 --- a/src/postprocess/plot_pressure.jl +++ b/src/postprocess/plot_pressure.jl @@ -10,15 +10,17 @@ plot_pressure(setup, p; kwargs...) = # 2D version function plot_pressure(::Dimension{2}, setup, p; kwargs...) - (; Nx, Ny, Npx, Npy, xp, yp, xlims, ylims) = setup.grid + (; xp, xlims) = setup.grid - # Reshape - p = reshape(p, Npx, Npy) + xp = Array.(xp) + p = Array(p) + + T = eltype(xp[1]) # Levels μ, σ = mean(p), std(p) - ≈(μ + σ, μ; rtol = 1e-8, atol = 1e-8) && (σ = 1e-4) - levels = LinRange(μ - 1.5σ, μ + 1.5σ, 10) + # ≈(μ + σ, μ; rtol = sqrt(eps(T)), atol = sqrt(eps(T))) && (σ = sqrt(sqrt(eps(T)))) + levels = LinRange(μ - T(1.5) * σ, μ + T(1.5) * σ, 10) # Plot pressure fig = Figure() @@ -29,9 +31,10 @@ function plot_pressure(::Dimension{2}, setup, p; kwargs...) xlabel = "x", ylabel = "y", ) - limits!(ax, xlims[1], xlims[2], ylims[1], ylims[2]) - cf = contourf!(ax, xp, yp, p; extendlow = :auto, extendhigh = :auto, levels, kwargs...) - # Colorbar(fig[1,2], cf) + limits!(ax, xlims[1]..., xlims[2]...) + cf = contourf!(ax, xp..., p; extendlow = :auto, extendhigh = :auto, + levels, + kwargs...) Colorbar(fig[1, 2], cf) # save("output/pressure.png", fig, pt_per_unit = 2) diff --git a/src/postprocess/plot_vorticity.jl b/src/postprocess/plot_vorticity.jl index 1d8a50e76..09790f36f 100644 --- a/src/postprocess/plot_vorticity.jl +++ b/src/postprocess/plot_vorticity.jl @@ -5,24 +5,16 @@ Plot vorticity field. """ function plot_vorticity end -plot_vorticity(setup, V, t; kwargs...) = - plot_vorticity(setup.grid.dimension, setup, V, t; kwargs...) +plot_vorticity(setup, u, t; kwargs...) = + plot_vorticity(setup.grid.dimension, setup, u, t; kwargs...) # 2D version -function plot_vorticity(::Dimension{2}, setup, V, t; kwargs...) +function plot_vorticity(::Dimension{2}, setup, u, t; kwargs...) (; grid, boundary_conditions) = setup - (; x, y, xlims, ylims) = grid - - if all(==(:periodic), (boundary_conditions.u.x[1], boundary_conditions.v.y[1])) - xω = x - yω = y - else - xω = x[2:(end-1)] - yω = y[2:(end-1)] - end + (; x, xlims) = grid # Get fields - ω = get_vorticity(setup, V, t) + ω = vorticity(u, t) # Levels μ, σ = mean(ω), std(ω) @@ -31,7 +23,7 @@ function plot_vorticity(::Dimension{2}, setup, V, t; kwargs...) # Plot vorticity fig = Figure() - ax = Axis( + ax = Makie.Axis( fig[1, 1]; aspect = DataAspect(), title = "Vorticity", @@ -39,7 +31,8 @@ function plot_vorticity(::Dimension{2}, setup, V, t; kwargs...) ylabel = "y", ) limits!(ax, xlims[1], xlims[2], ylims[1], ylims[2]) - cf = contourf!(ax, xω, yω, ω; extendlow = :auto, extendhigh = :auto, levels, kwargs...) + # cf = contourf!(ax, xω, yω, ω; extendlow = :auto, extendhigh = :auto, levels, kwargs...) + cf = heatmap!(ax, xω, yω, ω; kwargs...) Colorbar(fig[1, 2], cf) # save("output/vorticity.png", fig, pt_per_unit = 2) diff --git a/src/preprocess/create_initial_conditions.jl b/src/preprocess/create_initial_conditions.jl deleted file mode 100644 index 3cd132a6c..000000000 --- a/src/preprocess/create_initial_conditions.jl +++ /dev/null @@ -1,253 +0,0 @@ -""" - create_initial_conditions( - setup, - initial_velocity_u, - initial_velocity_v, - [initial_velocity_w,] - t; - initial_pressure = nothing, - pressure_solver = DirectPressureSolver(setup), - ) - -Create initial vectors at starting time `t`. If `p_initial` is a function instead of -`nothing`, calculate compatible IC for the pressure. -""" -function create_initial_conditions end - -# 2D version -function create_initial_conditions( - setup, - initial_velocity_u, - initial_velocity_v, - t; - initial_pressure = nothing, - pressure_solver = DirectPressureSolver(setup), -) - (; grid, operators) = setup - (; xu, yu, xv, yv, xpp, ypp, Ω) = grid - (; G, M) = operators - - T = eltype(xu) - - # Boundary conditions - bc_vectors = get_bc_vectors(setup, t) - (; yM) = bc_vectors - - # Allocate velocity and pressure - u = zero(xu)[:] - v = zero(xv)[:] - p = zero(xpp)[:] - - # Initial velocities - u .= initial_velocity_u.(xu, yu)[:] - v .= initial_velocity_v.(xv, yv)[:] - V = [u[:]; v[:]] - - # Kinetic energy and momentum of initial velocity field - # Iteration 1 corresponds to t₀ = 0 (for unsteady simulations) - maxdiv, umom, vmom, k = compute_conservation(V, t, setup; bc_vectors) - - # TODO: Maybe eps(T)^(3//4) - if maxdiv > 1e-12 - @warn "Initial velocity field not (discretely) divergence free: $maxdiv.\n" * - "Performing additional projection." - - # Make velocity field divergence free - f = M * V + yM - Δp = pressure_poisson(pressure_solver, f) - V .-= 1 ./ Ω .* (G * Δp) - end - - # Initial pressure: should in principle NOT be prescribed (will be calculated if p_initial) - if isnothing(initial_pressure) - p = pressure_additional_solve(pressure_solver, V, p, t, setup) - else - p .= initial_pressure.(xpp, ypp)[:] - end - - V, p -end - -# 3D version -function create_initial_conditions( - setup, - initial_velocity_u, - initial_velocity_v, - initial_velocity_w, - t; - initial_pressure = nothing, - pressure_solver = DirectPressureSolver(setup), -) - (; grid, operators) = setup - (; xu, yu, zu, xv, yv, zv, xw, yw, zw, xpp, ypp, zpp, Ω) = grid - (; G, M) = operators - - # Boundary conditions - bc_vectors = get_bc_vectors(setup, t) - (; yM) = bc_vectors - - # Allocate velocity and pressure - u = zero(xu)[:] - v = zero(xv)[:] - w = zero(xw)[:] - p = zero(xpp)[:] - - # Initial velocities - u .= initial_velocity_u.(xu, yu, zu)[:] - v .= initial_velocity_v.(xv, yv, zv)[:] - w .= initial_velocity_w.(xw, yw, zw)[:] - V = [u; v; w] - - # Kinetic energy and momentum of initial velocity field - # Iteration 1 corresponds to t₀ = 0 (for unsteady simulations) - maxdiv, umom, vmom, wmom, k = compute_conservation(V, t, setup; bc_vectors) - - # TODO: Maybe eps(T)^(3//4) - if maxdiv > 1e-12 - @warn "Initial velocity field not (discretely) divergence free: $maxdiv.\n" * - "Performing additional projection." - - # Make velocity field divergence free - f = M * V + yM - Δp = pressure_poisson(pressure_solver, f) - V .-= 1 ./ Ω .* (G * Δp) - end - - # Initial pressure: should in principle NOT be prescribed (will be calculated if p_initial) - if isnothing(initial_pressure) - p = pressure_additional_solve(pressure_solver, V, p, t, setup) - else - p .= initial_pressure.(xpp, ypp, zpp)[:] - end - - V, p -end - -function create_spectrum_2(x, A, σ, s) - T = typeof(A) - AT = typeof(x) - nx, ny = size(x) - Kx = nx ÷ 2 - Ky = ny ÷ 2 - i = AT(reshape(1:Kx, :, 1)) - j = AT(reshape(1:Ky, 1, :)) - p = T(π) - a = @. A * ( - 1 / sqrt((2p)^2 * 2σ^2) * - exp(-((i - s)^2 + (j - s)^2) / 2σ^2) * - exp(-2p * im * rand(T)) - ) - [ - a reverse(a; dims = 2) - reverse(a; dims = 1) reverse(a) - ] -end - -function create_spectrum_3(x, A, σ, s) - T = typeof(A) - AT = typeof(x) - nx, ny, nz = size(x) - Kx = nx ÷ 2 - Ky = ny ÷ 2 - Kz = nz ÷ 2 - i = AT(reshape(1:Kx, :, 1, 1)) - j = AT(reshape(1:Ky, 1, :, 1)) - k = AT(reshape(1:Kz, 1, 1, :)) - p = T(π) - a = @. A * ( - 1 / sqrt((2p)^3 * 3σ^2) * - exp(-((i - s)^2 + (j - s)^2 + (k - s)^2) / 2σ^2) * - exp(-2p * im * rand(T)) - ) - [ - a reverse(a; dims = 2); reverse(a; dims = 1) reverse(a; dims = (1, 2));;; - reverse(a; dims = 3) reverse(a; dims = (2, 3)); reverse(a; dims = (1, 3)) reverse(a) - ] -end - -""" - random_field( - setup, K; - A = 1_000_000, - σ = 30, - s = 5, - pressure_solver = DirectPressureSolver(setup), - ) - -Create random field. - -- `K`: Maximum wavenumber -- `A`: Eddy amplitude -- `σ`: Variance -- `s` Wavenumber offset before energy starts decaying -""" -function random_field end - -random_field(setup; kwargs...) = random_field(setup.grid.dimension, setup; kwargs...) - -# 2D version -function random_field( - ::Dimension{2}, - setup; - A = convert(eltype(setup.grid.x), 1_000_000), - σ = convert(eltype(setup.grid.x), 30), - s = 5, - pressure_solver = DirectPressureSolver(setup), -) - (; xu, xv, Ω) = setup.grid - (; G, M) = setup.operators - - T = eltype(Ω) - - u = real.(ifft(create_spectrum_2(xu, A, σ, s))) - v = real.(ifft(create_spectrum_2(xv, A, σ, s))) - V = [reshape(u, :); reshape(v, :)] - f = M * V - p = zero(f) - - # Boundary conditions - bc_vectors = get_bc_vectors(setup, T(0)) - (; yM) = bc_vectors - - # Make velocity field divergence free - f = M * V + yM - Δp = pressure_poisson(pressure_solver, f) - V .-= 1 ./ Ω .* (G * Δp) - p = pressure_additional_solve(pressure_solver, V, p, T(0), setup; bc_vectors) - - V, p -end - -# 3D version -function random_field( - ::Dimension{3}, - setup; - A = convert(eltype(setup.grid.x), 1_000_000), - σ = convert(eltype(setup.grid.x), 30), - s = 5, - pressure_solver = DirectPressureSolver(setup), -) - (; xu, xv, xw, Ω) = setup.grid - (; G, M) = setup.operators - - T = eltype(Ω) - - u = real.(ifft(create_spectrum_3(xu, A, σ, s))) - v = real.(ifft(create_spectrum_3(xv, A, σ, s))) - w = real.(ifft(create_spectrum_3(xw, A, σ, s))) - V = [reshape(u, :); reshape(v, :); reshape(w, :)] - f = M * V - p = zero(f) - - # Boundary conditions - bc_vectors = get_bc_vectors(setup, T(0)) - (; yM) = bc_vectors - - # Make velocity field divergence free - f = M * V + yM - Δp = pressure_poisson(pressure_solver, f) - V .-= 1 ./ Ω .* (G * Δp) - p = pressure_additional_solve(pressure_solver, V, p, T(0), setup; bc_vectors) - - V, p -end diff --git a/src/processors/real_time_plot.jl b/src/processors/real_time_plot.jl index 97cf8660f..dbeca4707 100644 --- a/src/processors/real_time_plot.jl +++ b/src/processors/real_time_plot.jl @@ -22,6 +22,7 @@ Available fieldnames are: Available plot `type`s for 2D are: - `heatmap` (default), +- `image`, - `contour`, - `contourf`. @@ -45,18 +46,12 @@ function field_plot( displayfig = true, ) (; boundary_conditions, grid) = setup - (; xlims, ylims, x, y, xp, yp) = grid + (; xlims, x, xp) = grid if fieldname == :velocity - xf, yf = xp, yp + xf = xp elseif fieldname == :vorticity - if all(==(:periodic), (boundary_conditions.u.x[1], boundary_conditions.v.y[1])) - xf = x - yf = y - else - xf = x[2:(end-1)] - yf = y[2:(end-1)] - end + xf = Array.(x) elseif fieldname == :streamfunction if boundary_conditions.u.x[1] == :periodic xf = x @@ -70,21 +65,21 @@ function field_plot( end elseif fieldname == :pressure error("Not implemented") - xf, yf = xp, yp + xf = xp else error("Unknown fieldname") end field = @lift begin isnothing(sleeptime) || sleep(sleeptime) - (; V, p, t) = $state + (; u, p, t) = $state f = if fieldname == :velocity - up, vp = get_velocity(setup, V, t) + up, vp = get_velocity(setup, u, t) map((u, v) -> √sum(u^2 + v^2), up, vp) elseif fieldname == :vorticity - get_vorticity(setup, V, t) + vorticity(u, setup) elseif fieldname == :streamfunction - get_streamfunction(setup, V, t) + get_streamfunction(setup, u, t) elseif fieldname == :pressure error("Not implemented") reshape(p, length(xp), length(yp)) @@ -114,14 +109,11 @@ function field_plot( fig = Figure() if type ∈ (heatmap, image) - ax, hm = type(fig[1, 1], xf, yf, field; - colormap = :viridis, - colorrange = lims) + ax, hm = type(fig[1, 1], xf..., field; colormap = :viridis, colorrange = lims) elseif type ∈ (contour, contourf) ax, hm = type( fig[1, 1], - xf, - yf, + xf..., field; extendlow = :auto, extendhigh = :auto, @@ -136,7 +128,7 @@ function field_plot( equal_axis && (ax.aspect = DataAspect()) ax.xlabel = "x" ax.ylabel = "y" - limits!(ax, xlims[1], xlims[2], ylims[1], ylims[2]) + limits!(ax, xlims[1]..., xlims[2]...) Colorbar(fig[1, 2], hm) displayfig && display(fig) @@ -287,7 +279,13 @@ function energy_spectrum_plot(::Dimension{2}, setup, state; displayfig = true) espec end -function energy_spectrum_plot(::Dimension{3}, setup, state, displayfig = true, checktime = 0.0001) +function energy_spectrum_plot( + ::Dimension{3}, + setup, + state, + displayfig = true, + checktime = 0.0001, +) (; xpp) = setup.grid Kx, Ky, Kz = size(xpp) .÷ 2 kx = 1:(Kx-1) diff --git a/src/setup.jl b/src/setup.jl index 8db7e3c29..d9fccc73a 100644 --- a/src/setup.jl +++ b/src/setup.jl @@ -1,169 +1,33 @@ """ Setup( - x, y; - Re = 1000, + x; + boundary_conditions = ntuple(d -> (PeriodicBC(), PeriodicBC()), length(x)), + Re = convert(eltype(x[1]), 1_000), viscosity_model = LaminarModel(), convection_model = NoRegConvectionModel(), - u_bc = (x, y, t) -> 0, - v_bc = (x, y, t) -> 0, - dudt_bc = nothing, - dvdt_bc = nothing, - bc_type = (; - u = (; x = (:periodic, :periodic), y = (:periodic, :periodic)), - v = (; x = (:periodic, :periodic), y = (:periodic, :periodic)), - ν = (; x = (:periodic, :periodic), y = (:periodic, :periodic)), - ), - order4 = false, - bodyforce_u = (x, y) -> 0, - bodyforce_v = (x, y) -> 0, + bodyforce = nothing, closure_model = nothing, ) -Create 2D setup. +Create setup. """ function Setup( - x, - y; - Re = convert(eltype(x), 1000), + x; + boundary_conditions = ntuple(d -> (PeriodicBC(), PeriodicBC()), length(x)), + Re = convert(eltype(x[1]), 1_000), viscosity_model = LaminarModel(), convection_model = NoRegConvectionModel(), - u_bc = (x, y, t) -> 0, - v_bc = (x, y, t) -> 0, - dudt_bc = nothing, - dvdt_bc = nothing, - bc_type = (; - u = (; x = (:periodic, :periodic), y = (:periodic, :periodic)), - v = (; x = (:periodic, :periodic), y = (:periodic, :periodic)), - ν = (; x = (:periodic, :periodic), y = (:periodic, :periodic)), - ), - order4 = false, - bodyforce_u = (x, y) -> 0, - bodyforce_v = (x, y) -> 0, - steady_force = true, - closure_model = V -> zero(V), + bodyforce = nothing, + closure_model = nothing, ) - boundary_conditions = - BoundaryConditions(u_bc, v_bc; dudt_bc, dvdt_bc, bc_type, T = eltype(x)) - grid = Grid(x, y; boundary_conditions, order4) - force = SteadyBodyForce(bodyforce_u, bodyforce_v, grid) - operators = Operators(grid, boundary_conditions) + grid = Grid(x, boundary_conditions) (; grid, boundary_conditions, Re, viscosity_model, convection_model, - force, + bodyforce, closure_model, - operators, - ) -end - -""" - Setup( - x, y, z; - Re = convert(eltype(x), 1000), - viscosity_model = LaminarModel(), - convection_model = NoRegConvectionModel(), - u_bc = (x, y, w, t) -> 0.0, - v_bc = (x, y, w, t) -> 0.0, - w_bc = (x, y, w, t) -> 0.0, - dudt_bc = nothing, - dvdt_bc = nothing, - dwdt_bc = nothing, - bc_type = (; - u = (; - x = (:periodic, :periodic), - y = (:periodic, :periodic), - z = (:periodic, :periodic), - ), - v = (; - x = (:periodic, :periodic), - y = (:periodic, :periodic), - z = (:periodic, :periodic), - ), - w = (; - x = (:periodic, :periodic), - y = (:periodic, :periodic), - z = (:periodic, :periodic), - ), - ν = (; - x = (:periodic, :periodic), - y = (:periodic, :periodic), - z = (:periodic, :periodic), - ), - ), - order4 = false, - bodyforce_u = (x, y, z) -> 0, - bodyforce_v = (x, y, z) -> 0, - bodyforce_w = (x, y, z) -> 0, - closure_model = nothing, - ) - -Create 3D setup. -""" -function Setup( - x, - y, - z; - Re = convert(eltype(x), 1000), - viscosity_model = LaminarModel(), - convection_model = NoRegConvectionModel(), - u_bc = (x, y, w, t) -> 0, - v_bc = (x, y, w, t) -> 0, - w_bc = (x, y, w, t) -> 0, - dudt_bc = nothing, - dvdt_bc = nothing, - dwdt_bc = nothing, - bc_type = (; - u = (; - x = (:periodic, :periodic), - y = (:periodic, :periodic), - z = (:periodic, :periodic), - ), - v = (; - x = (:periodic, :periodic), - y = (:periodic, :periodic), - z = (:periodic, :periodic), - ), - w = (; - x = (:periodic, :periodic), - y = (:periodic, :periodic), - z = (:periodic, :periodic), - ), - ν = (; - x = (:periodic, :periodic), - y = (:periodic, :periodic), - z = (:periodic, :periodic), - ), - ), - order4 = false, - bodyforce_u = (x, y, z) -> 0, - bodyforce_v = (x, y, z) -> 0, - bodyforce_w = (x, y, z) -> 0, - closure_model = V -> zero(V), -) - boundary_conditions = BoundaryConditions( - u_bc, - v_bc, - w_bc; - dudt_bc, - dvdt_bc, - dwdt_bc, - bc_type, - T = eltype(x), - ) - grid = Grid(x, y, z; boundary_conditions, order4) - force = SteadyBodyForce(bodyforce_u, bodyforce_v, bodyforce_w, grid) - operators = Operators(grid, boundary_conditions) - (; - grid, - boundary_conditions, - Re, - viscosity_model, - convection_model, - force, - closure_model, - operators, ) end diff --git a/src/solvers/pressure/pressure_additional_solve.jl b/src/solvers/pressure/pressure_additional_solve.jl index f55c6b330..3ac87c757 100644 --- a/src/solvers/pressure/pressure_additional_solve.jl +++ b/src/solvers/pressure/pressure_additional_solve.jl @@ -1,71 +1,39 @@ """ - pressure_additional_solve(pressure_solver, V, p, t, setup; bc_vectors = nothing) + pressure_additional_solve!(pressure_solver, u, p, t, setup, F, f, M) Do additional pressure solve. This makes the pressure compatible with the velocity field, resulting in same order pressure as velocity. """ -function pressure_additional_solve(pressure_solver, V, p, t, setup; bc_vectors = nothing) - (; grid, operators, boundary_conditions) = setup - (; bc_unsteady) = boundary_conditions - (; Ω) = grid - (; M) = operators +function pressure_additional_solve!(pressure_solver, u, p, t, setup, F, M) + (; grid) = setup + (; dimension, Iu, Ip, Ωu) = grid + D = dimension() - # Get updated BC for ydM (time derivative of BC in ydM) - # FIXME: `get_bc_vectors` are called to often (also inside `momentum!`) - if isnothing(bc_vectors) || bc_unsteady - bc_vectors = get_bc_vectors(setup, t) - end - (; ydM) = bc_vectors - - # Momentum already contains G*p with the current p, we therefore effectively solve for - # the pressure difference - F, = momentum(V, V, p, t, setup; bc_vectors) - f = M * (1 ./ Ω .* F) + ydM + momentum!(F, u, t, setup) - Δp = pressure_poisson(pressure_solver, f) - p .+ Δp + for α = 1:D + @. F[α] = 1 ./ Ωu[α] .* F[α] + end + apply_bc_u!(F, t, setup; dudt = true) + divergence!(M, F, setup) + + Min = view(M, Ip) + pin = view(p, Ip) + pressure_poisson!(pressure_solver, pin, Min) + apply_bc_p!(p, t, setup) + p end """ - pressure_additional_solve!(pressure_solver, V, p, t, setup, momentum_cache, F, f; bc_vectors) + pressure_additional_solve(pressure_solver, u, t, setup) Do additional pressure solve. This makes the pressure compatible with the velocity field, resulting in same order pressure as velocity. """ -function pressure_additional_solve!( - pressure_solver, - V, - p, - t, - setup, - momentum_cache, - F, - f, - Δp; - bc_vectors, -) - (; grid, operators, boundary_conditions) = setup - (; bc_unsteady) = boundary_conditions - (; Ω) = grid - (; M) = operators - - # Get updated BC for ydM (time derivative of BC in ydM) - # FIXME: `get_bc_vectors` are called to often (also inside `momentum!`) - if isnothing(bc_vectors) || bc_unsteady - bc_vectors = get_bc_vectors(setup, t) - end - (; ydM) = bc_vectors - - # Momentum already contains G*p with the current p, we therefore effectively solve for - # the pressure difference - momentum!(F, nothing, V, V, p, t, setup, momentum_cache; bc_vectors) - - # f = M * (1 ./ Ω .* F) + ydM - @. F = 1 ./ Ω .* F - mul!(f, M, F) - @. f = f + ydM - - pressure_poisson!(pressure_solver, Δp, f) - - p .+= Δp +function pressure_additional_solve(pressure_solver, u, t, setup) + D = setup.grid.dimension() + p = KernelAbstractions.zeros(get_backend(u[1]), eltype(u[1]), setup.grid.N) + F = ntuple(α -> KernelAbstractions.zeros(get_backend(u[1]), eltype(u[1]), setup.grid.N), D) + M = KernelAbstractions.zeros(get_backend(u[1]), eltype(u[1]), setup.grid.N) + pressure_additional_solve!(pressure_solver, u, p, t, setup, F, M) end diff --git a/src/solvers/pressure/pressure_poisson.jl b/src/solvers/pressure/pressure_poisson.jl index 81c82db29..6add28b5c 100644 --- a/src/solvers/pressure/pressure_poisson.jl +++ b/src/solvers/pressure/pressure_poisson.jl @@ -54,30 +54,34 @@ function pressure_poisson! end function pressure_poisson!(solver::DirectPressureSolver, p, f) # Assume the Laplace matrix is known (A) and is possibly factorized + f = view(f, :) + p = view(p, :) + # Use pre-determined decomposition p .= solver.A_fact \ f end function pressure_poisson!(solver::CGPressureSolver, p, f) - # TODO: Preconditioner (; A, abstol, reltol, maxiter) = solver + f = view(f, :) + p = view(p, :) cg!(p, A, f; abstol, reltol, maxiter) end function pressure_poisson!(solver::SpectralPressureSolver, p, f) (; Ahat, fhat, phat) = solver - fhat[:] .= complex.(f) + phat .= complex.(f) # Fourier transform of right hand side - fft!(fhat) + fft!(phat) # Solve for coefficients in Fourier space - @. phat = -fhat / Ahat + @. phat = -phat / Ahat # Transform back ifft!(phat) - @. p[:] = real(@view phat[:]) + @. p = real(phat) p end diff --git a/src/solvers/pressure/pressure_solvers.jl b/src/solvers/pressure/pressure_solvers.jl index 33f864e6f..0c8b00216 100644 --- a/src/solvers/pressure/pressure_solvers.jl +++ b/src/solvers/pressure/pressure_solvers.jl @@ -75,86 +75,42 @@ Adapt.adapt_structure(to, s::SpectralPressureSolver) = Build spectral pressure solver from setup. """ -SpectralPressureSolver(setup) = SpectralPressureSolver(setup.grid.dimension, setup) - -function SpectralPressureSolver(::Dimension{2}, setup) +function SpectralPressureSolver(setup) (; grid, boundary_conditions) = setup - (; hx, hy, Npx, Npy) = grid - - T = eltype(hx) - AT = typeof(hx) + (; dimension, Δ, Np, x) = grid - if any( - !isequal((:periodic, :periodic)), - (boundary_conditions.u.x, boundary_conditions.v.y), - ) - error("SpectralPressureSolver only implemented for periodic boundary conditions") - end + D = dimension() + T = eltype(Δ[1]) - Δx = first(hx) - Δy = first(hy) - if any(!≈(Δx), hx) || any(!≈(Δy), hy) - error("SpectralPressureSolver requires uniform grid along each dimension") - end - - # Fourier transform of the discretization - # Assuming uniform grid, although Δx and Δy do not need to be the same - i = AT(0:(Npx-1)) - j = reshape(AT(0:(Npy-1)), 1, :) + Δx = first.(Array.(Δ)) - # Scale with Δx*Δy, since we solve the PDE in integrated form - Ahat = @. 4 * Δx * Δy * (sin(i * π / Npx)^2 / Δx^2 + sin(j * π / Npy)^2 / Δy^2) - - # Pressure is determined up to constant, fix at 0 - Ahat[1] = 1 - - Ahat = complex(Ahat) - - # Placeholders for intermediate results - phat = similar(Ahat) - fhat = similar(Ahat) + @assert( + all(bc -> bc[1] isa PeriodicBC && bc[2] isa PeriodicBC, boundary_conditions), + "SpectralPressureSolver only implemented for periodic boundary conditions", + ) - SpectralPressureSolver{T,typeof(Ahat)}(Ahat, phat, fhat) -end + @assert( + all(α -> all(≈(Δx[α]), Δ[α]), 1:D), + "SpectralPressureSolver requires uniform grid along each dimension", + ) -function SpectralPressureSolver(::Dimension{3}, setup) - (; grid, boundary_conditions) = setup - (; hx, hy, hz, Npx, Npy, Npz) = grid - T = eltype(hx) - AT = typeof(hx) + # Fourier transform of the discretization + # Assuming uniform grid, although Δx[1] and Δx[2] do not need to be the same - if any( - !isequal((:periodic, :periodic)), - [boundary_conditions.u.x, boundary_conditions.v.y, boundary_conditions.w.z], - ) - error("SpectralPressureSolver only implemented for periodic boundary conditions") - end + k = ntuple(d -> reshape(0:Np[d]-1, ntuple(Returns(1), d - 1)..., :, ntuple(Returns(1), D - d)...), D) - Δx = first(hx) - Δy = first(hy) - Δz = first(hz) - if any(!≈(Δx), hx) || any(!≈(Δy), hy) || any(!≈(Δz), hz) - error("SpectralPressureSolver requires uniform grid along each dimension") + Ahat = KernelAbstractions.zeros(get_backend(x[1]), Complex{T}, Np...) + Tπ = T(π) # CUDA doesn't like pi + for d = 1:D + @. Ahat += sin(k[d] * Tπ / Np[d])^2 / Δx[d]^2 end - # Fourier transform of the discretization - # Assuming uniform grid, although Δx and Δy do not need to be the same - i = AT(0:(Npx-1)) - j = reshape(AT(0:(Npy-1)), 1, :) - k = reshape(AT(0:(Npz-1)), 1, 1, :) - # Scale with Δx*Δy*Δz, since we solve the PDE in integrated form - Ahat = @. 4 * - Δx * - Δy * - Δz * - (sin(i * π / Npx)^2 / Δx^2 + sin(j * π / Npy)^2 / Δy^2 + sin(k * π / Npz)^2 / Δz^2) + Ahat .*= 4 * prod(Δx) # Pressure is determined up to constant, fix at 0 - Ahat[1] = 1 - - Ahat = complex(Ahat) + Ahat[1:1] .= 1 # Placeholders for intermediate results phat = similar(Ahat) diff --git a/src/solvers/solve_unsteady.jl b/src/solvers/solve_unsteady.jl index 620bf8a5b..31e3f3c61 100644 --- a/src/solvers/solve_unsteady.jl +++ b/src/solvers/solve_unsteady.jl @@ -1,14 +1,13 @@ """ function solve_unsteady( - setup, V₀, p₀, tlims; - method = RK44(; T = eltype(V₀)), + setup, u₀, p₀, tlims; + method = RK44(; T = eltype(u₀)), pressure_solver = DirectPressureSolver(setup), - Δt = nothing, + Δt = zero(eltype(u₀)), cfl = 1, n_adapt_Δt = 1, - inplace = false, + inplace = true, processors = (), - device = identity, ) Solve unsteady problem using `method`. @@ -25,7 +24,7 @@ This allows for performing computations on a different device than the host (CPU To compute on an Nvidia GPU using CUDA, change ``` -solve_unsteady(setup, V₀, p₀, tlims; kwargs...) +solve_unsteady(setup, u₀, p₀, tlims; kwargs...) ``` to the following: @@ -33,7 +32,7 @@ to the following: ``` using CUDA solve_unsteady( - setup, V₀, p₀, tlims; + setup, u₀, p₀, tlims; device = cu, kwargs... ) @@ -41,22 +40,20 @@ solve_unsteady( Note that the `state` observable passed to the `processor.initialize` function contains vector living on the device, and you may have to move them back to -the host using `Array(V)` and `Array(p)` in the processor. +the host using `Array(u)` and `Array(p)` in the processor. """ function solve_unsteady( setup, - V₀, + u₀, p₀, tlims; - method = RK44(; T = eltype(V₀)), + method = RK44(; T = eltype(u₀[1])), pressure_solver = DirectPressureSolver(setup), - Δt = zero(eltype(V₀)), + Δt = zero(eltype(u₀[1])), cfl = 1, n_adapt_Δt = 1, - inplace = false, + inplace = true, processors = (), - device = identity, - devsetup = device(setup), ) t_start, t_end = tlims isadaptive = isnothing(Δt) @@ -65,20 +62,8 @@ function solve_unsteady( Δt = (t_end - t_start) / nstep end - # Initialize BC arrays (currently only done on host, due to Kronecker - # products) - bc_vectors = get_bc_vectors(setup, t_start) - - # Move vectors and operators to device (if any). - setup = devsetup - V₀ = device(V₀) - p₀ = device(p₀) - bc_vectors = device(bc_vectors) - pressure_solver = device(pressure_solver) - if inplace - cache = ode_method_cache(method, setup, V₀, p₀) - momentum_cache = MomentumCache(setup, V₀, p₀) + cache = ode_method_cache(method, setup, u₀, p₀) end # Time stepper @@ -86,8 +71,7 @@ function solve_unsteady( method; setup, pressure_solver, - bc_vectors, - V = copy(V₀), + u = copy.(u₀), p = copy(p₀), t = t_start, ) @@ -113,9 +97,9 @@ function solve_unsteady( # Perform a single time step with the time integration method if inplace - stepper = step!(method, stepper, Δt; cache, momentum_cache) + stepper = timestep!(method, stepper, Δt; cache) else - stepper = step(method, stepper, Δt) + stepper = timestep(method, stepper, Δt) end # Process iteration results with each processor @@ -125,17 +109,17 @@ function solve_unsteady( end end - (; V, p, t, n) = stepper + (; u, p, t, n) = stepper finalized = map((ps, i) -> ps.finalize(i, get_state(stepper)), processors, initialized) # Final state - (; V, p) = stepper + (; u, p) = stepper # Move output arrays to host - Array(V), Array(p), finalized + u, p, finalized end function get_state(stepper) - (; V, p, t, n) = stepper - (; V, p, t, n) + (; u, p, t, n) = stepper + (; u, p, t, n) end diff --git a/src/time_steppers/step.jl b/src/time_steppers/step.jl index fbc99169d..e61fb4c08 100644 --- a/src/time_steppers/step.jl +++ b/src/time_steppers/step.jl @@ -7,7 +7,7 @@ Non-mutating/allocating/out-of-place version. See also [`step!`](@ref). """ -function step end +function timestep end # step(stepper, Δt; bc_vectors = nothing) = step(stepper.method, stepper, Δt; bc_vectors = nothing) """ @@ -19,7 +19,7 @@ Mutating/non-allocating/in-place version. See also [`step`](@ref). """ -function step! end +function timestep! end # step!(stepper, Δt; kwargs...) = step!(stepper.method, stepper, Δt; kwargs...) include("step_ab_cn.jl") diff --git a/src/time_steppers/step_ab_cn.jl b/src/time_steppers/step_ab_cn.jl index fa88a59a7..f7c23b616 100644 --- a/src/time_steppers/step_ab_cn.jl +++ b/src/time_steppers/step_ab_cn.jl @@ -16,7 +16,7 @@ create_stepper( Diff_fact = spzeros(eltype(V), 0, 0), ) = (; setup, pressure_solver, bc_vectors, V, p, t, n, Vₙ, pₙ, cₙ, tₙ, Diff_fact) -function step(method::AdamsBashforthCrankNicolsonMethod, stepper, Δt) +function timestep(method::AdamsBashforthCrankNicolsonMethod, stepper, Δt) (; setup, pressure_solver, bc_vectors, V, p, t, n, Vₙ, pₙ, cₙ, tₙ, Diff_fact) = stepper (; convection_model, viscosity_model, Re, force, grid, operators, boundary_conditions) = setup @@ -47,7 +47,7 @@ function step(method::AdamsBashforthCrankNicolsonMethod, stepper, Δt) Diff_fact = lu(I(NV) - θ * Δt / Re * Diagonal(1 ./ Ω) * Diff) end - (; V, p, t) = step(method_startup, stepper_startup, Δt) + (; V, p, t) = timestep(method_startup, stepper_startup, Δt) return create_stepper( method; setup, @@ -160,7 +160,7 @@ function step(method::AdamsBashforthCrankNicolsonMethod, stepper, Δt) ) end -function step!( +function timestep!( method::AdamsBashforthCrankNicolsonMethod, stepper, Δt; @@ -198,7 +198,7 @@ function step!( Diff_fact = lu(I(NV) - θ * Δt / Re * Diagonal(1 ./ Ω) * Diff) # Note: We do one out-of-place step here, with a few allocations - (; V, p, t) = step(method_startup, stepper_startup, Δt) + (; V, p, t) = timestep(method_startup, stepper_startup, Δt) return create_stepper( method; setup, diff --git a/src/time_steppers/step_explicit_runge_kutta.jl b/src/time_steppers/step_explicit_runge_kutta.jl index 62619d75e..c9d79bf57 100644 --- a/src/time_steppers/step_explicit_runge_kutta.jl +++ b/src/time_steppers/step_explicit_runge_kutta.jl @@ -1,41 +1,30 @@ -create_stepper( - ::ExplicitRungeKuttaMethod; - setup, - pressure_solver, - bc_vectors, - V, - p, - t, - n = 0, -) = (; setup, pressure_solver, bc_vectors, V, p, t, n) - -function step(method::ExplicitRungeKuttaMethod, stepper, Δt) - (; setup, pressure_solver, bc_vectors, V, p, t, n) = stepper - (; grid, operators, boundary_conditions) = setup - (; bc_unsteady) = boundary_conditions +create_stepper(::ExplicitRungeKuttaMethod; setup, pressure_solver, u, p, t, n = 0) = + (; setup, pressure_solver, u, p, t, n) + +function timestep(method::ExplicitRungeKuttaMethod, stepper, Δt) + (; setup, pressure_solver, u, p, t, n) = stepper + (; grid, boundary_conditions) = setup (; Ω) = grid - (; G, M) = operators (; A, b, c, p_add_solve) = method T = typeof(Δt) # Update current solution (does not depend on previous step size) n += 1 - Vₙ = V + Vₙ = u pₙ = p tₙ = t Δtₙ = Δt # Number of stages - nV = length(V) + nV = length(u) np = length(p) nstage = length(b) # Reset RK arrays tᵢ = tₙ # kV = zeros(T, nV, 0) - # kp = zeros(T, np, 0) - kV = fill(V, 0) + ku = fill(u, 0) ## Start looping over stages @@ -47,13 +36,13 @@ function step(method::ExplicitRungeKuttaMethod, stepper, Δt) # includes force evaluation at tᵢ and pressure gradient. Boundary conditions will be # set through `get_bc_vectors` inside momentum. The pressure p is not important here, # it will be removed again in the next step - F, ∇F = momentum(V, V, p, tᵢ, setup; bc_vectors) + F = momentum(u, p, tᵢ, setup) # Store right-hand side of stage i # Remove the -G*p contribution (but not y_p) kVᵢ = 1 ./ Ω .* (F + G * p) # kV = [kV kVᵢ] - kV = [kV; [kVᵢ]] + kV = [ku; [kVᵢ]] # Update velocity current stage by sum of Fᵢ's until this stage, weighted # with Butcher tableau coefficients. This gives uᵢ₊₁, and for i=s gives uᵢ₊₁ @@ -92,31 +81,31 @@ function step(method::ExplicitRungeKuttaMethod, stepper, Δt) t = tₙ + Δtₙ - create_stepper(method; setup, pressure_solver, bc_vectors, V, p, t, n) + create_stepper(method; setup, pressure_solver, bc_vectors, u, p, t, n) end -function step!(method::ExplicitRungeKuttaMethod, stepper, Δt; cache, momentum_cache) - (; setup, pressure_solver, bc_vectors, V, p, t, n) = stepper - (; grid, operators, boundary_conditions) = setup - (; bc_unsteady) = boundary_conditions - (; Ω) = grid - (; G, M) = operators +function timestep!(method::ExplicitRungeKuttaMethod, stepper, Δt; cache) + (; setup, pressure_solver, u, p, t, n) = stepper + (; grid, boundary_conditions) = setup + (; dimension, Ip, Ωu) = grid (; A, b, c, p_add_solve) = method - (; Vₙ, pₙ, kV, kp, Vtemp, Vtemp2, F, ∇F, Δp, f) = cache + (; uₙ, ku, v, F, M, G, f) = cache + + D = dimension() # Update current solution (does not depend on previous step size) n += 1 - Vₙ .= V - pₙ .= p tₙ = t Δtₙ = Δt + for α = 1:D + uₙ[α] .= u[α] + end # Number of stages nstage = length(b) # # Reset RK arrays - # kV .= 0 - # kp .= 0 + # ku .= 0 tᵢ = tₙ @@ -127,72 +116,55 @@ function step!(method::ExplicitRungeKuttaMethod, stepper, Δt; cache, momentum_c # At i = s we calculate Fₛ, pₙ₊₁, and uₙ₊₁ for i = 1:nstage # Right-hand side for tᵢ based on current velocity field uₕ, vₕ at level i. This - # includes force evaluation at tᵢ and pressure gradient. Boundary conditions will be - # set through `get_bc_vectors` inside momentum. The pressure p is not important here, + # includes force evaluation at tᵢ and pressure gradient. The pressure p is not important here, # it will be removed again in the next step - momentum!(F, ∇F, V, V, p, tᵢ, setup, momentum_cache; bc_vectors) + momentum!(F, u, tᵢ, setup) # Store right-hand side of stage i - # Remove the -G*p contribution (but not y_p) - # kVᵢ = @view kV[:, i] - kVᵢ = kV[i] - mul!(kVᵢ, G, p) - @. kVᵢ = 1 ./ Ω * (F + kVᵢ) - # kVᵢ .= 1 ./ Ω .* (F + G * p) + for α = 1:D + @. ku[i][α] = 1 / Ωu[α] * F[α] + end # Update velocity current stage by sum of Fᵢ's until this stage, weighted # with Butcher tableau coefficients. This gives uᵢ₊₁, and for i=s gives uᵢ₊₁ - # mul!(Vtemp, kV, A[i, :]) - Vtemp .= 0 + for α = 1:D + v[α] .= uₙ[α] + end for j = 1:i - Vtemp .= Vtemp .+ A[i, j] .* kV[j] + for α = 1:D + v[α] .= v[α] .+ Δtₙ .* A[i, j] .* ku[j][α] + end end # Boundary conditions at tᵢ₊₁ tᵢ = tₙ + c[i] * Δtₙ - if bc_unsteady - bc_vectors = get_bc_vectors(setup, tᵢ) - end - (; yM) = bc_vectors + apply_bc_u!(v, tᵢ, setup) - # Divergence of intermediate velocity field - @. Vtemp2 = Vₙ / Δtₙ + Vtemp - mul!(f, M, Vtemp2) - @. f = (f + yM / Δtₙ) / c[i] - # f = (M * (Vₙ / Δtₙ + Vtemp) + yM / Δtₙ) / c[i] + # Divergence of tentative velocity field + divergence!(M, v, setup) - # Solve the Poisson equation, but not for the first step if the boundary conditions are steady - if boundary_conditions.bc_unsteady || i > 1 - pressure_poisson!(pressure_solver, p, f) - else - # Bc steady AND i = 1 - p .= pₙ - end + @. M = M / (c[i] * Δtₙ) - mul!(Vtemp2, G, p) + # Solve the Poisson equation + Min = view(M, Ip) + pin = view(p, Ip) + pressure_poisson!(pressure_solver, pin, Min) + apply_bc_p!(p, t, setup) + + # Compute pressure correction term + pressuregradient!(G, p, setup) # Update velocity current stage, which is now divergence free - @. V = Vₙ + Δtₙ * (Vtemp - c[i] / Ω * Vtemp2) + for α = 1:D + @. u[α] = v[α] - c[i] * Δtₙ / Ωu[α] * G[α] + end end - # For steady bc we do an additional pressure solve - # That saves a pressure solve for i = 1 in the next time step - if !bc_unsteady || p_add_solve - pressure_additional_solve!( - pressure_solver, - V, - p, - tₙ + Δtₙ, - setup, - momentum_cache, - F, - f, - Δp; - bc_vectors, - ) - end + # Do additional pressure solve to avoid first order pressure + p_add_solve && + pressure_additional_solve!(pressure_solver, u, p, tₙ + Δtₙ, setup, F, M) t = tₙ + Δtₙ - create_stepper(method; setup, pressure_solver, bc_vectors, V, p, t, n) + create_stepper(method; setup, pressure_solver, u, p, t, n) end diff --git a/src/time_steppers/step_implicit_runge_kutta.jl b/src/time_steppers/step_implicit_runge_kutta.jl index 64f17475b..c63bfefa0 100644 --- a/src/time_steppers/step_implicit_runge_kutta.jl +++ b/src/time_steppers/step_implicit_runge_kutta.jl @@ -9,7 +9,7 @@ create_stepper( n = 0, ) = (; setup, pressure_solver, bc_vectors, V, p, t, n) -function step(method::ImplicitRungeKuttaMethod, stepper, Δt) +function timestep(method::ImplicitRungeKuttaMethod, stepper, Δt) # TODO: Implement out-of-place IRK error() @@ -174,7 +174,7 @@ function step(method::ImplicitRungeKuttaMethod, stepper, Δt) create_stepper(method; setup, pressure_solver, bc_vectors, V, p, t, n) end -function step!(method::ImplicitRungeKuttaMethod, stepper, Δt; cache, momentum_cache) +function timestep!(method::ImplicitRungeKuttaMethod, stepper, Δt; cache, momentum_cache) (; setup, pressure_solver, bc_vectors, V, p, t, n) = stepper (; grid, operators, boundary_conditions) = setup (; bc_unsteady) = boundary_conditions diff --git a/src/time_steppers/step_one_leg.jl b/src/time_steppers/step_one_leg.jl index a050340c6..2fb305f15 100644 --- a/src/time_steppers/step_one_leg.jl +++ b/src/time_steppers/step_one_leg.jl @@ -14,7 +14,7 @@ create_stepper( tₙ = t, ) = (; setup, pressure_solver, bc_vectors, V, p, t, n, Vₙ, pₙ, tₙ) -function step(method::OneLegMethod, stepper, Δt) +function timestep(method::OneLegMethod, stepper, Δt) (; setup, pressure_solver, bc_vectors, V, p, t, n, Vₙ, pₙ, tₙ) = stepper (; p_add_solve, β, method_startup) = method (; grid, operators, boundary_conditions) = setup @@ -31,7 +31,7 @@ function step(method::OneLegMethod, stepper, Δt) Vₙ = V pₙ = p tₙ = t - (; V, p, t) = step(method_startup, stepper_startup, Δt) + (; V, p, t) = timestep(method_startup, stepper_startup, Δt) return create_stepper( method; setup, @@ -105,7 +105,7 @@ function step(method::OneLegMethod, stepper, Δt) create_stepper(method; setup, pressure_solver, bc_vectors, V, p, t, n, Vₙ, pₙ, tₙ) end -function step!(method::OneLegMethod, stepper, Δt; cache, momentum_cache) +function timestep!(method::OneLegMethod, stepper, Δt; cache, momentum_cache) (; setup, pressure_solver, bc_vectors, n, V, p, t, Vₙ, pₙ, tₙ) = stepper (; p_add_solve, β, method_startup) = method (; grid, operators, boundary_conditions) = setup @@ -125,7 +125,7 @@ function step!(method::OneLegMethod, stepper, Δt; cache, momentum_cache) tₙ = t # Note: We do one out-of-place step here, with a few allocations - (; V, p, t) = step(method_startup, stepper_startup, Δt) + (; V, p, t) = timestep(method_startup, stepper_startup, Δt) return create_stepper( method; setup, diff --git a/src/time_steppers/time_stepper_caches.jl b/src/time_steppers/time_stepper_caches.jl index b8b4716af..8de8c31d9 100644 --- a/src/time_steppers/time_stepper_caches.jl +++ b/src/time_steppers/time_stepper_caches.jl @@ -34,28 +34,21 @@ function ode_method_cache(::OneLegMethod{T}, setup, V, p) where {T} (; Vₙ₋₁, pₙ₋₁, F, f, Δp, GΔp) end -function ode_method_cache(method::ExplicitRungeKuttaMethod{T}, setup, V, p) where {T} - (; NV, Np) = setup.grid +function ode_method_cache(method::ExplicitRungeKuttaMethod{T}, setup, u, p) where {T} - Vₙ = similar(V) - pₙ = similar(p) + uₙ = similar.(u) ns = nstage(method) - # kV = zeros(T, NV, ns) - # kp = zeros(T, Np, ns) - - kV = [similar(V) for i = 1:ns] - kp = [similar(p) for i = 1:ns] + ku = [similar.(u) for i = 1:ns] - Vtemp = similar(V) - Vtemp2 = similar(V) - F = similar(V) - ∇F = spzeros(T, NV, NV) + v = similar.(u) + F = similar.(u) + G = similar.(u) + M = similar(p) f = similar(p) - Δp = similar(p) - (; Vₙ, pₙ, kV, kp, Vtemp, Vtemp2, F, ∇F, f, Δp) + (; uₙ, ku, v, F, M, G, f) end function ode_method_cache(method::ImplicitRungeKuttaMethod{T}, setup, V, p) where {T} diff --git a/src/utils/filter_convection.jl b/src/utils/filter_convection.jl deleted file mode 100644 index 632fbc58d..000000000 --- a/src/utils/filter_convection.jl +++ /dev/null @@ -1,28 +0,0 @@ -""" - filter_convection(u, diff_matrix, bc, α) - -Construct filter for convective terms. - -Non-mutating/allocating/out-of-place version. - -See also [`filter_convection!`](@ref). -""" -function filter_convection(u, diff_matrix, bc, α) - u + α * (diff_matrix * u + bc) -end - -""" - filter_convection!(ū, u, diff_matrix, bc, α) - -Construct filter for convective terms. - -Mutating/non-allocating/in-place version. - -See also [`filter_convection`](@ref). -""" -function filter_convection!(ū, u, diff_matrix, bc, α) - # ū = u + α * (diff_matrix * u + bc) - mul!(ū, diff_matrix, u) - @. ū = u + α * (ū + bc) - ū -end diff --git a/src/utils/get_lims.jl b/src/utils/get_lims.jl index cc7f1a730..5ba772d8a 100644 --- a/src/utils/get_lims.jl +++ b/src/utils/get_lims.jl @@ -6,8 +6,9 @@ deviation (``\\mu \\pm n \\sigma``). If `x` is constant, a margin of `1e-4` is e plotting functions that require a certain range. """ function get_lims(x, n = 1.5) + T = eltype(x) μ = mean(x) σ = std(x) - ≈(μ + σ, μ; rtol = 1e-8, atol = 1e-8) && (σ = 1e-4) + # ≈(μ + σ, μ; rtol = sqrt(eps(T)), atol = sqrt(eps(T))) && (σ = sqrt(sqrt(eps(T)))) (μ - n * σ, μ + n * σ) end From e7065fa805553f5a9e909b4cdf91ad7041b2d444 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Thu, 14 Sep 2023 11:30:24 +0200 Subject: [PATCH 052/379] fix: separate types --- src/boundary_conditions.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/boundary_conditions.jl b/src/boundary_conditions.jl index f4cd4e3f6..d15e78154 100644 --- a/src/boundary_conditions.jl +++ b/src/boundary_conditions.jl @@ -2,9 +2,9 @@ abstract type AbstractBC end struct PeriodicBC <: AbstractBC end -struct DirichletBC{F} <: AbstractBC +struct DirichletBC{F,G} <: AbstractBC u::F - dudt::F + dudt::G end struct SymmetricBC <: AbstractBC end From 710546b80d746650d17d8e0069b311b5caa6c6b9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Thu, 14 Sep 2023 12:09:11 +0200 Subject: [PATCH 053/379] Rename momentum --- src/IncompressibleNavierStokes.jl | 4 ++-- src/{momentum.jl => operators.jl} | 3 +-- 2 files changed, 3 insertions(+), 4 deletions(-) rename src/{momentum.jl => operators.jl} (99%) diff --git a/src/IncompressibleNavierStokes.jl b/src/IncompressibleNavierStokes.jl index 305a51c63..b73526365 100644 --- a/src/IncompressibleNavierStokes.jl +++ b/src/IncompressibleNavierStokes.jl @@ -64,8 +64,8 @@ include("processors/processors.jl") include("processors/real_time_plot.jl") include("processors/animator.jl") -# Momentum equation -include("momentum.jl") +# Discrete operators +include("operators.jl") # Solvers include("solvers/get_timestep.jl") diff --git a/src/momentum.jl b/src/operators.jl similarity index 99% rename from src/momentum.jl rename to src/operators.jl index d791d5e77..ad651432c 100644 --- a/src/momentum.jl +++ b/src/operators.jl @@ -1,6 +1,5 @@ # Let this be constant for now -# const WORKGROUP = 64 -const WORKGROUP = 256 +const WORKGROUP = 64 # See https://b-fg.github.io/2023/05/07/waterlily-on-gpu.html # for writing kernel loops From e95f440499e062fb4afc0ca0e4e91f6b7475f8ce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Thu, 14 Sep 2023 13:42:29 +0200 Subject: [PATCH 054/379] refactor: Divide by volume sizes --- docs/src/equations/spatial.md | 34 +++++++++++-------- docs/src/equations/time.md | 14 ++++---- src/create_initial_conditions.jl | 4 +-- src/operators.jl | 20 +++++------ .../pressure/pressure_additional_solve.jl | 7 +--- .../step_explicit_runge_kutta.jl | 4 +-- 6 files changed, 41 insertions(+), 42 deletions(-) diff --git a/docs/src/equations/spatial.md b/docs/src/equations/spatial.md index ee13d714f..a0e47b4d5 100644 --- a/docs/src/equations/spatial.md +++ b/docs/src/equations/spatial.md @@ -393,19 +393,25 @@ The discrete momentum equations become ```math \begin{split} - \Omega_h \frac{\mathrm{d} u_h}{\mathrm{d} t} & = -C(u_h) + \nu (D u_h + + \frac{\mathrm{d} u_h}{\mathrm{d} t} & = -C(u_h) + \nu (D u_h + y_D) + f_h - (G p_h + y_G) \\ & = F(V_h) - (G p_h + y_G), \end{split} ``` -where ``\Omega_h`` is a diagonal matrix containing the velocity volumes, ``C`` -is the convection operator (including boundary contributions), ``D`` is the -diffusion operator, ``y_D`` is boundary contribution to the diffusion term, ``G -= M^\mathsf{T}`` is the pressure gradient operator, and ``y_G`` contains the -boundary contribution of the pressure to the pressure gradient (only non-zero -for pressure boundary conditions). The term ``F`` refers to all the forces -except for the pressure gradient. +where ``C`` is the convection operator (including boundary contributions), +``D`` is the diffusion operator, ``y_D`` is boundary contribution to the +diffusion term, ``G = \Omega_h^{-1} M^\mathsf{T}`` is the pressure gradient +operator, ``y_G`` contains the boundary contribution of the pressure to the +pressure gradient (only non-zero for pressure boundary conditions), and +``\Omega_h`` is a diagonal matrix containing the velocity volumes. The term +``F`` refers to all the forces except for the pressure gradient. + +!!! note "Volume normalization" + + All the operators (except for ``M``) have been divided by the velocity + volume sizes ``\Omega_h``. As a result, the operators have the same units + as their continuous counterparts. ## Discrete pressure Poisson equation @@ -417,11 +423,11 @@ discrete divergence operator ``M`` to the discrete momentum equations yields the discrete pressure Poisson equation ```math -L p_h = M \Omega_h^{-1} (F(V_h) - y_G) + \frac{\mathrm{d} y_M}{\mathrm{d} t}, +L p_h = M (F(V_h) - y_G) + \frac{\mathrm{d} y_M}{\mathrm{d} t}, ``` -where ``L = M \Omega_h^{-1} G`` is a discrete Laplace operator. It is positive -symmetric since ``G = M^\mathsf{T}``. +where ``L = M G`` is a discrete Laplace operator. It is positive +symmetric since ``G = \Omega_h^{-1} M^\mathsf{T}``. !!! note "Unsteady Dirichlet boundary conditions" @@ -449,17 +455,17 @@ symmetric since ``G = M^\mathsf{T}``. discrete Poisson equation: ```math - p_h = L^{-1} M \Omega_h^{-1} (F(u_h) - y_G) + L^{-1} \frac{\mathrm{d} y_M}{\mathrm{d} t}. + p_h = L^{-1} M (F(u_h) - y_G) + L^{-1} \frac{\mathrm{d} y_M}{\mathrm{d} t}. ``` The momentum equations then become ```math - \Omega_h \frac{\mathrm{d} u_h}{\mathrm{d} t} = (I - G L^{-1} M \Omega_h^{-1}) + \frac{\mathrm{d} u_h}{\mathrm{d} t} = (I - G L^{-1} M) (F(u_h) - y_G) - G L^{-1} \frac{\mathrm{d} y_M}{\mathrm{d} t}. ``` - The matrix ``(I - G L^{-1} M \Omega^{-1})`` is a projector onto the space + The matrix ``(I - G L^{-1} M)`` is a projector onto the space of discretely divergence free velocities. However, using this formulation would require an efficient way to perform the projection without assembling the operator matrix ``L^{-1}``, which would be very costly. diff --git a/docs/src/equations/time.md b/docs/src/equations/time.md index 257505eb7..2a5c39b22 100644 --- a/docs/src/equations/time.md +++ b/docs/src/equations/time.md @@ -4,7 +4,7 @@ The spatially discretized Navier-Stokes equations form a differential-algebraic system, with an ODE for the velocity ```math -\Omega_h \frac{\mathrm{d} u_h}{\mathrm{d} t} = F(u_h, t) - (G p_h + y_G) +\frac{\mathrm{d} u_h}{\mathrm{d} t} = F(u_h, t) - (G p_h + y_G) ``` subject to the algebraic constraint formed by the mass equation @@ -49,13 +49,13 @@ are computed as follows: ```math \begin{split} -F_i & = \Omega_h^{-1} F(U_{i - 1}, t_{i - 1}) \\ +F_i & = F(U_{i - 1}, t_{i - 1}) \\ V_i & = U_0 + \Delta t \sum_{j = 1}^i a_{i j} F_j \\ L P_i & = \frac{1}{c_i} \sum_{j = 1}^i a_{i j} F_j + \frac{y_M(t_i) - y_M(t_0)}{\Delta t_i} \\ & = \frac{(M V_i + y_M(t_i)) - (M U_0 + y_M(t_0))}{\Delta t_i^n} \\ & = \frac{M V_i + y_M(t_i)}{\Delta t_i^n} \\ -U_i & = V_i - \Delta t_i \Omega_h^{-1} (G P_i + y_G(t_i)), +U_i & = V_i - \Delta t_i (G P_i + y_G(t_i)), \end{split} ``` @@ -136,7 +136,7 @@ L \Delta P = \frac{M V + y_M(t)}{\Delta t} - M (y_G(t) - y_G(t_0)), after which a divergence free velocity ``U`` can be enforced: ```math -U = V - \Delta t \Omega_h^{-1} (G \Delta P + y_G(t) - y_G(t_0)). +U = V - \Delta t (G \Delta P + y_G(t) - y_G(t_0)). ``` A first order accurate prediction of the corresponding pressure is ``P = P_0 + @@ -159,8 +159,8 @@ A tentative velocity field ``W`` is then computed as follows: ```math W = \frac{1}{\beta + \frac{1}{2}} \left( 2 \beta U_0 - \left( \beta - -\frac{1}{2} \right) U_{-1} + \Delta t \Omega_h^{-1} F(V, t) - \Delta t -\Omega_h^{-1} (G Q + y_G(t)) \right). +\frac{1}{2} \right) U_{-1} + \Delta t F(V, t) - \Delta t +(G Q + y_G(t)) \right). ``` A pressure correction ``\Delta P `` is obtained by solving the Poisson equation @@ -171,7 +171,7 @@ L \Delta P = \frac{\beta + \frac{1}{2}}{\Delta t} (M W + y_M(t)). Finally, the divergence free velocity field is given by ```math -U = W - \frac{\Delta t}{\beta + \frac{1}{2}} \Omega_h^{-1} G \Delta P, +U = W - \frac{\Delta t}{\beta + \frac{1}{2}} G \Delta P, ``` while the second order accurate pressure is given by diff --git a/src/create_initial_conditions.jl b/src/create_initial_conditions.jl index 730fcf7b0..c66b0a107 100644 --- a/src/create_initial_conditions.jl +++ b/src/create_initial_conditions.jl @@ -48,7 +48,7 @@ function create_initial_conditions( apply_bc_p!(p, t, setup) G = pressuregradient(p, setup) for α = 1:D - u[α] .-= 1 ./ Ωu[α] .* G[α] + u[α] .-= G[α] end end @@ -120,7 +120,7 @@ function random_field( apply_bc_p!(p, t, setup) G = pressuregradient(p, setup) for α = 1:D - @. u[α] -= 1 / Ωu[α] * G[α] + @. u[α] -= G[α] end apply_bc_u!(u, t, setup) p = pressure_additional_solve(pressure_solver, u, t, setup) diff --git a/src/operators.jl b/src/operators.jl index ad651432c..b2ac5b812 100644 --- a/src/operators.jl +++ b/src/operators.jl @@ -91,17 +91,18 @@ end function convection!(F, u, setup) (; boundary_conditions, grid, Re, bodyforce) = setup - (; dimension, Δ, Δu, Nu, Iu, Γu) = grid + (; dimension, Δ, Δu, Nu, Iu, Γu, Ωu) = grid D = dimension() δ = Offset{D}() @kernel function _convection!(F, u, α, β, I0) I = @index(Global, Cartesian) I = I + I0 + Δuαβ = (α == β ? Δu[β] : Δ[β]) F[α][I] -= - Γu[α][β][I] * ( + ( (u[α][I] + u[α][I+δ(β)]) / 2 * (u[β][I] + u[β][I+δ(α)]) / 2 - (u[α][I-δ(β)] + u[α][I]) / 2 * (u[β][I-δ(β)] + u[β][I-δ(β)+δ(α)]) / 2 - ) + ) / Δuαβ[I[β]] end for α = 1:D I0 = first(Iu[α]) @@ -124,13 +125,12 @@ function diffusion!(F, u, setup; ϵ = eps(eltype(F[1]))) @kernel function _diffusion!(F, u, α, β, I0) I = @index(Global, Cartesian) I = I + I0 + Δuαβ = (α == β ? Δu[β] : Δ[β]) F[α][I] += - ν * - Γu[α][β][I] * - ( + ν * ( (u[α][I+δ(β)] - u[α][I]) / ((β == α ? Δ : Δu)[β][I[β]] + ϵ) - (u[α][I] - u[α][I-δ(β)]) / ((β == α ? Δ : Δu)[β][(I-δ(β))[β]] + ϵ) - ) + ) / Δuαβ[I[β]] end for α = 1:D I0 = first(Iu[α]) @@ -151,8 +151,6 @@ function bodyforce!(F, u, t, setup) @kernel function _bodyforce!(F, force, α, t, I0) I = @index(Global, Cartesian) I = I + I0 - vol = prod(β -> (β == α ? Δu : Δ)[k][I[k]], ntuple(identity, D)) - # vol = Δu[i] * prod(Δ) / Δ[i] F[α][I] += force[α](ntuple(β -> α == β ? x[β][1+I[β]] : xp[β][I[β]], D)..., t) end for α = 1:D @@ -187,13 +185,13 @@ end function pressuregradient!(G, p, setup) (; boundary_conditions, grid) = setup - (; dimension, Δ, Np, Iu, Ω) = grid + (; dimension, Δu, Np, Iu) = grid D = dimension() δ = Offset{D}() @kernel function _pressuregradient!(G, p, α, I0) I = @index(Global, Cartesian) I = I0 + I - G[α][I] = Ω[I] / Δ[α][I[α]] * (p[I+δ(α)] - p[I]) + G[α][I] = (p[I+δ(α)] - p[I]) / Δu[α][I[α]] end D = dimension() for α = 1:D diff --git a/src/solvers/pressure/pressure_additional_solve.jl b/src/solvers/pressure/pressure_additional_solve.jl index 3ac87c757..a6d8aa19f 100644 --- a/src/solvers/pressure/pressure_additional_solve.jl +++ b/src/solvers/pressure/pressure_additional_solve.jl @@ -6,14 +6,9 @@ field, resulting in same order pressure as velocity. """ function pressure_additional_solve!(pressure_solver, u, p, t, setup, F, M) (; grid) = setup - (; dimension, Iu, Ip, Ωu) = grid - D = dimension() + (; Ip) = grid momentum!(F, u, t, setup) - - for α = 1:D - @. F[α] = 1 ./ Ωu[α] .* F[α] - end apply_bc_u!(F, t, setup; dudt = true) divergence!(M, F, setup) diff --git a/src/time_steppers/step_explicit_runge_kutta.jl b/src/time_steppers/step_explicit_runge_kutta.jl index c9d79bf57..fd11d2df6 100644 --- a/src/time_steppers/step_explicit_runge_kutta.jl +++ b/src/time_steppers/step_explicit_runge_kutta.jl @@ -122,7 +122,7 @@ function timestep!(method::ExplicitRungeKuttaMethod, stepper, Δt; cache) # Store right-hand side of stage i for α = 1:D - @. ku[i][α] = 1 / Ωu[α] * F[α] + @. ku[i][α] = F[α] end # Update velocity current stage by sum of Fᵢ's until this stage, weighted @@ -156,7 +156,7 @@ function timestep!(method::ExplicitRungeKuttaMethod, stepper, Δt; cache) # Update velocity current stage, which is now divergence free for α = 1:D - @. u[α] = v[α] - c[i] * Δtₙ / Ωu[α] * G[α] + @. u[α] = v[α] - c[i] * Δtₙ * G[α] end end From a8217d495de4c23d81f08e1be3c2289677264f98 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Mon, 18 Sep 2023 17:40:40 +0200 Subject: [PATCH 055/379] Reremove files --- src/boundary_conditions/bc_diff3.jl | 93 -------------------- src/boundary_conditions/bc_general.jl | 119 -------------------------- 2 files changed, 212 deletions(-) delete mode 100644 src/boundary_conditions/bc_diff3.jl delete mode 100644 src/boundary_conditions/bc_general.jl diff --git a/src/boundary_conditions/bc_diff3.jl b/src/boundary_conditions/bc_diff3.jl deleted file mode 100644 index 0f66cb28d..000000000 --- a/src/boundary_conditions/bc_diff3.jl +++ /dev/null @@ -1,93 +0,0 @@ -# total solution u is written as u = Bb*ub + Bin*uin -# the boundary conditions can be written as Bbc*u = ybc -# then u can be written entirely in terms of uin and ybc as: -# u = (Bin-Btemp*Bbc*Bin)*uin + Btemp*ybc, where -# Btemp = Bb*(Bbc*Bb)^(-1) -# Bb, Bin and Bbc depend on type of bc (Neumann/Dirichlet/periodic) -# -# val1 and val2 can be scalars or vectors with either the value or the -# derivative -# -# (ghost) points on boundary / grid lines -function bc_diff3(Nt, Nin, Nb, bc1, bc2, h1, h2) - T = typeof(h1) - - # some input checking: - if Nt != Nin + Nb - error("Number of inner points plus boundary points is not equal to total points") - end - - if Nb == 0 # no boundary points, so simply diagonal matrix without boundary contribution - B1D = I(Nt) - Btemp = spzeros(T, Nt, 2) - ybc1 = zeros(T, 2) - ybc2 = zeros(T, 2) - else - # boundary conditions - Bbc = spzeros(T, Nb, Nt) - ybc1_1D = zeros(T, Nb) - ybc2_1D = zeros(T, Nb) - - if Nb == 6 - # normal situation, 2 boundary points - - # boundary matrices - Bin = spdiagm(Nt, Nin, -3 => ones(T, Nin)) - Bb = spzeros(T, Nt, Nb) - Bb[1:3, 1:3] = I(3) - Bb[end-2:end, end-2:end] = I(3) - if bc1 == :dirichlet - Bbc[1, 1] = 1 / 2 # Dirichlet - Bbc[1, 5] = 1 / 2 - Bbc[2, 2] = 1 / 2 # Dirichlet uLe - Bbc[2, 4] = 1 / 2 - Bbc[3, 3] = 1 # Dirichlet uLe - ybc1_1D[1] = 1 - ybc1_1D[2] = 1 - ybc1_1D[3] = 1 - elseif bc1 == :pressure - error("not implemented") - elseif bc1 == :periodic - Bbc[1:3, 1:3] = -I(3) - Bbc[1:3, end-5:end-3] = I(3) - Bbc[end-2:end, 4:6] = -I(3) - Bbc[end-2:end, end-2:end] = I(3) - else - error("not implemented") - end - - if bc2 == :dirichlet - Bbc[end, end] = 1 / 2 - Bbc[end, end-4] = 1 / 2 - Bbc[end-1, end-1] = 1 / 2 - Bbc[end-1, end-3] = 1 / 2 - Bbc[end-2, end-2] = 1 - ybc2_1D[end-2] = 1 # uRi - ybc2_1D[end-1] = 1 # uRi - ybc2_1D[end] = 1 - elseif bc2 == :pressure - error("not implemented") - elseif bc2 == :periodic - # actually redundant - Bbc[1:3, 1:3] = -I(3) - Bbc[1:3, end-5:end-3] = I(3) - Bbc[end-2:end, 4:6] = -I(3) - Bbc[end-2:end, end-2:end] = I(3) - else - error("not implemented") - end - - elseif Nb == 1 - # one boundary point - error("not implemented") - end - - ybc1 = ybc1_1D - ybc2 = ybc2_1D - - Btemp = Bb / (Bbc * Bb) - B1D = Bin - Btemp * Bbc * Bin - end - - (; B1D, Btemp, ybc1, ybc2) -end diff --git a/src/boundary_conditions/bc_general.jl b/src/boundary_conditions/bc_general.jl deleted file mode 100644 index ae842697c..000000000 --- a/src/boundary_conditions/bc_general.jl +++ /dev/null @@ -1,119 +0,0 @@ -# bc_general(Nt, Nin, Nb, bc1, bc2, h1, h2) -# -# Total solution `u` is written as `u = Bb*ub + Bin*uin` -# -# The boundary conditions can be written as `Bbc*u = ybc` -# -# Then `u` can be written entirely in terms of `uin` and `ybc` as: `u = -# (Bin-Btemp*Bbc*Bin)*uin + Btemp*ybc`, where `Btemp = Bb/(Bbc*Bb)`. -# -# `Bb`, `Bin` and `Bbc` depend on type of bc (Neumann/Dirichlet/periodic) `val1` and `val2` -# can be scalars or vectors with either the value or the derivative (ghost) points on -# boundary/grid lines -function bc_general(Nt, Nin, Nb, bc1, bc2, h1, h2) - if Nt != Nin + Nb - error("Number of inner points plus boundary points is not equal to total points") - end - - T = typeof(h1) - - # Boundary conditions - Bbc = spzeros(T, Nb, Nt) - ybc1_1D = zeros(T, Nb) - ybc2_1D = zeros(T, Nb) - - if Nb == 0 - # No boundary points, so simply diagonal matrix without boundary contribution - B1D = I(Nt) - Btemp = spzeros(T, Nt, 2) - ybc1 = zeros(T, 2) - ybc2 = zeros(T, 2) - elseif Nb == 1 - # One boundary point - Bb = spzeros(T, Nt, Nb) - diagpos = -1 - if bc1 == :dirichlet - Bbc[1, 1] = 1 - ybc1_1D[1] = 1 # uLe - Bb[1, 1] = 1 - elseif bc1 == :pressure - diagpos = 0 - elseif bc1 == :periodic - diagpos = 0 - Bbc[1, 1] = -1 - Bbc[1, end] = 1 - Bb[end, 1] = 1 - else - error("not implemented") - end - - if bc2 == :dirichlet - Bbc[Nb, end] = 1 - ybc2_1D[1] = 1 # uRi - Bb[end, Nb] = 1 - elseif bc2 == :pressure - - elseif bc2 == :periodic # Actually redundant - diagpos = 0 - Bbc[1, 1] = -1 - Bbc[1, end] = 1 - Bb[end, 1] = 1 - else - error("not implemented") - end - - # Boundary matrices - Bin = spdiagm(Nt, Nin, diagpos => ones(T, Nin)) - elseif Nb == 2 - # Normal situation, 2 boundary points - # Boundary matrices - Bin = spdiagm(Nt, Nin, -1 => ones(T, Nin)) - Bb = spzeros(T, Nt, Nb) - Bb[1, 1] = 1 - Bb[end, Nb] = 1 - - if bc1 == :dirichlet - Bbc[1, 1] = 1 - ybc1_1D[1] = 1 # uLe - elseif bc1 == :pressure - Bbc[1, 1] = -1 - Bbc[1, 2] = 1 - ybc1_1D[1] = 2 * h1 # DuLe - elseif bc1 == :periodic - Bbc[1, 1] = -1 - Bbc[1, end-1] = 1 - Bbc[Nb, 2] = -1 - Bbc[Nb, end] = 1 - else - error("not implemented") - end - - if bc2 == :dirichlet - Bbc[Nb, end] = 1 - ybc2_1D[2] = 1 # uRi - elseif bc2 == :pressure - Bbc[Nb, end-1] = -1 - Bbc[Nb, end] = 1 - ybc2_1D[2] = 2 * h2 # duRi - elseif bc2 == :periodic # Actually redundant - Bbc[1, 1] = -1 - Bbc[1, end-1] = 1 - Bbc[Nb, 2] = -1 - Bbc[Nb, end] = 1 - else - error("not implemented") - end - else - error("Nb must be 0, 1, or 2") - end - - if Nb ∈ (1, 2) - ybc1 = ybc1_1D - ybc2 = ybc2_1D - - Btemp = Bb / (Bbc * Bb) - B1D = Bin - Btemp * Bbc * Bin - end - - (; B1D, Btemp, ybc1, ybc2) -end From f17da7e85c2e25cc1e8d0b4e47015876f14656c7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Mon, 18 Sep 2023 17:43:09 +0200 Subject: [PATCH 056/379] docs: minor changes --- docs/src/equations/ns.md | 41 +++++++---- docs/src/equations/spatial.md | 130 ++++++++++++++++++++++++++++++++-- docs/src/equations/time.md | 24 +++---- 3 files changed, 164 insertions(+), 31 deletions(-) diff --git a/docs/src/equations/ns.md b/docs/src/equations/ns.md index 194670717..769f96646 100644 --- a/docs/src/equations/ns.md +++ b/docs/src/equations/ns.md @@ -24,7 +24,7 @@ written as \frac{\partial u^\alpha}{\partial t} + \sum_{\beta = 1}^d \frac{\partial}{\partial x^\beta} (u^\alpha u^\beta) & = -\frac{\partial p}{\partial x^\alpha} + \nu \sum_{\beta = 1}^d \frac{\partial^2 u^\alpha}{\partial -(x^\beta)^2} + f^\alpha. +x^\beta \partial x^\beta} + f^\alpha. \end{align*} ``` @@ -101,7 +101,7 @@ equation for the pressure: In scalar notation, this becomes ```math -- \sum_{\alpha = 1}^d \frac{\partial^2}{\partial (x^\alpha)^2} p = \sum_{\alpha +- \sum_{\alpha = 1}^d \frac{\partial^2}{\partial x^\alpha \partial x^\alpha} p = \sum_{\alpha = 1}^d \sum_{\beta = 1}^d \frac{\partial^2 }{\partial x^\alpha \partial x^\beta} (u^\alpha u^\beta) - \sum_{\alpha = 1}^d \frac{\partial f^\alpha}{\partial x^\alpha}. @@ -117,6 +117,18 @@ a constant. We set this constant to ``1``. ## Other quantities of interest +### Reynolds number + +The Reynolds number is the inverse of the viscosity: ``Re = \frac{1}{\nu}``. It +is the only flow parameter governing the incompressible Navier-Stokes +equations. + +### Kinetic energy + +The local and total kinetic energy are defined by ``k = \frac{1}{2} \| u +\|_2^2`` and ``K = \frac{1}{2} \| u \|_{L^2(\Omega)}^2 = \int_\Omega k \, +\mathrm{d} \Omega``. + ### Vorticity The vorticity is defined as ``\omega = \nabla \times u``. @@ -143,26 +155,27 @@ to the ``x^3``-component of the 3D vorticity. ### Stream function -The stream function ``\psi`` is a field (scalar in 2D, vector in 3D) defined -such that +In 2D, the stream function ``\psi`` is a scalar field such that ```math -u = \nabla \times \psi. +u^1 = \frac{\partial \psi}{\partial x^2}, \quad +u^2 = -\frac{\partial \psi}{\partial x^1}. ``` -It is related to the vorticity as +It can be found by solving ```math -\nabla^2 \psi = \nabla \times \omega. +\nabla^2 \psi = - \omega. ``` -### Kinetic energy +In 3D, the stream function is a vector field such that -The local and total kinetic energy are defined by ``k = \frac{1}{2} \| u -\|_2^2`` and ``K = \frac{1}{2} \| u \|_{L^2(\Omega)}^2 = \int_\Omega k \, -\mathrm{d} \Omega``. +```math +u = \nabla \times \psi. +``` -### Reynolds number +It can be found by solving -The Reynolds number is the inverse of the viscosity: ``Re = -\frac{1}{\nu}``. +```math +\nabla^2 \psi = \nabla \times \omega. +``` diff --git a/docs/src/equations/spatial.md b/docs/src/equations/spatial.md index a0e47b4d5..96456a202 100644 --- a/docs/src/equations/spatial.md +++ b/docs/src/equations/spatial.md @@ -268,8 +268,8 @@ Finally, the discrete ``\alpha``-momentum equations are given by ```math \begin{split} \left| \Omega_{I + \delta(\alpha) / 2} \right| - \frac{\mathrm{d} }{\mathrm{d} t} u^\alpha_{I + \delta(\alpha) / 2} = - & - \sum_{\beta = 1}^d \left| \Gamma^\beta_{I + \delta(\alpha) / 2} \right| + & \frac{\mathrm{d} }{\mathrm{d} t} u^\alpha_{I + \delta(\alpha) / 2} = \\ + - & \sum_{\beta = 1}^d \left| \Gamma^\beta_{I + \delta(\alpha) / 2} \right| \left( (u^\alpha u^\beta)_{I + \delta(\alpha) / 2 + \delta(\beta) / 2} @@ -277,9 +277,9 @@ Finally, the discrete ``\alpha``-momentum equations are given by (u^\alpha u^\beta )_{I + \delta(\alpha) / 2 - \delta(\beta) / 2} \right) \\ - & - \left| \Gamma^\alpha_{I + \delta(\alpha) / 2} \right| + - & \left| \Gamma^\alpha_{I + \delta(\alpha) / 2} \right| \left( p_{I + \delta(\alpha)} - p_{I} \right) \\ - & + \nu \sum_{\beta = 1}^d \left| \Gamma^\beta_{I + \delta(\alpha) / 2} + + & \nu \sum_{\beta = 1}^d \left| \Gamma^\beta_{I + \delta(\alpha) / 2} \right| \left( \frac{u^\alpha_{I + \delta(\alpha) / 2 + \delta(\beta)} - u^\alpha_{I + @@ -288,7 +288,7 @@ Finally, the discrete ``\alpha``-momentum equations are given by \delta(\alpha) / 2 - \delta(\beta)}}{x^\beta_{I(\beta)} - x^\beta_{I(\beta) - 1}} \right) \\ - & + \left| \Omega_{I + \delta(\alpha) / 2} \right| f^\alpha(x_{I + + + & \left| \Omega_{I + \delta(\alpha) / 2} \right| f^\alpha(x_{I + \delta(\alpha) / 2}). \end{split} ``` @@ -469,3 +469,123 @@ symmetric since ``G = \Omega_h^{-1} M^\mathsf{T}``. of discretely divergence free velocities. However, using this formulation would require an efficient way to perform the projection without assembling the operator matrix ``L^{-1}``, which would be very costly. + +## Discrete output quantities + +### Kinetic energy + +The local kinetic energy is defined by ``k = \frac{1}{2} \| u \|_2^2 = +\frac{1}{2} \sum_{\alpha = 1}^d u^\alpha u^\alpha``. On the staggered grid +however, the different velocity components are not located at the same point. +We will therefore interpolate the velocity to the pressure point before summing +the squares. + +### Vorticity + +In 2D, the vorticity is a scalar. Integrating the vorticity ``\omega = +-\frac{\partial u^1}{\partial x^2} + \frac{\partial u^2}{\partial x^1}`` over +the vorticity volume ``\Omega_{I + \delta(1) / 2 + \delta(2) / 2}`` gives + +```math +\begin{split} +\int_{\Omega_{I + \delta(1) / 2 + \delta(2) / 2}} \omega \, \mathrm{d} \Omega += & - \left( +\int_{\Gamma^2_{I + \delta(1) / 2 + \delta(2)}} u^1 \, \mathrm{d} \Gamma +- \int_{\Gamma^2_{I + \delta(1) / 2}} u^1 \, \mathrm{d} \Gamma +\right) \\ +& + \left( +\int_{\Gamma^1_{I + \delta(1) + \delta(2) / 2}} u^2 \, \mathrm{d} \Gamma +- \int_{\Gamma^1_{I + \delta(2) / 2}} u^2 \, \mathrm{d} \Gamma +\right) +\end{split}. +``` + +Using quadrature, the discrete vorticity in the corner is given by + +```math +\begin{split} +\left| \Omega_{I + \delta(1) / 2 + \delta(2) / 2} \right| +\omega_{I + \delta(1) / 2 + \delta(2) / 2} = +& - \left| \Gamma^2_{I + \delta(1) / 2 + \delta(2) / 2} \right| +(u^1_{I + \delta(1) / 2 + \delta(2)} +- u^1_{I + \delta(1) / 2}) \\ +& + \left| \Gamma^1_{I + \delta(1) / 2 + \delta(2) / 2} \right| +(u^2_{I + \delta(1) + \delta(2) / 2} +- u^2_{I + \delta(2) / 2}) +\end{split}. +``` + +The 3D vorticity is a vector field ``(\omega^1, \omega^2, \omega^3)``. +Noting ``\alpha^+ = \operatorname{mod}_3(\alpha + 1)`` and +``\alpha^- = \operatorname{mod}_3(\alpha - 1)``, the vorticity is obtained +through + +```math +\begin{split} +\int_{\Omega_{I + \delta(\alpha^+) / 2 + \delta(\alpha^-) / 2}} \omega \, \mathrm{d} \Omega += & - \left( +\int_{\Gamma^{\alpha^-}_{I + \delta(\alpha^+) / 2 + \delta(\alpha^-)}} u^{\alpha^+} \, \mathrm{d} \Gamma +- \int_{\Gamma^{\alpha^-}_{I + \delta(\alpha^+) / 2}} u^{\alpha^+} \, \mathrm{d} \Gamma +\right) \\ +& + \left( +\int_{\Gamma^{\alpha^+}_{I + \delta(\alpha^+) + \delta(\alpha^-) / 2}} u^{\alpha^-} \, \mathrm{d} \Gamma +- \int_{\Gamma^{\alpha^+}_{I + \delta(\alpha^-) / 2}} u^{\alpha^-} \, \mathrm{d} \Gamma +\right) +\end{split}. +``` + +```math +\begin{split} +\left| \Omega_{I + \delta(\alpha^+) / 2 + \delta(\alpha^-) / 2} \right| +\omega_{I + \delta(\alpha^+) / 2 + \delta(\alpha^-) / 2} = +& - \left| \Gamma^{\alpha^-}_{I + \delta(\alpha^+) / 2 + \delta(\alpha^-) / 2} \right| +(u^{\alpha^+}_{I + \delta(\alpha^+) / 2 + \delta(\alpha^-)} +- u^{\alpha^+}_{I + \delta(\alpha^+) / 2}) \\ +& + \left| \Gamma^{\alpha^+}_{I + \delta(\alpha^+) / 2 + \delta(\alpha^-) / 2} \right| +(u^{\alpha^-}_{I + \delta(\alpha^+) + \delta(\alpha^-) / 2} +- u^{\alpha^-}_{I + \delta(\alpha^-) / 2}) +\end{split}. +``` + +## Stream function + +In 2D, the stream function is defined at the corners with the vorticity. +Integrating the stream function Poisson equation over the vorticity volume +yields + +```math +\begin{split} +- \int_{\Omega_{I + \delta(1) / 2 + \delta(2) / 2}} \omega \, \mathrm{d} \Omega +& = \int_{\Omega_{I + \delta(1) / 2 + \delta(2) / 2}} \nabla^2 \psi \, +\mathrm{d} \Omega \\ +& = \int_{\Gamma^1_{I + \delta(1) + \delta(2) / 2}} \frac{\partial \psi}{\partial x^1} +\, \mathrm{d} \Gamma +- \int_{\Gamma^1_{I + \delta(2) / 2}} \frac{\partial \psi}{\partial x^1} +\, \mathrm{d} \Gamma \\ +& + \int_{\Gamma^2_{I + \delta(1) / 2 + \delta(2)}} \frac{\partial \psi}{\partial x^2} +\, \mathrm{d} \Gamma +- \int_{\Gamma^2_{I + \delta(1) / 2}} \frac{\partial \psi}{\partial x^2} +\, \mathrm{d} \Gamma. +\end{split} +``` + +Replacing the integrals with the mid-point quadrature rule and the spatial +derivatives with central finite differences yields the discrete Poisson +equation for the stream function at the vorticity point: + +```math +\begin{split} +\left| \Gamma^1_{I + \delta(1) / 2 + \delta(2) / 2} \right| +\left( + \frac{\psi_{I + 3 / 2 \delta(1) + \delta(2) / 2} - \psi_{I + \delta(1) / 2 + \delta(2) / 2}}{x^1_{I(1) + 3 / 2} - x^1_{I(1) + 1 /2}} +- \frac{\psi_{I + \delta(1) / 2 + \delta(2) / 2} - \psi_{I - \delta(1) / 2 + \delta(2) / 2}}{x^1_{I(1) + 1 / 2} - x^1_{I(1) - 1 / 2}} +\right) & + \\ +\left| \Gamma^2_{I + \delta(1) / 2 + \delta(2) / 2} \right| +\left( +\frac{\psi_{I + \delta(1) / 2 + 3 / 2 \delta(2)} - \psi_{I + \delta(1) / 2 + \delta(2) / 2}}{x^2_{I(1) + 3 / 2} - x^2_{I(1) + 1 / 2}} +-\frac{\psi_{I + \delta(1) / 2 + \delta(2) / 2} - \psi_{I + \delta(1) / 2 - \delta(2) / 2}}{x^2_{I(2) + 1 / 2} - x^2_{I(2) - 1 / 2}} +\right) & = \\ +\left| \Omega_{I + \delta(1) / 2 + \delta(2) / 2} \right| +\omega_{I + \delta(1) / 2 + \delta(2) / 2} & +\end{split} +``` diff --git a/docs/src/equations/time.md b/docs/src/equations/time.md index 2a5c39b22..ae39eac7c 100644 --- a/docs/src/equations/time.md +++ b/docs/src/equations/time.md @@ -49,28 +49,28 @@ are computed as follows: ```math \begin{split} -F_i & = F(U_{i - 1}, t_{i - 1}) \\ +F_i & = F(U_{i - 1}, t_{i - 1}) - y_G(t_{i - 1}) \\ V_i & = U_0 + \Delta t \sum_{j = 1}^i a_{i j} F_j \\ L P_i & = \frac{1}{c_i} \sum_{j = 1}^i a_{i j} F_j + \frac{y_M(t_i) - y_M(t_0)}{\Delta t_i} \\ & = \frac{(M V_i + y_M(t_i)) - (M U_0 + y_M(t_0))}{\Delta t_i^n} \\ & = \frac{M V_i + y_M(t_i)}{\Delta t_i^n} \\ -U_i & = V_i - \Delta t_i (G P_i + y_G(t_i)), +U_i & = V_i - \Delta t_i G P_i, \end{split} ``` -where ``(a_{i j})_{i j}`` are the Butcher tableau coefficients of the RK-method, -with the convention ``c_i = \sum_{j = 1}^i a_{i j}``. +where ``(a_{i j})_{i j}`` are the Butcher tableau coefficients of the +RK-method, with the convention ``c_i = \sum_{j = 1}^i a_{i j}``. Finally, we set ``U = U_s``. If ``U_0 = u_h(t_0)``, we get the accuracy ``U = u_h(t) + \mathcal{O}(\Delta t^{r + 1})``, where ``r`` is the order of the RK-method. If we perform ``n`` RK time steps instead of one, starting at exact -initial conditions ``U^0 = u_h(0)``, then ``U^n = u_h(t^n) + -\mathcal{O}(\Delta t^r)`` for all ``n \in \{1, \dots, N\}``. Note that for a -given ``U``, the corresponding pressure ``P`` can be calculated to the same -accuracy as ``U`` by doing an additional pressure projection after each outer -time step ``\Delta t`` (if we know ``\frac{\mathrm{d} y_M}{\mathrm{d} t}(t)``), -or to first order accuracy by simply taking ``P = P_s``. +initial conditions ``U^0 = u_h(0)``, then ``U^n = u_h(t^n) + \mathcal{O}(\Delta +t^r)`` for all ``n \in \{1, \dots, N\}``. Note that for a given ``U``, the +corresponding pressure ``P`` can be calculated to the same accuracy as ``U`` by +doing an additional pressure projection after each outer time step ``\Delta t`` +(if we know ``\frac{\mathrm{d} y_M}{\mathrm{d} t}(t)``), or to first order +accuracy by simply taking ``P = P_s``. Note that each of the sub-step velocities ``U_i`` is divergence free, after projecting the tentative velocities ``V_i``. This is ensured due to the @@ -113,8 +113,8 @@ terms containing ``V`` on the left hand side, to obtain ```math \begin{split} -\left( \frac{1}{\Delta t} I - \theta D \right) V -& = \left(\frac{1}{\Delta t} I - (1 - \theta) D \right) U_0 \\ +\left( \frac{1}{\Delta t} I - (1 - \theta) D \right) V +& = \left(\frac{1}{\Delta t} I - \theta D \right) U_0 \\ & - (\alpha_0 C(U_0, t_0) + \alpha_{-1} C(U_{-1}, t_{-1})) \\ & + \theta y_D(t_0) + (1 - \theta) y_D(t) \\ & + \theta f(t_0) + (1 - \theta) f(t) \\ From bfce3d15cbc598155d83a49b6e4c7d60f319e940 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Mon, 18 Sep 2023 17:44:30 +0200 Subject: [PATCH 057/379] Force p=0 for boundary --- src/boundary_conditions.jl | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/boundary_conditions.jl b/src/boundary_conditions.jl index d15e78154..2ae637ae5 100644 --- a/src/boundary_conditions.jl +++ b/src/boundary_conditions.jl @@ -9,9 +9,7 @@ end struct SymmetricBC <: AbstractBC end -struct PressureBC{F} <: AbstractBC - p::F -end +struct PressureBC <: AbstractBC end function ghost_a! end function ghost_b! end From d92c548a86348abc6fb3d28d1fc35e6a9c6080d3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Mon, 18 Sep 2023 17:47:25 +0200 Subject: [PATCH 058/379] docs: Document BC --- src/boundary_conditions.jl | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/src/boundary_conditions.jl b/src/boundary_conditions.jl index 2ae637ae5..09aa9ff5d 100644 --- a/src/boundary_conditions.jl +++ b/src/boundary_conditions.jl @@ -2,13 +2,44 @@ abstract type AbstractBC end struct PeriodicBC <: AbstractBC end +""" + DirichletBC() + +No split boundary conditions, where all velocity components are zero. + + DirichletBC(u, dudt) + +Dirichlet boundary conditions for the velocity, where `u[1] = (x..., t) -> +u1_BC` up to `u[d] = (x..., t) -> ud_BC`, where `d` is the dimension. + +To make the pressure the same order as velocity, also provide `dudt`. +""" struct DirichletBC{F,G} <: AbstractBC u::F dudt::G end +DirichletBC() = DirichletBC(nothing, nothing) + +""" + SymmetricBC() + +Symmetric boundary conditions. +The parallel velocity and pressure is the same at each side of the boundary. +The normal velocity is zero. +""" struct SymmetricBC <: AbstractBC end +""" + PressureBC() + +Pressure boundary conditions. +The pressure is prescribed on the boundary (usually an "outlet"). +The velocity has zero Neumann conditions. + +Note: Currently, the pressure is prescribed with the constant value of +zero on the entire boundary. +""" struct PressureBC <: AbstractBC end function ghost_a! end From 380c53d26196a641c09aea74e5b84502911020ea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Mon, 18 Sep 2023 17:47:41 +0200 Subject: [PATCH 059/379] Put constant at top level --- src/IncompressibleNavierStokes.jl | 4 ++++ src/operators.jl | 3 --- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/IncompressibleNavierStokes.jl b/src/IncompressibleNavierStokes.jl index b73526365..7e549e0e5 100644 --- a/src/IncompressibleNavierStokes.jl +++ b/src/IncompressibleNavierStokes.jl @@ -22,6 +22,10 @@ using Statistics using WriteVTK: CollectionFile, paraview_collection, vtk_grid, vtk_save using Zygote +# Workgroup size for kernels +# Let this be constant for now +const WORKGROUP = 64 + # Convenience notation const ⊗ = kron diff --git a/src/operators.jl b/src/operators.jl index b2ac5b812..e19079a1c 100644 --- a/src/operators.jl +++ b/src/operators.jl @@ -1,6 +1,3 @@ -# Let this be constant for now -const WORKGROUP = 64 - # See https://b-fg.github.io/2023/05/07/waterlily-on-gpu.html # for writing kernel loops From d6525fa5f67438177b38503ae3bc438b725f001e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Mon, 18 Sep 2023 17:50:13 +0200 Subject: [PATCH 060/379] Update stepper --- .../step_explicit_runge_kutta.jl | 60 ++++++++----------- src/time_steppers/time_stepper_caches.jl | 32 ++++------ 2 files changed, 39 insertions(+), 53 deletions(-) diff --git a/src/time_steppers/step_explicit_runge_kutta.jl b/src/time_steppers/step_explicit_runge_kutta.jl index fd11d2df6..2adf842c9 100644 --- a/src/time_steppers/step_explicit_runge_kutta.jl +++ b/src/time_steppers/step_explicit_runge_kutta.jl @@ -87,63 +87,54 @@ end function timestep!(method::ExplicitRungeKuttaMethod, stepper, Δt; cache) (; setup, pressure_solver, u, p, t, n) = stepper (; grid, boundary_conditions) = setup - (; dimension, Ip, Ωu) = grid + (; dimension, Ip) = grid (; A, b, c, p_add_solve) = method - (; uₙ, ku, v, F, M, G, f) = cache + (; u₀, ku, v, F, M, G) = cache D = dimension() # Update current solution (does not depend on previous step size) - n += 1 - tₙ = t - Δtₙ = Δt + t₀ = t for α = 1:D - uₙ[α] .= u[α] + u₀[α] .= u[α] end # Number of stages nstage = length(b) - # # Reset RK arrays - # ku .= 0 - - tᵢ = tₙ - ## Start looping over stages - # At i = 1 we calculate F₁, p₂ and u₂ + # At i = 1 we calculate F₁ = F(u₀), p₁ and u₁ # ⋮ - # At i = s we calculate Fₛ, pₙ₊₁, and uₙ₊₁ + # At i = s we calculate Fₛ = F(uₛ₋₁), pₛ, and uₛ for i = 1:nstage - # Right-hand side for tᵢ based on current velocity field uₕ, vₕ at level i. This - # includes force evaluation at tᵢ and pressure gradient. The pressure p is not important here, - # it will be removed again in the next step - momentum!(F, u, tᵢ, setup) + # Right-hand side for tᵢ₋₁ based on current velocity field uᵢ₋₁, vᵢ₋₁ at + # level i-1. This includes force evaluation at tᵢ₋₁. + momentum!(F, u, t, setup) # Store right-hand side of stage i for α = 1:D @. ku[i][α] = F[α] end + # Intermediate time step + t = t₀ + c[i] * Δt + # Update velocity current stage by sum of Fᵢ's until this stage, weighted - # with Butcher tableau coefficients. This gives uᵢ₊₁, and for i=s gives uᵢ₊₁ + # with Butcher tableau coefficients. This gives vᵢ for α = 1:D - v[α] .= uₙ[α] - end - for j = 1:i - for α = 1:D - v[α] .= v[α] .+ Δtₙ .* A[i, j] .* ku[j][α] + v[α] .= u₀[α] + for j = 1:i + @. v[α] = v[α] + Δt * A[i, j] * ku[j][α] end end - # Boundary conditions at tᵢ₊₁ - tᵢ = tₙ + c[i] * Δtₙ - apply_bc_u!(v, tᵢ, setup) + # Boundary conditions at tᵢ + apply_bc_u!(v, t, setup) # Divergence of tentative velocity field divergence!(M, v, setup) - - @. M = M / (c[i] * Δtₙ) + @. M = M / (c[i] * Δt) # Solve the Poisson equation Min = view(M, Ip) @@ -156,15 +147,16 @@ function timestep!(method::ExplicitRungeKuttaMethod, stepper, Δt; cache) # Update velocity current stage, which is now divergence free for α = 1:D - @. u[α] = v[α] - c[i] * Δtₙ * G[α] + @. u[α] = v[α] - c[i] * Δt * G[α] end + apply_bc_u!(u, tᵢ, setup) end - # Do additional pressure solve to avoid first order pressure - p_add_solve && - pressure_additional_solve!(pressure_solver, u, p, tₙ + Δtₙ, setup, F, M) + # Complete time step + t = t₀ + Δt - t = tₙ + Δtₙ + # Do additional pressure solve to avoid first order pressure + p_add_solve && pressure_additional_solve!(pressure_solver, u, p, t, setup, F, M) - create_stepper(method; setup, pressure_solver, u, p, t, n) + create_stepper(method; setup, pressure_solver, u, p, t, n = n + 1) end diff --git a/src/time_steppers/time_stepper_caches.jl b/src/time_steppers/time_stepper_caches.jl index 8de8c31d9..3b2ac1e5b 100644 --- a/src/time_steppers/time_stepper_caches.jl +++ b/src/time_steppers/time_stepper_caches.jl @@ -7,48 +7,42 @@ Get time stepper cache for the given ODE method. function ode_method_cache end function ode_method_cache(::AdamsBashforthCrankNicolsonMethod, setup, V, p) - cₙ = similar(V) - cₙ₋₁ = similar(V) + c₀ = similar(V) + c₋₁ = similar(V) F = similar(V) f = similar(p) Δp = similar(p) Rr = similar(V) b = similar(V) - bₙ = similar(V) - bₙ₊₁ = similar(V) - yDiffₙ = similar(V) - yDiffₙ₊₁ = similar(V) - Gpₙ = similar(V) + b₀ = similar(V) + b₁ = similar(V) + yDiff₀ = similar(V) + yDiff₁ = similar(V) + Gp₀ = similar(V) - (; cₙ, cₙ₋₁, F, f, Δp, Rr, b, bₙ, bₙ₊₁, yDiffₙ, yDiffₙ₊₁, Gpₙ) + (; c₀, c₋₁, F, f, Δp, Rr, b, b₀, b₁, yDiff₀, yDiff₁, Gp₀) end function ode_method_cache(::OneLegMethod{T}, setup, V, p) where {T} (; NV, Np) = setup.grid - Vₙ₋₁ = similar(V) - pₙ₋₁ = similar(p) + u₋₁ = similar(V) + p₋₁ = similar(p) F = similar(V) f = similar(p) Δp = similar(p) GΔp = similar(V) - (; Vₙ₋₁, pₙ₋₁, F, f, Δp, GΔp) + (; u₋₁, p₋₁, F, f, Δp, GΔp) end function ode_method_cache(method::ExplicitRungeKuttaMethod{T}, setup, u, p) where {T} - - uₙ = similar.(u) - + u₀ = similar.(u) ns = nstage(method) - ku = [similar.(u) for i = 1:ns] - v = similar.(u) F = similar.(u) G = similar.(u) M = similar(p) - f = similar(p) - - (; uₙ, ku, v, F, M, G, f) + (; u₀, ku, v, F, M, G) end function ode_method_cache(method::ImplicitRungeKuttaMethod{T}, setup, V, p) where {T} From f880b49786566bbb239df30fff98209394a70799 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Mon, 18 Sep 2023 17:50:38 +0200 Subject: [PATCH 061/379] Remove obsolete method --- .../step_explicit_runge_kutta.jl | 83 ------------------- 1 file changed, 83 deletions(-) diff --git a/src/time_steppers/step_explicit_runge_kutta.jl b/src/time_steppers/step_explicit_runge_kutta.jl index 2adf842c9..e578a7cc4 100644 --- a/src/time_steppers/step_explicit_runge_kutta.jl +++ b/src/time_steppers/step_explicit_runge_kutta.jl @@ -1,89 +1,6 @@ create_stepper(::ExplicitRungeKuttaMethod; setup, pressure_solver, u, p, t, n = 0) = (; setup, pressure_solver, u, p, t, n) -function timestep(method::ExplicitRungeKuttaMethod, stepper, Δt) - (; setup, pressure_solver, u, p, t, n) = stepper - (; grid, boundary_conditions) = setup - (; Ω) = grid - (; A, b, c, p_add_solve) = method - - T = typeof(Δt) - - # Update current solution (does not depend on previous step size) - n += 1 - Vₙ = u - pₙ = p - tₙ = t - Δtₙ = Δt - - # Number of stages - nV = length(u) - np = length(p) - nstage = length(b) - - # Reset RK arrays - tᵢ = tₙ - # kV = zeros(T, nV, 0) - ku = fill(u, 0) - - ## Start looping over stages - - # At i = 1 we calculate F₁, p₂ and u₂ - # ⋮ - # At i = s we calculate Fₛ, pₙ₊₁, and uₙ₊₁ - for i = 1:nstage - # Right-hand side for tᵢ based on current velocity field uₕ, vₕ at level i. This - # includes force evaluation at tᵢ and pressure gradient. Boundary conditions will be - # set through `get_bc_vectors` inside momentum. The pressure p is not important here, - # it will be removed again in the next step - F = momentum(u, p, tᵢ, setup) - - # Store right-hand side of stage i - # Remove the -G*p contribution (but not y_p) - kVᵢ = 1 ./ Ω .* (F + G * p) - # kV = [kV kVᵢ] - kV = [ku; [kVᵢ]] - - # Update velocity current stage by sum of Fᵢ's until this stage, weighted - # with Butcher tableau coefficients. This gives uᵢ₊₁, and for i=s gives uᵢ₊₁ - # V = dot(kV * A[i, 1:i] - k = sum(A[i, j] * kV[j] for j = 1:i) - - # Boundary conditions at tᵢ₊₁ - tᵢ = tₙ + c[i] * Δtₙ - if bc_unsteady - bc_vectors = get_bc_vectors(setup, tᵢ) - end - (; yM) = bc_vectors - - # Divergence of intermediate velocity field - f = (M * (Vₙ / Δtₙ + k) + yM / Δtₙ) / c[i] - - # Solve the Poisson equation, but not for the first step if the boundary conditions are steady - if boundary_conditions.bc_unsteady || i > 1 - p = pressure_poisson(pressure_solver, f) - else - # Bc steady AND i = 1 - p = pₙ - end - - Gp = G * p - - # Update velocity current stage, which is now divergence free - V = @. Vₙ + Δtₙ * (k - c[i] / Ω * Gp) - end - - # For steady bc we do an additional pressure solve - # That saves a pressure solve for i = 1 in the next time step - if !bc_unsteady || p_add_solve - p = pressure_additional_solve(pressure_solver, V, p, tₙ + Δtₙ, setup; bc_vectors) - end - - t = tₙ + Δtₙ - - create_stepper(method; setup, pressure_solver, bc_vectors, u, p, t, n) -end - function timestep!(method::ExplicitRungeKuttaMethod, stepper, Δt; cache) (; setup, pressure_solver, u, p, t, n) = stepper (; grid, boundary_conditions) = setup From 45362528d6f51693225ac827b3fc7d67fed2537a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Mon, 18 Sep 2023 17:52:26 +0200 Subject: [PATCH 062/379] Update processors --- src/operators.jl | 95 ++++++++++++++++++++++++++++--- src/postprocess/plot_pressure.jl | 8 +-- src/postprocess/plot_velocity.jl | 38 +++++++------ src/postprocess/plot_vorticity.jl | 53 +++++++---------- src/postprocess/save_vtk.jl | 34 +++++------ src/processors/real_time_plot.jl | 31 ++++------ 6 files changed, 158 insertions(+), 101 deletions(-) diff --git a/src/operators.jl b/src/operators.jl index e19079a1c..22700b427 100644 --- a/src/operators.jl +++ b/src/operators.jl @@ -64,23 +64,21 @@ end function vorticity!(::Dimension{3}, ω, u, setup) (; boundary_conditions, grid) = setup - (; dimension, Δu, Nu, Iω, Ωω) = grid + (; dimension, Δu, N) = grid D = dimension() δ = Offset{D}() - @kernel function _vorticity!(ω, u, α, I0; Δu) + @kernel function _vorticity!(ω, u, α, I0) T = eltype(ω) I = @index(Global, Cartesian) I = I + I0 β = mod1(α + 1, D) γ = mod1(α - 1, D) - ω[α][I] = - -Ωω[I] / Δu[γ] * (u[β][I+δ(γ)] - u[β][I]) + - Ωω[I] / Δu[β] * (u[γ][I+δ(β)] - u[γ][I]) + ω[α][I] = -(u[β][I+δ(γ)] - u[β][I]) / Δu[γ][I[γ]] + (u[γ][I+δ(β)] - u[γ][I]) / Δu[β][I[β]] end for α = 1:D - I0 = first(Iu[α]) + I0 = CartesianIndex(ntuple(Returns(1), D)) I0 -= oneunit(I0) - _vorticity!(get_backend(ω), WORKGROUP)(ω, u, α, I0; Δu, ndrange = N .- 1) + _vorticity!(get_backend(ω[1]), WORKGROUP)(ω, u, α, I0; ndrange = N .- 1) synchronize(get_backend(ω[1])) end ω @@ -88,7 +86,7 @@ end function convection!(F, u, setup) (; boundary_conditions, grid, Re, bodyforce) = setup - (; dimension, Δ, Δu, Nu, Iu, Γu, Ωu) = grid + (; dimension, Δ, Δu, Nu, Iu) = grid D = dimension() δ = Offset{D}() @kernel function _convection!(F, u, α, β, I0) @@ -115,7 +113,7 @@ end # Add ϵ in denominator for "infinitely thin" volumes function diffusion!(F, u, setup; ϵ = eps(eltype(F[1]))) (; boundary_conditions, grid, Re, bodyforce) = setup - (; dimension, Δ, Δu, Nu, Iu, Ωu, Γu) = grid + (; dimension, Δ, Δu, Nu, Iu) = grid D = dimension() δ = Offset{D}() ν = 1 / Re @@ -208,3 +206,82 @@ pressuregradient(p, setup) = pressuregradient!( p, setup, ) + +interpolate_u_p(setup, u) = interpolate_u_p!( + setup, + ntuple( + α -> KernelAbstractions.zeros(get_backend(u[1]), eltype(u[1]), setup.grid.N), + setup.grid.dimension(), + ), + u, +) + +function interpolate_u_p!(setup, up, u) + (; boundary_conditions, grid, Re, bodyforce) = setup + (; dimension, Np, Ip) = grid + D = dimension() + δ = Offset{D}() + @kernel function _interpolate_u_p!(up, u, α, I0) + I = @index(Global, Cartesian) + I = I + I0 + up[α][I] = (u[α][I-δ(α)] + u[α][I]) / 2 + end + for α = 1:D + I0 = first(Ip) + I0 -= oneunit(I0) + _interpolate_u_p!(get_backend(up[1]), WORKGROUP)(up, u, α, I0; ndrange = Np) + synchronize(get_backend(up[1])) + end + up +end + +interpolate_ω_p(setup, ω) = interpolate_ω_p!( + setup, + setup.grid.dimension() == 2 ? + KernelAbstractions.zeros(get_backend(ω), eltype(ω), setup.grid.N) : + ntuple( + α -> KernelAbstractions.zeros(get_backend(ω[1]), eltype(ω[1]), setup.grid.N), + setup.grid.dimension(), + ), + ω, +) + +interpolate_ω_p!(setup, ωp, ω) = interpolate_ω_p!(setup.grid.dimension, setup, ωp, ω) + +function interpolate_ω_p!(::Dimension{2}, setup, ωp, ω) + (; boundary_conditions, grid, Re, bodyforce) = setup + (; dimension, Np, Ip) = grid + D = dimension() + δ = Offset{D}() + @kernel function _interpolate_ω_p!(ωp, ω, I0) + I = @index(Global, Cartesian) + I = I + I0 + ωp[I] = (ω[I-δ(1)-δ(2)] + ω[I]) / 2 + end + I0 = first(Ip) + I0 -= oneunit(I0) + _interpolate_ω_p!(get_backend(ωp), WORKGROUP)(ωp, ω, I0; ndrange = Np) + synchronize(get_backend(ωp)) + ωp +end + +function interpolate_ω_p!(::Dimension{3}, setup, ωp, ω) + (; boundary_conditions, grid, Re, bodyforce) = setup + (; dimension, Np, Ip) = grid + D = dimension() + δ = Offset{D}() + @kernel function _interpolate_ω_p!(ωp, ω, α, I0) + I = @index(Global, Cartesian) + I = I + I0 + β = mod1(α + 1, D) + γ = mod1(α - 1, D) + ωp[α][I] = (ω[α][I-δ(β)-δ(γ)] + ω[α][I]) / 2 + end + I0 = first(Ip) + I0 -= oneunit(I0) + for α = 1:D + _interpolate_ω_p!(get_backend(ωp[1]), WORKGROUP)(ωp, ω, α, I0; ndrange = Np) + end + synchronize(get_backend(ωp[1])) + ωp +end diff --git a/src/postprocess/plot_pressure.jl b/src/postprocess/plot_pressure.jl index 1a417bad3..d734f1984 100644 --- a/src/postprocess/plot_pressure.jl +++ b/src/postprocess/plot_pressure.jl @@ -44,16 +44,14 @@ end # 3D version function plot_pressure(::Dimension{3}, setup, p; kwargs...) - (; Nx, Ny, Nz, Npx, Npy, Npz, xp, yp, zp) = setup.grid - - # Reshape - p = reshape(p, Npx, Npy, Npz) + (; xp) = setup.grid # Levels μ, σ = mean(p), std(p) levels = LinRange(μ - 5σ, μ + 5σ, 10) - contour(xp, yp, zp, p; levels, kwargs...) + p = Array(p) + contour(xp..., p; levels, kwargs...) # save("output/pressure.png", fig, pt_per_unit = 2) end diff --git a/src/postprocess/plot_velocity.jl b/src/postprocess/plot_velocity.jl index e37deee90..b67e56541 100644 --- a/src/postprocess/plot_velocity.jl +++ b/src/postprocess/plot_velocity.jl @@ -5,21 +5,23 @@ Plot velocity. """ function plot_velocity end -plot_velocity(setup, V, t; kwargs...) = - plot_velocity(setup.grid.dimension, setup, V, t; kwargs...) +plot_velocity(setup, u; kwargs...) = + plot_velocity(setup.grid.dimension, setup, u; kwargs...) # 2D version -function plot_velocity(::Dimension{2}, setup, V, t; kwargs...) - (; xp, yp, xlims, ylims) = setup.grid +function plot_velocity(::Dimension{2}, setup, u; kwargs...) + (; xp, xlims) = setup.grid + T = eltype(xp[1]) # Get velocity at pressure points - up, vp = get_velocity(setup, V, t) - qp = map((u, v) -> √(u^2 + v^2), up, vp) + up = interpolate_u_p(setup, u) + # qp = map((u, v) -> √(u^2 + v^2), up, vp) + qp = sqrt.(up[1] .^ 2 .+ up[2] .^ 2) # Levels μ, σ = mean(qp), std(qp) - ≈(μ + σ, μ; rtol = 1e-8, atol = 1e-8) && (σ = 1e-4) - levels = LinRange(μ - 1.5σ, μ + 1.5σ, 10) + # ≈(μ + σ, μ; rtol = 1e-8, atol = 1e-8) && (σ = 1e-4) + levels = LinRange(μ - T(1.5) * σ, μ + T(1.5) * σ, 10) fig = Figure() ax = Axis( @@ -29,27 +31,27 @@ function plot_velocity(::Dimension{2}, setup, V, t; kwargs...) xlabel = "x", ylabel = "y", ) - limits!(ax, xlims[1], xlims[2], ylims[1], ylims[2]) - cf = contourf!(ax, xp, yp, qp; extendlow = :auto, extendhigh = :auto, levels, kwargs...) + limits!(ax, xlims[1]..., xlims[2]...) + cf = contourf!(ax, xp..., qp; extendlow = :auto, extendhigh = :auto, levels, kwargs...) Colorbar(fig[1, 2], cf) # Colorbar(fig[2,1], cf; vertical = false) fig end # 3D version -function plot_velocity(::Dimension{3}, setup, V, t; kwargs...) - (; xu, yu, zu, indu) = setup.grid +function plot_velocity(::Dimension{3}, setup, u; kwargs...) + (; xp) = setup.grid # Get velocity at pressure points - # up, vp, wp = get_velocity(setup, V, t) - # qp = map((u, v, w) -> √sum(u^2 + v^2 + w^2), up, vp, wp) - qp = reshape(V[indu], size(xu)) - xp, yp, zp = xu[:, 1, 1], yu[1, :, 1], zu[1, 1, :] + up = interpolate_u_p(setup, u) + qp = map((u, v, w) -> √sum(u^2 + v^2 + w^2), up...) # Levels μ, σ = mean(qp), std(qp) levels = LinRange(μ - 3σ, μ + 3σ, 10) - # contour(xp, yp, zp, qp; levels, kwargs...) - contour(xp, yp, zp, qp; kwargs...) + xp = Array(xp) + qp = Array(qp) + contour(xp..., qp; levels, kwargs...) + # contour(xp..., qp; kwargs...) end diff --git a/src/postprocess/plot_vorticity.jl b/src/postprocess/plot_vorticity.jl index 09790f36f..2cac6b6b0 100644 --- a/src/postprocess/plot_vorticity.jl +++ b/src/postprocess/plot_vorticity.jl @@ -5,21 +5,23 @@ Plot vorticity field. """ function plot_vorticity end -plot_vorticity(setup, u, t; kwargs...) = - plot_vorticity(setup.grid.dimension, setup, u, t; kwargs...) +plot_vorticity(setup, u; kwargs...) = + plot_vorticity(setup.grid.dimension, setup, u; kwargs...) # 2D version -function plot_vorticity(::Dimension{2}, setup, u, t; kwargs...) +function plot_vorticity(::Dimension{2}, setup, u; kwargs...) (; grid, boundary_conditions) = setup - (; x, xlims) = grid + (; xp, xlims) = grid + T = eltype(xp[1]) # Get fields - ω = vorticity(u, t) + ω = vorticity(u, setup) + ωp = interpolate_ω_p(setup, ω) # Levels μ, σ = mean(ω), std(ω) - ≈(μ + σ, μ; rtol = 1e-8, atol = 1e-8) && (σ = 1e-4) - levels = LinRange(μ - 1.5σ, μ + 1.5σ, 10) + # ≈(μ + σ, μ; rtol = 1e-8, atol = 1e-8) && (σ = 1e-4) + levels = LinRange(μ - T(1.5) * σ, μ + T(1.5) * σ, 10) # Plot vorticity fig = Figure() @@ -30,9 +32,9 @@ function plot_vorticity(::Dimension{2}, setup, u, t; kwargs...) xlabel = "x", ylabel = "y", ) - limits!(ax, xlims[1], xlims[2], ylims[1], ylims[2]) - # cf = contourf!(ax, xω, yω, ω; extendlow = :auto, extendhigh = :auto, levels, kwargs...) - cf = heatmap!(ax, xω, yω, ω; kwargs...) + limits!(ax, xlims[1]..., xlims[2]...) + cf = contourf!(ax, xp..., ω; extendlow = :auto, extendhigh = :auto, levels, kwargs...) + # cf = heatmap!(ax, xp..., ωp; kwargs...) Colorbar(fig[1, 2], cf) # save("output/vorticity.png", fig, pt_per_unit = 2) @@ -41,32 +43,19 @@ function plot_vorticity(::Dimension{2}, setup, u, t; kwargs...) end # 3D version -function plot_vorticity(::Dimension{3}, setup, V, t; kwargs...) +function plot_vorticity(::Dimension{3}, setup, u; kwargs...) (; grid, boundary_conditions) = setup - (; x, y, z) = grid + (; xp) = grid - if all( - ==(:periodic), - ( - boundary_conditions.u.x[1], - boundary_conditions.v.y[1], - boundary_conditions.w.z[1], - ), - ) - xω = x - yω = y - zω = z - else - xω = x[2:(end-1)] - yω = y[2:(end-1)] - zω = z[2:(end-1)] - end - - ω = get_vorticity(setup, V, t) + ωp = interpolate_ω_p(setup, vorticity(u, setup)) + qp = map((u, v, w) -> √sum(u^2 + v^2 + w^2), ωp...) # Levels - μ, σ = mean(ω), std(ω) + μ, σ = mean(qp), std(qp) levels = LinRange(μ - 3σ, μ + 3σ, 10) - contour(xω, yω, zω, ω; levels, kwargs...) + + xp = Array.(xp) + qp = Array(qp) + contour(xp..., qp; levels, kwargs...) end diff --git a/src/postprocess/save_vtk.jl b/src/postprocess/save_vtk.jl index de11ebee5..91bfe149a 100644 --- a/src/postprocess/save_vtk.jl +++ b/src/postprocess/save_vtk.jl @@ -1,32 +1,32 @@ """ - save_vtk(setup, V, p, t, filename = "output/solution") + save_vtk(setup, u, p, filename = "output/solution") Save velocity and pressure field to a VTK file. In the case of a 2D setup, the velocity field is saved as a 3D vector with a z-component of zero, as this seems to be preferred by ParaView. """ -function save_vtk(setup, V, p, t, filename = "output/solution") +function save_vtk(setup, u, p, filename = "output/solution") parts = split(filename, "/") path = join(parts[1:end-1], "/") isdir(path) || mkpath(path) (; grid) = setup - N = setup.grid.dimension() - if N == 2 - (; xp, yp) = grid - coords = (xp, yp) - elseif N == 3 - (; xp, yp, zp) = grid - coords = (xp, yp, zp) - end - vtk_grid(filename, coords...) do vtk - vels = get_velocity(setup, V, t) - if N == 2 + (; dimension, xp) = grid + D = dimension() + xp = Array.(xp) + vtk_grid(filename, xp...) do vtk + up = interpolate_u_p(setup, u) + ωp = interpolate_ω_p(setup, vorticity(u, setup)) + if D == 2 # ParaView prefers 3D vectors. Add zero z-component. - wp = zero(vels[1]) - vels = (vels..., wp) + up3 = zero(up[1]) + up = (up..., up3) + ωp = Array(ωp) + else + ωp = Array.(ωp) end - vtk["velocity"] = vels - vtk["pressure"] = p + vtk["velocity"] = Array.(up) + vtk["pressure"] = Array(p) + vtk["vorticity"] = ωp end end diff --git a/src/processors/real_time_plot.jl b/src/processors/real_time_plot.jl index dbeca4707..9e37f68d3 100644 --- a/src/processors/real_time_plot.jl +++ b/src/processors/real_time_plot.jl @@ -148,20 +148,12 @@ function field_plot( displayfig = true, ) (; boundary_conditions, grid) = setup - (; xlims, ylims, x, y, z, xp, yp, zp) = grid + (; xlims, x, xp) = grid if fieldname == :velocity xf, yf, zf = xp, yp, zp elseif fieldname == :vorticity - if all(==(:periodic), (boundary_conditions.u.x[1], boundary_conditions.v.y[1])) - xf = x - yf = y - zf = y - else - xf = x[2:(end-1)] - yf = y[2:(end-1)] - zf = z[2:(end-1)] - end + xf = xp elseif fieldname == :streamfunction if boundary_conditions.u.x[1] == :periodic xf = x @@ -181,14 +173,15 @@ function field_plot( field = @lift begin isnothing(sleeptime) || sleep(sleeptime) - (; V, p, t) = $state + (; u, p, t) = $state f = if fieldname == :velocity - up, vp, wp = get_velocity(setup, V, t) - map((u, v, w) -> √sum(u^2 + v^2 + w^2), up, vp, wp) + up = interpolate_u_p(setup, u) + map((u, v, w) -> √sum(u^2 + v^2 + w^2), up...) elseif fieldname == :vorticity - get_vorticity(setup, V, t) + ωp = interpolate_ω_p(setup, vorticity(u, setup)) + map((u, v, w) -> √sum(u^2 + v^2 + w^2), ωp...) elseif fieldname == :streamfunction - get_streamfunction(setup, V, t) + get_streamfunction(setup, u, t) elseif fieldname == :pressure reshape(copy(p), length(xp), length(yp), length(zp)) end @@ -204,12 +197,10 @@ function field_plot( ax = Axis3(fig[1, 1]; title = titlecase(string(fieldname)), aspect...) hm = contour!( ax, - xf, - yf, - zf, + xf..., field; - levels, - colorrange = lims, + # levels, + # colorrange = lims, shading = false, alpha, highclip = :red, From b3998ca6e51556b7216967614428eefbbd6468f8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Mon, 18 Sep 2023 17:57:18 +0200 Subject: [PATCH 063/379] Update examples --- examples/Actuator2D.jl | 96 +++++++++-------- examples/Actuator3D.jl | 127 ++++++++++------------ examples/BackwardFacingStep2D.jl | 63 ++++++----- examples/BackwardFacingStep3D.jl | 90 ++++++++-------- examples/DecayingTurbulence2D.jl | 31 +++--- examples/DecayingTurbulence3D.jl | 35 +++--- examples/LidDrivenCavity2D.jl | 70 ++++++------ examples/LidDrivenCavity3D.jl | 95 ++++++++-------- examples/PlaneJets2D.jl | 131 +++++++++++++---------- examples/ShearLayer2D.jl | 63 ++++++----- examples/TaylorGreenVortex2D.jl | 48 +++++---- examples/TaylorGreenVortex3D.jl | 70 ++++++------ src/create_initial_conditions.jl | 4 +- src/grid/grid.jl | 26 ++++- src/solvers/pressure/pressure_solvers.jl | 3 +- 15 files changed, 521 insertions(+), 431 deletions(-) diff --git a/examples/Actuator2D.jl b/examples/Actuator2D.jl index f1afaaf49..27ca949cf 100644 --- a/examples/Actuator2D.jl +++ b/examples/Actuator2D.jl @@ -24,62 +24,68 @@ using IncompressibleNavierStokes # Case name for saving results name = "Actuator2D" -# Viscosity model -Re = 100.0 +# Floating point type +T = Float64 + +# For CPU +device = identity + +# For GPU (note that `cu` converts to `Float32`) +## using CUDA +## device = cu + +# Reynolds number +Re = T(100) # Boundary conditions: Unsteady BC requires time derivatives -u_bc(x, y, t) = x ≈ 0.0 ? cos(π / 6 * sin(π / 6 * t)) : 0.0 -v_bc(x, y, t) = x ≈ 0.0 ? sin(π / 6 * sin(π / 6 * t)) : 0.0 -dudt_bc(x, y, t) = x ≈ 0.0 ? -(π / 6)^2 * cos(π / 6 * t) * sin(π / 6 * sin(π / 6 * t)) : 0.0 -dvdt_bc(x, y, t) = x ≈ 0.0 ? (π / 6)^2 * cos(π / 6 * t) * cos(π / 6 * sin(π / 6 * t)) : 0.0 -bc_type = (; - u = (; x = (:dirichlet, :pressure), y = (:symmetric, :symmetric)), - v = (; x = (:dirichlet, :symmetric), y = (:pressure, :pressure)), +U(x, y, t) = cos(π / 6 * sin(π / 6 * t)) +V(x, y, t) = sin(π / 6 * sin(π / 6 * t)) +dUdt(x, y, t) = -(π / 6)^2 * cos(π / 6 * t) * sin(π / 6 * sin(π / 6 * t)) +dVdt(x, y, t) = (π / 6)^2 * cos(π / 6 * t) * cos(π / 6 * sin(π / 6 * t)) +boundary_conditions = ( + ## x left, x right + (DirichletBC((U, V), (dUdt, dVdt)), PressureBC()), + + ## y rear, y front + (SymmetricBC(), SymmetricBC()), ) # A 2D grid is a Cartesian product of two vectors n = 40 -x = LinRange(0.0, 10.0, 5n) -y = LinRange(-2.0, 2.0, 2n) +x = LinRange(0.0, 10.0, 5n + 1) +y = LinRange(-2.0, 2.0, 2n + 1) plot_grid(x, y) # Actuator body force: A thrust coefficient `Cₜ` distributed over a thin rectangle -xc, yc = 2.0, 0.0 # Disk center -D = 1.0 # Disk diameter -δ = 0.11 # Disk thickness -Cₜ = 5e-4 # Thrust coefficient +xc, yc = T(2), T(0) # Disk center +D = T(1) # Disk diameter +δ = T(0.11) # Disk thickness +Cₜ = T(5e-4) # Thrust coefficient cₜ = Cₜ / (D * δ) inside(x, y) = abs(x - xc) ≤ δ / 2 && abs(y - yc) ≤ D / 2 -bodyforce_u(x, y) = -cₜ * inside(x, y) -bodyforce_v(x, y) = 0.0 +fu(x, y) = -cₜ * inside(x, y) +fv(x, y) = zero(x) # Build setup and assemble operators setup = Setup( - x, - y; + (x, y); Re, - u_bc, - v_bc, - dudt_bc, - dvdt_bc, - bc_type, - bodyforce_u, - bodyforce_v, + boundary_conditions, + bodyforce = (fu, fv), ); # Time interval -t_start, t_end = tlims = (0.0, 12.0) +t_start, t_end = tlims = T(0), T(12) # Initial conditions (extend inflow) -initial_velocity_u(x, y) = 1.0 -initial_velocity_v(x, y) = 0.0 -initial_pressure(x, y) = 0.0 -V₀, p₀ = create_initial_conditions( +initial_velocity = ( + (x, y) -> one(x), + (x, y) -> zero(x), +) +u₀, p₀ = create_initial_conditions( setup, - initial_velocity_u, - initial_velocity_v, + initial_velocity, t_start; - initial_pressure, ); # Iteration processors @@ -94,17 +100,16 @@ processors = ( ); # Solve unsteady problem -V, p, outputs = solve_unsteady( +u, p, outputs = solve_unsteady( setup, - V₀, + u₀, p₀, tlims; method = RK44P2(), - Δt = 0.05, + Δt = T(0.05), processors, inplace = true, ); -#md current_figure() # ## Post-process # @@ -117,7 +122,12 @@ box = ( ) # Export to VTK -save_vtk(setup, V, p, t_end, "output/solution") +save_vtk(setup, u, p, "output/solution") + +# Field plot +fig = outputs[1] +lines!(box...; color = :red) +fig # Plot pressure fig = plot_pressure(setup, p) @@ -125,21 +135,21 @@ lines!(box...; color = :red) fig # Plot velocity -fig = plot_velocity(setup, V, t_end) +fig = plot_velocity(setup, u) lines!(box...; color = :red) fig # Plot vorticity -fig = plot_vorticity(setup, V, t_end) +fig = plot_vorticity(setup, u) lines!(box...; color = :red) fig # Plot streamfunction -fig = plot_streamfunction(setup, V, t_end) +fig = plot_streamfunction(setup, u) lines!(box...; color = :red) fig # Plot force -fig = plot_force(setup, t_end) +fig = plot_force(setup) lines!(box...; color = :red) fig diff --git a/examples/Actuator3D.jl b/examples/Actuator3D.jl index c87af57dd..076f064df 100644 --- a/examples/Actuator3D.jl +++ b/examples/Actuator3D.jl @@ -24,86 +24,75 @@ using IncompressibleNavierStokes # Case name for saving results name = "Actuator3D" -# Viscosity model -Re = 100.0 +# Floating point type +T = Float64 + +# For CPU +device = identity + +# For GPU (note that `cu` converts to `Float32`) +## using CUDA +## device = cu + +# Reynolds number +Re = T(100) # Boundary conditions: Unsteady BC requires time derivatives -u_bc(x, y, z, t) = x ≈ 0.0 ? cos(π / 6 * sin(π / 6 * t)) : 0.0 -v_bc(x, y, z, t) = x ≈ 0.0 ? sin(π / 6 * sin(π / 6 * t)) : 0.0 -w_bc(x, y, z, t) = 0.0 -dudt_bc(x, y, z, t) = - x ≈ 0.0 ? -(π / 6)^2 * cos(π / 6 * t) * sin(π / 6 * sin(π / 6 * t)) : 0.0 -dvdt_bc(x, y, z, t) = - x ≈ 0.0 ? (π / 6)^2 * cos(π / 6 * t) * cos(π / 6 * sin(π / 6 * t)) : 0.0 -dwdt_bc(x, y, z, t) = 0.0 -bc_type = (; - u = (; - x = (:dirichlet, :pressure), - y = (:symmetric, :symmetric), - z = (:symmetric, :symmetric), - ), - v = (; - x = (:dirichlet, :symmetric), - y = (:pressure, :pressure), - z = (:symmetric, :symmetric), - ), - w = (; - x = (:dirichlet, :symmetric), - y = (:symmetric, :symmetric), - z = (:pressure, :pressure), - ), +U(x, y, z, t) = cos(π / 6 * sin(π / 6 * t)) +V(x, y, z, t) = sin(π / 6 * sin(π / 6 * t)) +W(x, y, z, t) = zero(x) +dUdt(x, y, z, t) = -(π / 6)^2 * cos(π / 6 * t) * sin(π / 6 * sin(π / 6 * t)) +dVdt(x, y, z, t) = (π / 6)^2 * cos(π / 6 * t) * cos(π / 6 * sin(π / 6 * t)) +dWdt(x, y, z, t) = zero(x) +boundary_conditions = ( + ## x left, x right + (DirichletBC((U, V, W), (dUdt, dVdt, dWdt)), PressureBC()), + + ## y rear, y front + (SymmetricBC(), SymmetricBC()), + + ## z rear, z front + (SymmetricBC(), SymmetricBC()), ) # A 3D grid is a Cartesian product of three vectors -x = LinRange(0.0, 6.0, 30) -y = LinRange(-2.0, 2.0, 40) -z = LinRange(-2.0, 2.0, 40) +x = LinRange(0.0, 6.0, 31) +y = LinRange(-2.0, 2.0, 41) +z = LinRange(-2.0, 2.0, 41) plot_grid(x, y, z) # Actuator body force: A thrust coefficient `Cₜ` distributed over a short cylinder -cx, cy, cz = 2.0, 0.0, 0.0 # Disk center -D = 1.0 # Disk diameter -δ = 0.11 # Cylinder height -Cₜ = 5e-4 # Thrust coefficient +cx, cy, cz = T(2), T(0), T(0) # Disk center +D = T(1) # Disk diameter +δ = T(0.11) # Disk thickness +Cₜ = T(5e-4) # Thrust coefficient cₜ = Cₜ / (π * (D / 2)^2 * δ) inside(x, y, z) = abs(x - cx) ≤ δ / 2 && (y - cy)^2 + (z - cz)^2 ≤ (D / 2)^2 -bodyforce_u(x, y, z) = -cₜ * inside(x, y, z) -bodyforce_v(x, y, z) = 0.0 -bodyforce_w(x, y, z) = 0.0 +fu(x, y, z) = -cₜ * inside(x, y, z) +fv(x, y, z) = zero(x) +fw(x, y, z) = zero(x) # Build setup and assemble operators setup = Setup( - x, - y, - z; + (x, y, z); Re, - u_bc, - v_bc, - w_bc, - dudt_bc, - dvdt_bc, - dwdt_bc, - bc_type, - bodyforce_u, - bodyforce_v, - bodyforce_w, + boundary_conditions, + bodyforce = (fu, fv, fw), ); # Time interval -t_start, t_end = tlims = (0.0, 3.0) +t_start, t_end = tlims = T(0), T(3) # Initial conditions (extend inflow) -initial_velocity_u(x, y, z) = 1.0 -initial_velocity_v(x, y, z) = 0.0 -initial_velocity_w(x, y, z) = 0.0 -initial_pressure(x, y, z) = 0.0 -V₀, p₀ = create_initial_conditions( +initial_velocity = ( + (x, y, z) -> one(x), + (x, y, z) -> zero(x), + (x, y, z) -> zero(x), +) +u₀, p₀ = create_initial_conditions( setup, - initial_velocity_u, - initial_velocity_v, - initial_velocity_w, + initial_velocity, t_start; - initial_pressure, ); # Iteration processors @@ -118,36 +107,38 @@ processors = ( ); # Solve unsteady problem -V, p, outputs = solve_unsteady( +u, p, outputs = solve_unsteady( setup, - V₀, + u₀, p₀, tlims; method = RK44P2(), - Δt = 0.05, + Δt = T(0.05), processors, inplace = true, ); -#md current_figure() # ## Post-process # # We may visualize or export the computed fields `(V, p)` +# Field plot +outputs[1] + # Export to VTK -save_vtk(setup, V, p, t_end, "output/solution") +save_vtk(setup, u, p, "output/solution") # Plot pressure plot_pressure(setup, p) # Plot velocity -plot_velocity(setup, V, t_end) +plot_velocity(setup, u) # Plot vorticity -plot_vorticity(setup, V, t_end) +plot_vorticity(setup, u) # Plot streamfunction -## plot_streamfunction(setup, V, t_end) +## plot_streamfunction(setup, V) # Plot force -plot_force(setup, t_end) +plot_force(setup) diff --git a/examples/BackwardFacingStep2D.jl b/examples/BackwardFacingStep2D.jl index 135db693a..924762c56 100644 --- a/examples/BackwardFacingStep2D.jl +++ b/examples/BackwardFacingStep2D.jl @@ -25,43 +25,57 @@ using IncompressibleNavierStokes # Case name for saving results name = "BackwardFacingStep2D" -# Viscosity model -Re = 3000.0 +# Floating point type +T = Float64 + +# For CPU +device = identity + +# For GPU (note that `cu` converts to `Float32`) +## using CUDA +## device = cu + +# Reynolds number +Re = T(3000) # Boundary conditions: steady inflow on the top half -u_bc(x, y, t) = x ≈ 0 && y ≥ 0 ? 24y * (1 / 2 - y) : 0.0 -v_bc(x, y, t) = 0.0 -bc_type = (; - u = (; x = (:dirichlet, :pressure), y = (:dirichlet, :dirichlet)), - v = (; x = (:dirichlet, :symmetric), y = (:dirichlet, :dirichlet)), +U(x, y, t) = y ≥ 0 ? 24y * (1 - y) / 2 : zero(x) +V(x, y, t) = zero(x) +dUdt(x, y, t) = zero(x) +dVdt(x, y, t) = zero(x) +boundary_conditions = ( + ## x left, x right + (DirichletBC((U, V), (dUdt, dVdt)), PressureBC()), + + ## y rear, y front + (DirichletBC(), DirichletBC()), ) # A 2D grid is a Cartesian product of two vectors. Here we refine the grid near # the walls. -x = LinRange(0.0, 10.0, 300) -y = cosine_grid(-0.5, 0.5, 50) +x = LinRange(T(0), T(10), 301) +y = cosine_grid(-T(0.5), T(0.5), 51) plot_grid(x, y) # Build setup and assemble operators -setup = Setup(x, y; Re, u_bc, v_bc, bc_type); +setup = Setup((x, y); Re, boundary_conditions); # Time interval -t_start, t_end = tlims = (0.0, 7.0) +t_start, t_end = tlims = T(0), T(7) # Initial conditions (extend inflow) -initial_velocity_u(x, y) = y ≥ 0.0 ? 24y * (1 / 2 - y) : 0.0 -initial_velocity_v(x, y) = 0.0 -initial_pressure(x, y) = 0.0 -V₀, p₀ = create_initial_conditions( +initial_velocity = ( + (x, y) -> U(x, y, zero(x)), + (x, y) -> zero(x), +) +u₀, p₀ = create_initial_conditions( setup, - initial_velocity_u, - initial_velocity_v, + initial_velocity, t_start; - initial_pressure, ); # Solve steady state problem -V, p = solve_steady_state(setup, V₀, p₀); +## u, p = solve_steady_state(setup, u₀, p₀); # Iteration processors processors = ( @@ -75,24 +89,23 @@ processors = ( ); # Solve unsteady problem -V, p, outputs = solve_unsteady(setup, V₀, p₀, tlims; Δt = 0.002, processors, inplace = true) -#md current_figure() +u, p, outputs = solve_unsteady(setup, u₀, p₀, tlims; Δt = T(0.002), processors, inplace = true); # ## Post-process # # We may visualize or export the computed fields `(V, p)` # Export to VTK -save_vtk(setup, V, p, t_end, "output/solution") +save_vtk(setup, u, p, "output/solution") # Plot pressure plot_pressure(setup, p) # Plot velocity -plot_velocity(setup, V, t_end) +plot_velocity(setup, u) # Plot vorticity -plot_vorticity(setup, V, t_end) +plot_vorticity(setup, u) # Plot streamfunction -plot_streamfunction(setup, V, t_end) +plot_streamfunction(setup, u) diff --git a/examples/BackwardFacingStep3D.jl b/examples/BackwardFacingStep3D.jl index 29392e15c..5ef998714 100644 --- a/examples/BackwardFacingStep3D.jl +++ b/examples/BackwardFacingStep3D.jl @@ -25,59 +25,63 @@ using IncompressibleNavierStokes # Case name for saving results name = "BackwardFacingStep3D" -# Viscosity model -Re = 3000.0 +# Floating point type +T = Float64 + +# For CPU +device = identity + +# For GPU (note that `cu` converts to `Float32`) +## using CUDA +## device = cu + +# Reynolds number +Re = T(3000) # Boundary conditions: steady inflow on the top half -u_bc(x, y, z, t) = x ≈ 0 && y ≥ 0 ? 24y * (1 / 2 - y) : 0.0 -v_bc(x, y, z, t) = 0.0 -w_bc(x, y, z, t) = 0.0 -bc_type = (; - u = (; - x = (:dirichlet, :pressure), - y = (:dirichlet, :dirichlet), - z = (:periodic, :periodic), - ), - v = (; - x = (:dirichlet, :symmetric), - y = (:dirichlet, :dirichlet), - z = (:periodic, :periodic), - ), - w = (; - x = (:dirichlet, :symmetric), - y = (:dirichlet, :dirichlet), - z = (:periodic, :periodic), - ), +U(x, y, z, t) = y ≥ 0 ? 24y * (1 - y) / 2 : zero(x) +V(x, y, z, t) = zero(x) +W(x, y, z, t) = zero(x) +dUdt(x, y, z, t) = zero(x) +dVdt(x, y, z, t) = zero(x) +dWdt(x, y, z, t) = zero(x) +boundary_conditions = ( + ## x left, x right + (DirichletBC((U, V, W), (dUdt, dVdt, dWdt)), PressureBC()), + + ## y rear, y front + (DirichletBC(), DirichletBC()), + + ## z bottom, z top + (PeriodicBC(), PeriodicBC()), ) -# A 3D grid is a Cartesian product of three vectors -x = LinRange(0, 10, 160) -y = LinRange(-0.5, 0.5, 16) -z = LinRange(-0.25, 0.25, 8) +# A 2D grid is a Cartesian product of two vectors. +x = LinRange(T(0), T(10), 129) +y = LinRange(-T(0.5), T(0.5), 17) +z = LinRange(-T(0.25), T(0.25), 9) plot_grid(x, y, z) # Build setup and assemble operators -setup = Setup(x, y, z; Re, u_bc, v_bc, w_bc, bc_type); +setup = Setup((x, y, z); Re, boundary_conditions); # Time interval -t_start, t_end = tlims = (0.0, 7.0) +t_start, t_end = tlims = T(0), T(7) # Initial conditions (extend inflow) -initial_velocity_u(x, y, z) = y ≥ 0 ? 24y * (1 / 2 - y) : 0.0 -initial_velocity_v(x, y, z) = 0.0 -initial_velocity_w(x, y, z) = 0.0 -initial_pressure(x, y, z) = 0.0 -V₀, p₀ = create_initial_conditions( +initial_velocity = ( + (x, y, z) -> U(x, y, z, zero(x)), + (x, y, z) -> zero(x), + (x, y, z) -> zero(x), +) +u₀, p₀ = create_initial_conditions( setup, - initial_velocity_u, - initial_velocity_v, - initial_velocity_w, + initial_velocity, t_start; - initial_pressure, ); # Solve steady state problem -V, p = solve_steady_state(setup, V₀, p₀); +## u, p = solve_steady_state(setup, u₀, p₀); # Iteration processors processors = ( @@ -91,24 +95,24 @@ processors = ( ); # Solve unsteady problem -V, p, outputs = solve_unsteady(setup, V₀, p₀, tlims; Δt = 0.01, processors, inplace = true) +u, p, outputs = solve_unsteady(setup, u₀, p₀, tlims; Δt = 0.01, processors, inplace = true) #md current_figure() # ## Post-process # -# We may visualize or export the computed fields `(V, p)` +# We may visualize or export the computed fields `(u, p)` # Export to VTK -save_vtk(setup, V, p, t_end, "output/solution") +save_vtk(setup, u, p, "output/solution") # Plot pressure plot_pressure(setup, p) # Plot velocity -plot_velocity(setup, V, t_end) +plot_velocity(setup, u) # Plot vorticity -plot_vorticity(setup, V, t_end) +plot_vorticity(setup, u) # Plot streamfunction -## plot_streamfunction(setup, V, t_end) +## plot_streamfunction(setup, u) diff --git a/examples/DecayingTurbulence2D.jl b/examples/DecayingTurbulence2D.jl index a08e1ca09..2e0fb1556 100644 --- a/examples/DecayingTurbulence2D.jl +++ b/examples/DecayingTurbulence2D.jl @@ -27,15 +27,15 @@ using IncompressibleNavierStokes name = "DecayingTurbulence2D" # Floating point precision -T = Float32 +T = Float64 # To use CPU: Do not move any arrays device = identity # To use GPU, use `cu` to move arrays to the GPU. # Note: `cu` converts to Float32 -using CUDA -device = cu +## using CUDA +## device = cu # Viscosity model Re = T(10_000) @@ -57,13 +57,13 @@ u₀, p₀ = random_field(setup, T(0); A = T(1_000_000), σ = T(30), s = T(5), p # Iteration processors processors = ( - field_plotter(setup; nupdate = 10), - # energy_history_plotter(device(setup); nupdate = 20, displayfig = false), - # energy_spectrum_plotter(device(setup); nupdate = 20, displayfig = false), - ## animator(device(setup), "vorticity.mp4"; nupdate = 16), + field_plotter(setup; nupdate = 20), + energy_history_plotter(setup; nupdate = 20, displayfig = false), + energy_spectrum_plotter(setup; nupdate = 20, displayfig = false), + ## animator(setup, "vorticity.mp4"; nupdate = 16), ## vtk_writer(setup; nupdate = 10, dir = "output/$name", filename = "solution"), ## field_saver(setup; nupdate = 10), - step_logger(; nupdate = 10), + step_logger(; nupdate = 100), ); # Time interval @@ -81,6 +81,11 @@ u, p, outputs = solve_unsteady( inplace = true, ); + +# ## Post-process +# +# We may visualize or export the computed fields `(u, p)` + # Field plot outputs[1] @@ -90,18 +95,14 @@ outputs[2] # Energy spectrum plot outputs[3] -# ## Post-process -# -# We may visualize or export the computed fields `(V, p)` - # Export to VTK -save_vtk(setup, u, p, t_end, "output/solution") +save_vtk(setup, u, p, "output/solution") # Plot pressure plot_pressure(setup, p₀) # Plot velocity -plot_velocity(setup, u, t_end) +plot_velocity(setup, u) # Plot vorticity -plot_vorticity(setup, u, t_end) +plot_vorticity(setup, u) diff --git a/examples/DecayingTurbulence3D.jl b/examples/DecayingTurbulence3D.jl index 602c6b045..60d5938f3 100644 --- a/examples/DecayingTurbulence3D.jl +++ b/examples/DecayingTurbulence3D.jl @@ -42,47 +42,44 @@ Re = T(10_000) # A 3D grid is a Cartesian product of three vectors n = 32 -lims = (T(0), T(1)) -x = LinRange(lims..., n + 1) -y = LinRange(lims..., n + 1) -z = LinRange(lims..., n + 1) -# plot_grid(x, y, z) +lims = T(0), T(1) +x = ntuple(α -> LinRange(lims..., n + 1), 3) +# plot_grid(x...) # Build setup and assemble operators -setup = Setup(x, y, z; Re); +setup = device(Setup(x; Re)); # Since the grid is uniform and identical for x, y, and z, we may use a # specialized spectral pressure solver pressure_solver = SpectralPressureSolver(setup); # Initial conditions -V₀, p₀ = random_field(setup; A = T(1_000_000), σ = T(30), s = 5, pressure_solver) +u₀, p₀ = random_field(setup; A = T(1_000_000), σ = T(30), s = 5, pressure_solver) # Time interval -t_start, t_end = tlims = (T(0), T(1.0)) +t_start, t_end = tlims = T(0), T(1.0) # Iteration processors processors = ( - field_plotter(device(setup); nupdate = 10), - energy_history_plotter(device(setup); nupdate = 10), - energy_spectrum_plotter(device(setup); nupdate = 10), - ## animator(device(setup), "vorticity.mp4"; nupdate = 4), + field_plotter(setup; nupdate = 10), + energy_history_plotter(setup; nupdate = 10), + energy_spectrum_plotter(setup; nupdate = 10), + ## animator(setup, "vorticity.mp4"; nupdate = 4), ## vtk_writer(setup; nupdate = 10, dir = "output/$name", filename = "solution"), ## field_saver(setup; nupdate = 10), step_logger(; nupdate = 1), ); # Solve unsteady problem -V, p, outputs = solve_unsteady( +u, p, outputs = solve_unsteady( setup, - V₀, + u₀, p₀, tlims; Δt = T(0.001), processors, pressure_solver, inplace = true, - device, ); # Field plot @@ -96,16 +93,16 @@ outputs[3] # ## Post-process # -# We may visualize or export the computed fields `(V, p)` +# We may visualize or export the computed fields `(u, p)` # Export to VTK -save_vtk(setup, V, p, t_end, "output/solution") +save_vtk(setup, u, p, "output/solution") # Plot pressure plot_pressure(setup, p) # Plot velocity -plot_velocity(setup, V, t_end) +plot_velocity(setup, u) # Plot vorticity -plot_vorticity(setup, V, t_end) +plot_vorticity(setup, u) diff --git a/examples/LidDrivenCavity2D.jl b/examples/LidDrivenCavity2D.jl index f29771f18..4d46c12af 100644 --- a/examples/LidDrivenCavity2D.jl +++ b/examples/LidDrivenCavity2D.jl @@ -56,28 +56,35 @@ device = identity # Here we choose a moderate Reynolds number. Note how we pass the floating point type. Re = T(1_000) -# Dirichlet boundary conditions are specified as plain Julia functions. They -# are marked by the `:dirichlet` symbol. Other possible BC types are -# `:periodic`, `:symmetric`, and `:pressure`. -u_bc(x, y, t) = y ≈ 1 ? T(1) : T(0) -v_bc(x, y, t) = T(0) -bc_type = (; - u = (; x = (:dirichlet, :dirichlet), y = (:dirichlet, :dirichlet)), - v = (; x = (:dirichlet, :dirichlet), y = (:dirichlet, :dirichlet)), +# Non-zero Dirichlet boundary conditions are specified as plain Julia functions. +# Other possible BC types are `PeriodicBC()`, `SymmetricBC`, and `PressureBC`. +lidvel = ( + (x, y, t) -> one(x), + (x, y, t) -> zero(x), +) +dlidveldt = ( + (x, y, t) -> zero(x), + (x, y, t) -> zero(x), +) +boundary_conditions = ( + ## x left, x right + (DirichletBC(), DirichletBC()), + + ## y bottom, y top + (DirichletBC(), DirichletBC(lidvel, dlidveldt)), ) # We create a two-dimensional domain with a box of size `[1, 1]`. The grid is # created as a Cartesian product between two vectors. We add a refinement near # the walls. n = 40 -lims = (T(0), T(1)) -x = cosine_grid(lims..., n) -y = cosine_grid(lims..., n) -plot_grid(x, y) +lims = T(0), T(1) +x = cosine_grid(lims..., n), cosine_grid(lims..., n) +plot_grid(x...) # We can now build the setup and assemble operators. # A 3D setup is built if we also provide a vector of z-coordinates. -setup = Setup(x, y; Re, u_bc, v_bc, bc_type); +setup = device(Setup(x; boundary_conditions, Re)); # The pressure solver is used to solve the pressure Poisson equation. # Available solvers are @@ -90,18 +97,17 @@ setup = Setup(x, y; Re, u_bc, v_bc, bc_type); pressure_solver = DirectPressureSolver(setup) # We will solve for a time interval of ten seconds. -t_start, t_end = tlims = (T(0), T(10)) +t_start, t_end = tlims = T(0), T(10) # The initial conditions are defined as plain Julia functions. -initial_velocity_u(x, y) = zero(x) -initial_velocity_v(x, y) = zero(x) -initial_pressure(x, y) = zero(x) -V₀, p₀ = create_initial_conditions( +initial_velocity = ( + (x, y) -> zero(x), + (x, y) -> zero(x), +) +u₀, p₀ = create_initial_conditions( setup, - initial_velocity_u, - initial_velocity_v, + initial_velocity, t_start; - initial_pressure, pressure_solver, ) @@ -111,7 +117,7 @@ V₀, p₀ = create_initial_conditions( # The [`solve_steady_state`](@ref) function is for computing a state where the right hand side of the # momentum equation is zero. -V, p = solve_steady_state(setup, V₀, p₀) +## u, p = solve_steady_state(setup, u₀, p₀) # For this test case, the same steady state may be obtained by solving an # unsteady problem for a sufficiently long time. @@ -121,10 +127,10 @@ V, p = solve_steady_state(setup, V₀, p₀) # later returned by `solve_unsteady`. processors = ( - field_plotter(device(setup); nupdate = 50), - ## energy_history_plotter(device(setup); nupdate = 1), - ## energy_spectrum_plotter(device(setup); nupdate = 100), - ## animator(device(setup), "vorticity.mkv"; nupdate = 4), + field_plotter(setup; nupdate = 50), + ## energy_history_plotter(setup; nupdate = 1), + ## energy_spectrum_plotter(setup; nupdate = 100), + ## animator(setup, "vorticity.mkv"; nupdate = 4), vtk_writer(setup; nupdate = 100, dir = "output/$name", filename = "solution"), ## field_saver(setup; nupdate = 10), step_logger(; nupdate = 1000), @@ -132,8 +138,8 @@ processors = ( # By default, a standard fourth order Runge-Kutta method is used. If we don't # provide the time step explicitly, an adaptive time step is used. -V, p, outputs = - solve_unsteady(setup, V₀, p₀, tlims; Δt = T(0.001), processors, pressure_solver, device); +u, p, outputs = + solve_unsteady(setup, u₀, p₀, tlims; Δt = T(0.001), processors, pressure_solver, device); # ## Post-process # @@ -142,22 +148,22 @@ V, p, outputs = # Export fields to VTK. The file `output/solution.vti` may be opened for # visulization in [ParaView](https://www.paraview.org/). This is particularly # useful for inspecting results from 3D simulations. -save_vtk(setup, V, p, t_end, "output/solution") +save_vtk(setup, u, p, "output/solution") # Plot pressure plot_pressure(setup, p) # Plot velocity. Note the time stamp used for computing boundary conditions, if # any. -plot_velocity(setup, V, t_end) +plot_velocity(setup, u) # Plot vorticity (with custom levels) levels = [-7, -5, -4, -3, -2, -1, -0.5, 0, 0.5, 1, 2, 3, 7] -plot_vorticity(setup, V, t_end; levels) +plot_vorticity(setup, u; levels) # Plot streamfunction. Note the time stamp used for computing boundary # conditions, if any -plot_streamfunction(setup, V, t_end) +plot_streamfunction(setup, u) # In addition, the tuple `outputs` contains quantities from our processors. # diff --git a/examples/LidDrivenCavity3D.jl b/examples/LidDrivenCavity3D.jl index 7a465facd..f1eff9c81 100644 --- a/examples/LidDrivenCavity3D.jl +++ b/examples/LidDrivenCavity3D.jl @@ -24,60 +24,69 @@ using IncompressibleNavierStokes # Case name for saving results name = "LidDrivenCavity3D" -# Viscosity model -Re = 1000.0 +# Floating point type +T = Float64 + +# For CPU +device = identity + +# For GPU (note that `cu` converts to `Float32`) +## using CUDA +## device = cu + +# Reynolds number +Re = T(1_000) # Boundary conditions: horizontal movement of the top lid -u_bc(x, y, z, t) = y ≈ 1.0 ? 1.0 : 0.0 -v_bc(x, y, z, t) = 0.0 -w_bc(x, y, z, t) = y ≈ 1.0 ? 0.2 : 0.0 -bc_type = (; - u = (; - x = (:dirichlet, :dirichlet), - y = (:dirichlet, :dirichlet), - z = (:periodic, :periodic), - ), - v = (; - x = (:dirichlet, :dirichlet), - y = (:dirichlet, :dirichlet), - z = (:periodic, :periodic), - ), - w = (; - x = (:dirichlet, :dirichlet), - y = (:dirichlet, :dirichlet), - z = (:periodic, :periodic), - ), +lidvel = ( + (x, y, z, t) -> one(x), + (x, y, z, t) -> zero(x), + (x, y, z, t) -> one(x) / 5, +) +dlidveldt = ( + (x, y, z, t) -> zero(x), + (x, y, z, t) -> zero(x), + (x, y, z, t) -> zero(x), +) +boundary_conditions = ( + ## x left, x right + (DirichletBC(), DirichletBC()), + + ## y rear, y front + (DirichletBC(), DirichletBC(lidvel, dlidveldt)), + + ## z bottom, z top + (PeriodicBC(), PeriodicBC()), ) # A 3D grid is a Cartesian product of three vectors. Here we refine the grid # near the walls. -x = cosine_grid(0.0, 1.0, 25) -y = cosine_grid(0.0, 1.0, 25) -z = LinRange(-0.2, 0.2, 10) +x = cosine_grid(T(0), T(1), 25) +y = cosine_grid(T(0), T(1), 25) +z = LinRange(-T(0.2), T(0.2), 11) plot_grid(x, y, z) # Build setup and assemble operators -setup = Setup(x, y, z; Re, u_bc, v_bc, w_bc, bc_type); +setup = Setup((x, y, z); Re, boundary_conditions); # Time interval -t_start, t_end = tlims = (0.0, 0.2) +t_start, t_end = tlims = T(0), T(0.2) # Initial conditions -initial_velocity_u(x, y, z) = 0.0 -initial_velocity_v(x, y, z) = 0.0 -initial_velocity_w(x, y, z) = 0.0 -initial_pressure(x, y, z) = 0.0 -V₀, p₀ = create_initial_conditions( +initial_velocity = ( + (x, y, z) -> zero(x), + (x, y, z) -> zero(x), + (x, y, z) -> zero(x), +) +u₀, p₀ = create_initial_conditions( setup, - initial_velocity_u, - initial_velocity_v, - initial_velocity_w, + initial_velocity, t_start; - initial_pressure, -); + pressure_solver, +) # Solve steady state problem -V, p = solve_steady_state(setup, V₀, p₀; npicard = 5, maxiter = 15); +## u, p = solve_steady_state(setup, u₀, p₀; npicard = 5, maxiter = 15); # Iteration processors processors = ( @@ -91,24 +100,24 @@ processors = ( ); # Solve unsteady problem -V, p, outputs = solve_unsteady(setup, V₀, p₀, tlims; Δt = 0.001, processors) -#md current_figure() +u, p, outputs = + solve_unsteady(setup, u₀, p₀, tlims; Δt = T(0.001), processors, device); # ## Post-process # # We may visualize or export the computed fields `(V, p)` # Export to VTK -save_vtk(setup, V, p, t_end, "output/solution") +save_vtk(setup, u, p, "output/solution") # Plot pressure plot_pressure(setup, p) # Plot velocity -plot_velocity(setup, V, t_end) +plot_velocity(setup, u) # Plot vorticity -plot_vorticity(setup, V, t_end) +plot_vorticity(setup, u) # Plot streamfunction -## plot_streamfunction(setup, V, t_end) +## plot_streamfunction(setup, u) diff --git a/examples/PlaneJets2D.jl b/examples/PlaneJets2D.jl index cde44faf1..8e20af670 100644 --- a/examples/PlaneJets2D.jl +++ b/examples/PlaneJets2D.jl @@ -25,75 +25,91 @@ using LaTeXStrings # Case name for saving results name = "PlaneJets2D" -# Viscosity model -Re = 6000.0 +# Floating point type +T = Float64 -# Test cases (A, B, C, D; in order) -# U() = sqrt(467.4) -U() = 21.619435700313733 - -u_A(y) = U() / 2 * (tanh((y + 1 / 2) / 0.1) - tanh((y - 1 / 2) / 0.1)) - -u_B(y) = - U() / 2 * (tanh((y + 1 + 1 / 2) / 0.1) - tanh((y + 1 - 1 / 2) / 0.1)) + - U() / 2 * (tanh((y - 1 + 1 / 2) / 0.1) - tanh((y - 1 - 1 / 2) / 0.1)) +# For CPU +device = identity -u_C(y) = - U() / 2 * (tanh(((y + 1.0) / 1 + 1 / 2) / 0.1) - tanh(((y + 1.0) / 1 - 1 / 2) / 0.1)) + - U() / 4 * (tanh(((y - 1.5) / 2 + 1 / 2) / 0.2) - tanh(((y - 1.5) / 2 - 1 / 2) / 0.2)) +# For GPU (note that `cu` converts to `Float32`) +## using CUDA +## device = cu -u_D(y) = - U() / 2 * (tanh(((y + 1.0) / 1 + 1 / 2) / 0.1) - tanh(((y + 1.0) / 1 - 1 / 2) / 0.1)) - - U() / 4 * (tanh(((y - 1.5) / 2 + 1 / 2) / 0.2) - tanh(((y - 1.5) / 2 - 1 / 2) / 0.2)) +# Reynolds number +Re = T(6_000) -# u(y) = u_A(y) -# u(y) = u_B(y) -u(y) = u_C(y) -# u(y) = u_D(y) +# Test cases (A, B, C, D; in order) +## V() = sqrt(T(467.4)) +V() = T(21.619435700313733) + +U_A(y) = u() / 2 * (tanh((y + T(0.5)) / T(0.1)) - tanh((y - T(0.5)) / T(0.1))) + +U_B(y) = + u() / 2 * (tanh((y + 1 + T(0.5)) / T(0.1)) - tanh((y + 1 - T(0.5)) / T(0.1))) + + u() / 2 * (tanh((y - 1 + T(0.5)) / T(0.1)) - tanh((y - 1 - T(0.5)) / T(0.1))) + +U_C(y) = + u() / 2 * ( + tanh(((y + T(1.0)) / 1 + T(0.5)) / T(0.1)) - + tanh(((y + T(1.0)) / 1 - T(0.5)) / T(0.1)) + ) + + u() / 4 * ( + tanh(((y - T(1.5)) / 2 + T(0.5)) / T(0.2)) - + tanh(((y - T(1.5)) / 2 - T(0.5)) / T(0.2)) + ) + +U_D(y) = + u() / 2 * ( + tanh(((y + T(1.0)) / 1 + T(0.5)) / T(0.1)) - + tanh(((y + T(1.0)) / 1 - T(0.5)) / T(0.1)) + ) - + u() / 4 * ( + tanh(((y - T(1.5)) / 2 + T(0.5)) / T(0.2)) - + tanh(((y - T(1.5)) / 2 - T(0.5)) / T(0.2)) + ) + +## U(y) = U_A(y) +## U(y) = U_B(y) +U(y) = U_C(y) +## U(y) = U_D(y) # Random noise to stimulate turbulence -u(x, y) = (1 + 0.1 * (rand() - 1 / 2)) * u(y) +U(x, y) = (1 + T(0.1) * (rand(T) - T(0.5))) * U(y) -# # Boundary conditions: Unsteady BC requires time derivatives -# u_bc(x, y, t) = x ≈ 0.0 ? u(x, y) : 0.0 -# v_bc(x, y, t) = 0.0 -# bc_type = (; -# u = (; x = (:periodic, :periodic), y = (:symmetric, :symmetric)), -# v = (; x = (:periodic, :periodic), y = (:pressure, :pressure)), -# ) +## boundary_conditions = ( +## (PeriodicBC(), PeriodicBC()), +## (PressureBC(), PressureBC()) +## ) # A 2D grid is a Cartesian product of two vectors n = 64 ## n = 128 ## n = 256 -x = LinRange(0.0, 16.0, 4n) -y = LinRange(-10.0, 10.0, 5n) -plot_grid(x, y) +x = LinRange(T(0), T(16), 4n), LinRange(-T(10), T(10), 5n) +plot_grid(x...) # Build setup and assemble operators -setup = Setup(x, y; Re); -# setup = Setup(x, y; Re, u_bc, v_bc, bc_type); +setup = device(Setup(x; Re)); +## setup = device(Setup(x, y; Re, boundary_conditions)); # Since the grid is uniform and identical for x and y, we may use a specialized # spectral pressure solver pressure_solver = SpectralPressureSolver(setup) # Time interval -t_start, t_end = tlims = (0.0, 1.0) +t_start, t_end = tlims = T(0), T(1) # Initial conditions -initial_velocity_u(x, y) = u(x, y) -initial_velocity_v(x, y) = 0.0 -initial_pressure(x, y) = 0.0 -V₀, p₀ = create_initial_conditions( +initial_velocity = ( + (x, y) -> U(x, y), + (x, y) -> zero(T), +) +u₀, p₀ = create_initial_conditions( setup, - initial_velocity_u, - initial_velocity_v, + initial_velocity, t_start; - initial_pressure, pressure_solver, ); -V, p = V₀, p₀ # Real time plot: Streamwise average and spectrum mean_plotter(setup; nupdate = 1) = processor( @@ -101,10 +117,10 @@ mean_plotter(setup; nupdate = 1) = processor( (; indu, yu, yin, Nux_in, Nuy_in) = setup.grid umean = @lift begin - (; V, p, t) = $state - u = V[indu] + (; u, p, t) = $state + u1 = u[1] sleep(0.001) - reshape(sum(reshape(u, size(yu)); dims = 1), :) ./ (Nux_in * U()) + reshape(sum(reshape(u1, size(yu)); dims = 1), :) ./ (Nux_in * V()) end K = Nux_in ÷ 2 @@ -166,14 +182,14 @@ processors = ( ## animator(setup, "vorticity.mkv"; nupdate = 4), ## vtk_writer(setup; nupdate = 10, dir = "output/$name", filename = "solution"), ## field_saver(setup; nupdate = 10), - step_logger(; nupdate = 1), mean_plotter(setup), + step_logger(; nupdate = 1), ); # Solve unsteady problem -V, p, outputs = solve_unsteady( +toto, p, outputs = solve_unsteady( setup, - V₀, + u₀, p₀, tlims; method = RK44P2(), @@ -182,24 +198,29 @@ V, p, outputs = solve_unsteady( pressure_solver, inplace = true, ); -#md current_figure() # ## Post-process # # We may visualize or export the computed fields `(V, p)` +outputs[1] + +#- + +outputs[2] + # Export to VTK -save_vtk(setup, V, p, t_end, "output/solution") +save_vtk(setup, toto, p, "output/solution") # Plot pressure plot_pressure(setup, p) # Plot velocity -plot_velocity(setup, V₀, t_end) -plot_velocity(setup, V, t_end) +plot_velocity(setup, u₀) +plot_velocity(setup, toto) # Plot vorticity -plot_vorticity(setup, V, t_end) +plot_vorticity(setup, toto) # Plot stream function -plot_streamfunction(setup, V, t_end) +plot_streamfunction(setup, toto) diff --git a/examples/ShearLayer2D.jl b/examples/ShearLayer2D.jl index 782c4ba18..8f30305b6 100644 --- a/examples/ShearLayer2D.jl +++ b/examples/ShearLayer2D.jl @@ -22,35 +22,47 @@ using IncompressibleNavierStokes # Case name for saving results name = "ShearLayer2D" -# Viscosity model -Re = Inf +# Floating point type +T = Float64 + +# For CPU +device = identity + +# For GPU (note that `cu` converts to `Float32`) +## using CUDA +## device = cu + +# Reynolds number +Re = T(Inf) # A 2D grid is a Cartesian product of two vectors n = 100 -x = LinRange(0, 2π, n + 1) -y = LinRange(0, 2π, n + 1) +lims = T(0), T(2π) +x = LinRange(lims..., n + 1), LinRange(lims..., n + 1) plot_grid(x, y) # Build setup and assemble operators -setup = Setup(x, y; Re); +setup = device(Setup(x; Re)); # Time interval -t_start, t_end = tlims = (0.0, 8.0) +t_start, t_end = tlims = T(0), T(8) # Initial conditions: We add 1 to u in order to make global momentum # conservation less trivial -d = π / 15 -e = 0.05 -initial_velocity_u(x, y) = y ≤ π ? tanh((y - π / 2) / d) : tanh((3π / 2 - y) / d) -## initial_velocity_u(x, y) = 1.0 + (y ≤ π ? tanh((y - π / 2) / d) : tanh((3π / 2 - y) / d)) -initial_velocity_v(x, y) = e * sin(x) -initial_pressure(x, y) = 0.0 -V₀, p₀ = create_initial_conditions( +d = T(π / 15) +e = T(0.05) +initial_velocity = ( + (x, y) -> y ≤ π ? tanh((y - T(π / 2)) / d) : tanh((T(3π / 2) - y) / d), + (x, y) -> e * sin(x), +) +## initial_velocity = ( +## (x, y) -> T(1) + (y ≤ π ? tanh((y - T(π / 2)) / d) : tanh((T(3π / 2) - y) / d)), +## (x, y) -> e * sin(x), +## ) +u₀, p₀ = create_initial_conditions( setup, - initial_velocity_u, - initial_velocity_v, + initial_velocity, t_start; - initial_pressure, ); # Iteration processors @@ -65,33 +77,34 @@ processors = ( ); # Solve unsteady problem -V, p, outputs = solve_unsteady( +u, p, outputs = solve_unsteady( setup, - V₀, + u₀, p₀, tlims; method = RK44(), - Δt = 0.01, + Δt = T(0.01), processors, inplace = true, ); -#md current_figure() # ## Post-process # -# We may visualize or export the computed fields `(V, p)` +# We may visualize or export the computed fields `(u, p)` + +outputs[1] # Export to VTK -save_vtk(setup, V, p, t_end, "output/solution") +save_vtk(setup, u, p, "output/solution") # Plot pressure plot_pressure(setup, p) # Plot velocity -plot_velocity(setup, V, t_end) +plot_velocity(setup, u) # Plot vorticity -plot_vorticity(setup, V, t_end) +plot_vorticity(setup, u) # Plot streamfunction -## plot_streamfunction(setup, V, t_end) +## plot_streamfunction(setup, u) diff --git a/examples/TaylorGreenVortex2D.jl b/examples/TaylorGreenVortex2D.jl index 6f7601eb3..dcec7f5ce 100644 --- a/examples/TaylorGreenVortex2D.jl +++ b/examples/TaylorGreenVortex2D.jl @@ -25,41 +25,46 @@ name = "TaylorGreenVortex2D" # Floating point type T = Float32 +# For CPU +device = identity + +# For GPU (note that `cu` converts to `Float32`) +## using CUDA +## device = cu + # Reynolds number Re = T(2_000) # A 2D grid is a Cartesian product of two vectors n = 128 -lims = (T(0), T(2π)) -x = LinRange(lims..., n + 1) -y = LinRange(lims..., n + 1) -plot_grid(x, y) +lims = T(0), T(2π) +x = LinRange(lims..., n + 1), LinRange(lims..., n + 1) +plot_grid(x...) # Build setup and assemble operators -setup = Setup(x, y; Re); +setup = device(Setup(x; Re)); # Since the grid is uniform and identical for x and y, we may use a specialized # spectral pressure solver pressure_solver = SpectralPressureSolver(setup) # Time interval -t_start, t_end = tlims = (T(0), T(1)) +t_start, t_end = tlims = T(0), T(1) # Initial conditions -initial_velocity_u(x, y) = -sin(x)cos(y) -initial_velocity_v(x, y) = cos(x)sin(y) -initial_pressure(x, y) = 1 // 4 * (cos(2x) + cos(2y)) -V₀, p₀ = create_initial_conditions( +initial_velocity = ( + (x, y) -> -sin(x) * cos(y), + (x, y) -> cos(x) * sin(y), +) +u₀, p₀ = create_initial_conditions( setup, - initial_velocity_u, - initial_velocity_v, + initial_velocity, t_start; - initial_pressure, pressure_solver, ); # Solve steady state problem -V, p = solve_steady_state(setup, V₀, p₀; npicard = 2); +## u, p = solve_steady_state(setup, u₀, p₀; npicard = 2); # Iteration processors processors = ( @@ -73,9 +78,9 @@ processors = ( ); # Solve unsteady problem -V, p, outputs = solve_unsteady( +u, p, outputs = solve_unsteady( setup, - V₀, + u₀, p₀, tlims; Δt = T(0.01), @@ -83,26 +88,25 @@ V, p, outputs = solve_unsteady( pressure_solver, inplace = true, ); -#md current_figure() # ## Post-process # -# We may visualize or export the computed fields `(V, p)` +# We may visualize or export the computed fields `(u, p)` # Export to VTK -save_vtk(setup, V, p, t_end, "output/solution") +save_vtk(setup, u, p, "output/solution") # Plot pressure plot_pressure(setup, p) # Plot velocity -plot_velocity(setup, V, t_end) +plot_velocity(setup, u) # Plot vorticity -plot_vorticity(setup, V, t_end) +plot_vorticity(setup, u) # Plot streamfunction -## plot_streamfunction(setup, V, t_end) +## plot_streamfunction(setup, u) # Energy history outputs[1] diff --git a/examples/TaylorGreenVortex3D.jl b/examples/TaylorGreenVortex3D.jl index 5c82a8d0e..f3aab4eda 100644 --- a/examples/TaylorGreenVortex3D.jl +++ b/examples/TaylorGreenVortex3D.jl @@ -23,7 +23,7 @@ using IncompressibleNavierStokes name = "TaylorGreenVortex3D" # Floating point precision -T = Float32 +T = Float64 # For CPU device = identity @@ -33,84 +33,84 @@ device = identity ## device = cu # Viscosity model -Re = T(2_000) +Re = T(6_000) # A 3D grid is a Cartesian product of three vectors n = 32 -lims = (T(0), T(2π)) -x = LinRange(lims..., n + 1) -y = LinRange(lims..., n + 1) -z = LinRange(lims..., n + 1) -plot_grid(x, y, z) +lims = T(0), T(2π) +x = ntuple(α -> LinRange(lims..., n + 1), 3) +plot_grid(x...) # Build setup and assemble operators -setup = Setup(x, y, z; Re); +setup = device(Setup(x; Re)); # Since the grid is uniform and identical for x, y, and z, we may use a # specialized spectral pressure solver -pressure_solver = SpectralPressureSolver(setup) +pressure_solver = SpectralPressureSolver(setup); # Initial conditions -initial_velocity_u(x, y, z) = sin(x)cos(y)cos(z) -initial_velocity_v(x, y, z) = -cos(x)sin(y)cos(z) -initial_velocity_w(x, y, z) = zero(x) -initial_pressure(x, y, z) = 1 // 4 * (cos(2x) + cos(2y) + cos(2z)) -V₀, p₀ = create_initial_conditions( +initial_velocity = ( + (x, y, z) -> sin(x)cos(y)cos(z), + (x, y, z) -> -cos(x)sin(y)cos(z), + (x, y, z) -> zero(x), +) +u₀, p₀ = create_initial_conditions( setup, - initial_velocity_u, - initial_velocity_v, - initial_velocity_w, + initial_velocity, T(0); - initial_pressure, pressure_solver, ); +GC.gc() +CUDA.reclaim() + # Solve steady state problem -V, p = solve_steady_state(setup, V₀, p₀; npicard = 6) +## u, p = solve_steady_state(setup, u₀, p₀; npicard = 6) # Iteration processors processors = ( - field_plotter(device(setup); nupdate = 5), - ## energy_history_plotter(device(setup); nupdate = 1), - ## energy_spectrum_plotter(device(setup); nupdate = 100), - ## animator(device(setup), "vorticity.mp4"; nupdate = 4), + field_plotter(setup; nupdate = 1), + ## energy_history_plotter(setup; nupdate = 1), + ## energy_spectrum_plotter(setup; nupdate = 100), + ## animator(setup, "vorticity.mp4"; nupdate = 4), ## vtk_writer(setup; nupdate = 10, dir = "output/$name", filename = "solution"), ## field_saver(setup; nupdate = 10), - step_logger(; nupdate = 10), + step_logger(; nupdate = 1), ); # Time interval -t_start, t_end = tlims = (T(0), T(5)) +t_start, t_end = tlims = T(0), T(5) # Solve unsteady problem -V, p, outputs = solve_unsteady( +u, p, outputs = solve_unsteady( setup, - V₀, - p₀, + # u₀, p₀, + u, p, tlims; Δt = T(0.01), processors, pressure_solver, inplace = true, - device, ); -#md current_figure() # ## Post-process # -# We may visualize or export the computed fields `(V, p)` +# We may visualize or export the computed fields `(u, p)` # Export to VTK -save_vtk(setup, setup, V, p, "output/solution") +save_vtk(setup, u, p, "output/solution") # Plot pressure plot_pressure(setup, p; levels = 3, alpha = 0.05) # Plot velocity -plot_velocity(setup, V, t_end; levels = 3, alpha = 0.05) +plot_velocity(setup, u; levels = 3, alpha = 0.05) # Plot vorticity -plot_vorticity(setup, V, t_end; levels = 5, alpha = 0.05) +plot_vorticity(setup, u; levels = 5, alpha = 0.05) # Plot streamfunction -## plot_streamfunction(setup, V, t_end) +## plot_streamfunction(setup, u) + +# Field plot +outputs[1] diff --git a/src/create_initial_conditions.jl b/src/create_initial_conditions.jl index c66b0a107..d67441cf0 100644 --- a/src/create_initial_conditions.jl +++ b/src/create_initial_conditions.jl @@ -15,7 +15,7 @@ function create_initial_conditions( pressure_solver = DirectPressureSolver(setup), ) (; grid) = setup - (; dimension, N, Iu, Ip, x, xp, Ωu) = grid + (; dimension, N, Iu, Ip, x, xp) = grid T = eltype(x[1]) D = dimension() @@ -103,7 +103,7 @@ function random_field( s = convert(eltype(setup.grid.x), 5), pressure_solver = DirectPressureSolver(setup), ) - (; dimension, x, N, Ip, Ωu) = setup.grid + (; dimension, x, N, Ip) = setup.grid D = dimension() T = eltype(x[1]) diff --git a/src/grid/grid.jl b/src/grid/grid.jl index 5a7a1b624..017bc50b7 100644 --- a/src/grid/grid.jl +++ b/src/grid/grid.jl @@ -86,10 +86,30 @@ function Grid(x, boundary_conditions) # Velocity volume mid-sections Γu = ntuple(α -> ntuple(β -> KernelAbstractions.ones(get_backend(x[1]), T, N), D), D) - for α = 1:D, β = 1:D, γ = ((1:β-1)..., (β+1:D)...) - Γu[α][β] .*= reshape(γ == β ? 1 : γ == α ? Δu[γ] : Δ[γ], ntuple(Returns(1), γ - 1)..., :) + for α = 1:D, β = 1:D, γ in ((1:β-1)..., (β+1:D)...) + Γu[α][β] .*= + reshape(γ == β ? 1 : γ == α ? Δu[γ] : Δ[γ], ntuple(Returns(1), γ - 1)..., :) end + # # Velocity points + # Xu = ntuple(α -> KernelAbstractions.ones(get_backend(x[1]), T, N) + # Grid quantities - (; dimension, N, Nu, Np, Iu, Ip, xlims, x, xp, Δ, Δu, Ω, Ωu, Ωω, Γu) + (; + dimension, + N, + Nu, + Np, + Iu, + Ip, + xlims, + x, + xp, + Δ, + Δu, + Ω, + # Ωu, + # Ωω, + # Γu, + ) end diff --git a/src/solvers/pressure/pressure_solvers.jl b/src/solvers/pressure/pressure_solvers.jl index 0c8b00216..d847a10b4 100644 --- a/src/solvers/pressure/pressure_solvers.jl +++ b/src/solvers/pressure/pressure_solvers.jl @@ -109,7 +109,8 @@ function SpectralPressureSolver(setup) # Scale with Δx*Δy*Δz, since we solve the PDE in integrated form Ahat .*= 4 * prod(Δx) - # Pressure is determined up to constant, fix at 0 + # Pressure is determined up to constant. By setting the constant + # scaling factor to 1, we preserve the average. Ahat[1:1] .= 1 # Placeholders for intermediate results From b72809c5289b4dad9e97e3a817f86e16b8ce07c9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Tue, 19 Sep 2023 13:14:27 +0200 Subject: [PATCH 064/379] fix: Correct name --- src/time_steppers/step_explicit_runge_kutta.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/time_steppers/step_explicit_runge_kutta.jl b/src/time_steppers/step_explicit_runge_kutta.jl index e578a7cc4..6b1686efb 100644 --- a/src/time_steppers/step_explicit_runge_kutta.jl +++ b/src/time_steppers/step_explicit_runge_kutta.jl @@ -66,7 +66,7 @@ function timestep!(method::ExplicitRungeKuttaMethod, stepper, Δt; cache) for α = 1:D @. u[α] = v[α] - c[i] * Δt * G[α] end - apply_bc_u!(u, tᵢ, setup) + apply_bc_u!(u, t, setup) end # Complete time step From 5208cc1ddf97cb3231147aa512a0d0ed1a4b995e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Tue, 19 Sep 2023 13:15:07 +0200 Subject: [PATCH 065/379] Initialize with zeros --- src/solvers/pressure/pressure_solvers.jl | 4 +- src/time_steppers/time_stepper_caches.jl | 60 ++++++++++++------------ 2 files changed, 32 insertions(+), 32 deletions(-) diff --git a/src/solvers/pressure/pressure_solvers.jl b/src/solvers/pressure/pressure_solvers.jl index d847a10b4..952191c16 100644 --- a/src/solvers/pressure/pressure_solvers.jl +++ b/src/solvers/pressure/pressure_solvers.jl @@ -114,8 +114,8 @@ function SpectralPressureSolver(setup) Ahat[1:1] .= 1 # Placeholders for intermediate results - phat = similar(Ahat) - fhat = similar(Ahat) + phat = zero(Ahat) + fhat = zero(Ahat) SpectralPressureSolver{T,typeof(Ahat)}(Ahat, phat, fhat) end diff --git a/src/time_steppers/time_stepper_caches.jl b/src/time_steppers/time_stepper_caches.jl index 3b2ac1e5b..b469f2a4a 100644 --- a/src/time_steppers/time_stepper_caches.jl +++ b/src/time_steppers/time_stepper_caches.jl @@ -7,41 +7,41 @@ Get time stepper cache for the given ODE method. function ode_method_cache end function ode_method_cache(::AdamsBashforthCrankNicolsonMethod, setup, V, p) - c₀ = similar(V) - c₋₁ = similar(V) - F = similar(V) - f = similar(p) - Δp = similar(p) - Rr = similar(V) - b = similar(V) - b₀ = similar(V) - b₁ = similar(V) - yDiff₀ = similar(V) - yDiff₁ = similar(V) - Gp₀ = similar(V) + c₀ = zero(V) + c₋₁ = zero(V) + F = zero(V) + f = zero(p) + Δp = zero(p) + Rr = zero(V) + b = zero(V) + b₀ = zero(V) + b₁ = zero(V) + yDiff₀ = zero(V) + yDiff₁ = zero(V) + Gp₀ = zero(V) (; c₀, c₋₁, F, f, Δp, Rr, b, b₀, b₁, yDiff₀, yDiff₁, Gp₀) end function ode_method_cache(::OneLegMethod{T}, setup, V, p) where {T} (; NV, Np) = setup.grid - u₋₁ = similar(V) - p₋₁ = similar(p) - F = similar(V) - f = similar(p) - Δp = similar(p) - GΔp = similar(V) + u₋₁ = zero(V) + p₋₁ = zero(p) + F = zero(V) + f = zero(p) + Δp = zero(p) + GΔp = zero(V) (; u₋₁, p₋₁, F, f, Δp, GΔp) end function ode_method_cache(method::ExplicitRungeKuttaMethod{T}, setup, u, p) where {T} - u₀ = similar.(u) + u₀ = zero.(u) ns = nstage(method) - ku = [similar.(u) for i = 1:ns] - v = similar.(u) - F = similar.(u) - G = similar.(u) - M = similar(p) + ku = [zero.(u) for i = 1:ns] + v = zero.(u) + F = zero.(u) + G = zero.(u) + M = zero(p) (; u₀, ku, v, F, M, G) end @@ -50,8 +50,8 @@ function ode_method_cache(method::ImplicitRungeKuttaMethod{T}, setup, V, p) wher (; G, M) = setup.operators (; A, b, c) = method - Vₙ = similar(V) - pₙ = similar(p) + Vₙ = zero(V) + pₙ = zero(p) # Number of stages s = length(b) @@ -72,11 +72,11 @@ function ode_method_cache(method::ImplicitRungeKuttaMethod{T}, setup, V, p) wher fⱼ = zeros(T, s * (NV + Np)) - F = similar(V) + F = zero(V) ∇F = spzeros(T, NV, NV) - f = similar(p) - Δp = similar(p) - Gp = similar(V) + f = zero(p) + Δp = zero(p) + Gp = zero(V) # Gradient operator (could also use 1 instead of c and later scale the pressure) Gtot = kron(A, G) From c24a0cddb31eee6fdf8682e5a177de49883b58b9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Wed, 20 Sep 2023 11:03:23 +0200 Subject: [PATCH 066/379] Update workflows --- .github/workflows/CI.yml | 1 + .github/workflows/FormatCheck.yml | 1 + .github/workflows/TagBot.yml | 17 ++++++++++++++++- 3 files changed, 18 insertions(+), 1 deletion(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 04bef0ffa..c04fc2fa9 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -6,6 +6,7 @@ on: - main tags: ['*'] pull_request: + workflow_dispatch: concurrency: # Skip intermediate builds: always. diff --git a/.github/workflows/FormatCheck.yml b/.github/workflows/FormatCheck.yml index e91759519..1f89190ce 100644 --- a/.github/workflows/FormatCheck.yml +++ b/.github/workflows/FormatCheck.yml @@ -5,6 +5,7 @@ on: branches: - "main" pull_request: + workflow_dispatch: jobs: build: diff --git a/.github/workflows/TagBot.yml b/.github/workflows/TagBot.yml index 6d2efc1ce..7667893fb 100644 --- a/.github/workflows/TagBot.yml +++ b/.github/workflows/TagBot.yml @@ -5,7 +5,22 @@ on: types: - created workflow_dispatch: - + inputs: + lookback: + default: 3 +permissions: + actions: read + checks: read + contents: write + deployments: read + issues: read + discussions: read + packages: read + pages: read + pull-requests: read + repository-projects: read + security-events: read + statuses: read jobs: TagBot: if: github.event_name == 'workflow_dispatch' || github.actor == 'JuliaTagBot' From fa0b5bfef9a0e441a8a7b2c70de0448f5ae59d59 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Wed, 20 Sep 2023 11:08:17 +0200 Subject: [PATCH 067/379] feat: Pass array type --- examples/Actuator2D.jl | 27 ++++++++------- examples/Actuator3D.jl | 27 ++++++++------- examples/BackwardFacingStep2D.jl | 14 ++++---- examples/BackwardFacingStep3D.jl | 26 +++++++------- examples/DecayingTurbulence2D.jl | 18 +++++----- examples/DecayingTurbulence3D.jl | 25 +++++++------- examples/LidDrivenCavity2D.jl | 32 +++++++++--------- examples/LidDrivenCavity3D.jl | 28 +++++++-------- examples/PlanarMixing2D.jl | 58 +++++++++++++++----------------- examples/PlaneJets2D.jl | 23 +++++++------ examples/ShearLayer2D.jl | 17 +++++----- examples/TaylorGreenVortex2D.jl | 16 ++++----- examples/TaylorGreenVortex3D.jl | 30 +++++++++-------- src/grid/grid.jl | 18 +++++----- src/setup.jl | 9 +++-- 15 files changed, 186 insertions(+), 182 deletions(-) diff --git a/examples/Actuator2D.jl b/examples/Actuator2D.jl index 27ca949cf..b58d41484 100644 --- a/examples/Actuator2D.jl +++ b/examples/Actuator2D.jl @@ -27,16 +27,22 @@ name = "Actuator2D" # Floating point type T = Float64 -# For CPU -device = identity - -# For GPU (note that `cu` converts to `Float32`) -## using CUDA -## device = cu +# Array type +ArrayType = Array +## using CUDA; ArrayType = CuArray +## using AMDGPU; ArrayType = ROCArray +## using oneAPI; ArrayType = oneArray +## using Metal; ArrayType = MtlArray # Reynolds number Re = T(100) +# A 2D grid is a Cartesian product of two vectors +n = 40 +x = LinRange(0.0, 10.0, 5n + 1) +y = LinRange(-2.0, 2.0, 2n + 1) +plot_grid(x, y) + # Boundary conditions: Unsteady BC requires time derivatives U(x, y, t) = cos(π / 6 * sin(π / 6 * t)) V(x, y, t) = sin(π / 6 * sin(π / 6 * t)) @@ -50,12 +56,6 @@ boundary_conditions = ( (SymmetricBC(), SymmetricBC()), ) -# A 2D grid is a Cartesian product of two vectors -n = 40 -x = LinRange(0.0, 10.0, 5n + 1) -y = LinRange(-2.0, 2.0, 2n + 1) -plot_grid(x, y) - # Actuator body force: A thrust coefficient `Cₜ` distributed over a thin rectangle xc, yc = T(2), T(0) # Disk center D = T(1) # Disk diameter @@ -68,10 +68,11 @@ fv(x, y) = zero(x) # Build setup and assemble operators setup = Setup( - (x, y); + x, y; Re, boundary_conditions, bodyforce = (fu, fv), + ArrayType, ); # Time interval diff --git a/examples/Actuator3D.jl b/examples/Actuator3D.jl index 076f064df..265752e32 100644 --- a/examples/Actuator3D.jl +++ b/examples/Actuator3D.jl @@ -27,16 +27,22 @@ name = "Actuator3D" # Floating point type T = Float64 -# For CPU -device = identity - -# For GPU (note that `cu` converts to `Float32`) -## using CUDA -## device = cu +# Array type +ArrayType = Array +## using CUDA; ArrayType = CuArray +## using AMDGPU; ArrayType = ROCArray +## using oneAPI; ArrayType = oneArray +## using Metal; ArrayType = MtlArray # Reynolds number Re = T(100) +# A 3D grid is a Cartesian product of three vectors +x = LinRange(0.0, 6.0, 31) +y = LinRange(-2.0, 2.0, 41) +z = LinRange(-2.0, 2.0, 41) +plot_grid(x, y, z) + # Boundary conditions: Unsteady BC requires time derivatives U(x, y, z, t) = cos(π / 6 * sin(π / 6 * t)) V(x, y, z, t) = sin(π / 6 * sin(π / 6 * t)) @@ -55,12 +61,6 @@ boundary_conditions = ( (SymmetricBC(), SymmetricBC()), ) -# A 3D grid is a Cartesian product of three vectors -x = LinRange(0.0, 6.0, 31) -y = LinRange(-2.0, 2.0, 41) -z = LinRange(-2.0, 2.0, 41) -plot_grid(x, y, z) - # Actuator body force: A thrust coefficient `Cₜ` distributed over a short cylinder cx, cy, cz = T(2), T(0), T(0) # Disk center D = T(1) # Disk diameter @@ -74,10 +74,11 @@ fw(x, y, z) = zero(x) # Build setup and assemble operators setup = Setup( - (x, y, z); + x, y, z; Re, boundary_conditions, bodyforce = (fu, fv, fw), + ArrayType, ); # Time interval diff --git a/examples/BackwardFacingStep2D.jl b/examples/BackwardFacingStep2D.jl index 924762c56..5ba286b63 100644 --- a/examples/BackwardFacingStep2D.jl +++ b/examples/BackwardFacingStep2D.jl @@ -28,12 +28,12 @@ name = "BackwardFacingStep2D" # Floating point type T = Float64 -# For CPU -device = identity - -# For GPU (note that `cu` converts to `Float32`) -## using CUDA -## device = cu +# Array type +ArrayType = Array +## using CUDA; ArrayType = CuArray +## using AMDGPU; ArrayType = ROCArray +## using oneAPI; ArrayType = oneArray +## using Metal; ArrayType = MtlArray # Reynolds number Re = T(3000) @@ -58,7 +58,7 @@ y = cosine_grid(-T(0.5), T(0.5), 51) plot_grid(x, y) # Build setup and assemble operators -setup = Setup((x, y); Re, boundary_conditions); +setup = Setup(x, y; Re, boundary_conditions, ArrayType); # Time interval t_start, t_end = tlims = T(0), T(7) diff --git a/examples/BackwardFacingStep3D.jl b/examples/BackwardFacingStep3D.jl index 5ef998714..69d2c1803 100644 --- a/examples/BackwardFacingStep3D.jl +++ b/examples/BackwardFacingStep3D.jl @@ -28,16 +28,22 @@ name = "BackwardFacingStep3D" # Floating point type T = Float64 -# For CPU -device = identity - -# For GPU (note that `cu` converts to `Float32`) -## using CUDA -## device = cu +# Array type +ArrayType = Array +## using CUDA; ArrayType = CuArray +## using AMDGPU; ArrayType = ROCArray +## using oneAPI; ArrayType = oneArray +## using Metal; ArrayType = MtlArray # Reynolds number Re = T(3000) +# A 3D grid is a Cartesian product of three vectors +x = LinRange(T(0), T(10), 129) +y = LinRange(-T(0.5), T(0.5), 17) +z = LinRange(-T(0.25), T(0.25), 9) +plot_grid(x, y, z) + # Boundary conditions: steady inflow on the top half U(x, y, z, t) = y ≥ 0 ? 24y * (1 - y) / 2 : zero(x) V(x, y, z, t) = zero(x) @@ -56,14 +62,8 @@ boundary_conditions = ( (PeriodicBC(), PeriodicBC()), ) -# A 2D grid is a Cartesian product of two vectors. -x = LinRange(T(0), T(10), 129) -y = LinRange(-T(0.5), T(0.5), 17) -z = LinRange(-T(0.25), T(0.25), 9) -plot_grid(x, y, z) - # Build setup and assemble operators -setup = Setup((x, y, z); Re, boundary_conditions); +setup = Setup(x, y, z; Re, boundary_conditions, ArrayType); # Time interval t_start, t_end = tlims = T(0), T(7) diff --git a/examples/DecayingTurbulence2D.jl b/examples/DecayingTurbulence2D.jl index 2e0fb1556..bc199ccf9 100644 --- a/examples/DecayingTurbulence2D.jl +++ b/examples/DecayingTurbulence2D.jl @@ -29,13 +29,12 @@ name = "DecayingTurbulence2D" # Floating point precision T = Float64 -# To use CPU: Do not move any arrays -device = identity - -# To use GPU, use `cu` to move arrays to the GPU. -# Note: `cu` converts to Float32 -## using CUDA -## device = cu +# Array type +ArrayType = Array +## using CUDA; ArrayType = CuArray +## using AMDGPU; ArrayType = ROCArray +## using oneAPI; ArrayType = oneArray +## using Metal; ArrayType = MtlArray # Viscosity model Re = T(10_000) @@ -47,7 +46,7 @@ x = LinRange(lims..., n + 1), LinRange(lims..., n + 1) # plot_grid(x...) # Build setup and assemble operators -setup = device(Setup(x; Re)); +setup = Setup(x...; Re, ArrayType); # Since the grid is uniform and identical for x and y, we may use a specialized # spectral pressure solver @@ -67,7 +66,7 @@ processors = ( ); # Time interval -t_start, t_end = tlims = T(0), T(1.0) +t_start, t_end = tlims = T(0), T(1) # Solve unsteady problem u, p, outputs = solve_unsteady( @@ -81,7 +80,6 @@ u, p, outputs = solve_unsteady( inplace = true, ); - # ## Post-process # # We may visualize or export the computed fields `(u, p)` diff --git a/examples/DecayingTurbulence3D.jl b/examples/DecayingTurbulence3D.jl index 60d5938f3..121481fe2 100644 --- a/examples/DecayingTurbulence3D.jl +++ b/examples/DecayingTurbulence3D.jl @@ -27,27 +27,28 @@ using IncompressibleNavierStokes name = "DecayingTurbulence3D" # Floating point precision -T = Float32 +T = Float64 -# To use CPU: Do not move any arrays -device = identity +# Array type +ArrayType = Array +## using CUDA; ArrayType = CuArray +## using AMDGPU; ArrayType = ROCArray +## using oneAPI; ArrayType = oneArray +## using Metal; ArrayType = MtlArray -# To use GPU, use `cu` to move arrays to the GPU. -# Note: `cu` converts to Float32 -## using CUDA -## device = cu - -# Viscosity model +# Reynolds number Re = T(10_000) # A 3D grid is a Cartesian product of three vectors n = 32 lims = T(0), T(1) -x = ntuple(α -> LinRange(lims..., n + 1), 3) -# plot_grid(x...) +x = LinRange(lims..., n + 1) +y = LinRange(lims..., n + 1) +z = LinRange(lims..., n + 1) +plot_grid(x, y, z) # Build setup and assemble operators -setup = device(Setup(x; Re)); +setup = Setup(x, y, z; Re, ArrayType); # Since the grid is uniform and identical for x, y, and z, we may use a # specialized spectral pressure solver diff --git a/examples/LidDrivenCavity2D.jl b/examples/LidDrivenCavity2D.jl index 4d46c12af..409dd3b0b 100644 --- a/examples/LidDrivenCavity2D.jl +++ b/examples/LidDrivenCavity2D.jl @@ -33,25 +33,24 @@ name = "LidDrivenCavity2D" # scaled judiciously to avoid vanishing digits when applying differential # operators of the form "right minus left divided by small distance". -## T = Float16 -## T = Float32 T = Float64 +## T = Float32 +## T = Float16 # Note how floating point type hygiene is enforced in the following using `T` # to avoid mixing different precisions. -# We can also choose to do the computations on a different device. -# By default, the computations are performed on the host (CPU). An optional -# `device` keyword allows for moving arrays and operator to a different device -# such as a GPU. Currently, only Nvidia GPUs with CUDA support sparse operators -# and fast Fourier transform used by IncompressibleNavierStokes. - -# For CPU -device = identity +# We can also choose to do the computations on a different device. By default, +# the computations are performed on the host (CPU). An optional `ArrayType` +# allows for moving arrays to a different device such as a GPU. +# +# Note: For GPUs, single precision is preferred. -# For GPU (note that `cu` converts to `Float32`) -## using CUDA -## device = cu +ArrayType = Array +## using CUDA; ArrayType = CuArray +## using AMDGPU; ArrayType = ROCArray +## using oneAPI; ArrayType = oneArray +## using Metal; ArrayType = MtlArray # Here we choose a moderate Reynolds number. Note how we pass the floating point type. Re = T(1_000) @@ -79,12 +78,13 @@ boundary_conditions = ( # the walls. n = 40 lims = T(0), T(1) -x = cosine_grid(lims..., n), cosine_grid(lims..., n) -plot_grid(x...) +x = cosine_grid(lims..., n) +y = cosine_grid(lims..., n) +plot_grid(x, y) # We can now build the setup and assemble operators. # A 3D setup is built if we also provide a vector of z-coordinates. -setup = device(Setup(x; boundary_conditions, Re)); +setup = Setup(x, y; boundary_conditions, Re, ArrayType); # The pressure solver is used to solve the pressure Poisson equation. # Available solvers are diff --git a/examples/LidDrivenCavity3D.jl b/examples/LidDrivenCavity3D.jl index f1eff9c81..cb08c6b9b 100644 --- a/examples/LidDrivenCavity3D.jl +++ b/examples/LidDrivenCavity3D.jl @@ -27,16 +27,23 @@ name = "LidDrivenCavity3D" # Floating point type T = Float64 -# For CPU -device = identity - -# For GPU (note that `cu` converts to `Float32`) -## using CUDA -## device = cu +# Array type +ArrayType = Array +## using CUDA; ArrayType = CuArray +## using AMDGPU; ArrayType = ROCArray +## using oneAPI; ArrayType = oneArray +## using Metal; ArrayType = MtlArray # Reynolds number Re = T(1_000) +# A 3D grid is a Cartesian product of three vectors. Here we refine the grid +# near the walls. +x = cosine_grid(T(0), T(1), 25) +y = cosine_grid(T(0), T(1), 25) +z = LinRange(-T(0.2), T(0.2), 11) +plot_grid(x, y, z) + # Boundary conditions: horizontal movement of the top lid lidvel = ( (x, y, z, t) -> one(x), @@ -59,15 +66,8 @@ boundary_conditions = ( (PeriodicBC(), PeriodicBC()), ) -# A 3D grid is a Cartesian product of three vectors. Here we refine the grid -# near the walls. -x = cosine_grid(T(0), T(1), 25) -y = cosine_grid(T(0), T(1), 25) -z = LinRange(-T(0.2), T(0.2), 11) -plot_grid(x, y, z) - # Build setup and assemble operators -setup = Setup((x, y, z); Re, boundary_conditions); +setup = Setup(x, y, z; Re, boundary_conditions, ArrayType); # Time interval t_start, t_end = tlims = T(0), T(0.2) diff --git a/examples/PlanarMixing2D.jl b/examples/PlanarMixing2D.jl index 90367cb35..a047f7adc 100644 --- a/examples/PlanarMixing2D.jl +++ b/examples/PlanarMixing2D.jl @@ -27,21 +27,17 @@ Re = 500.0 # Boundary conditions: Unsteady BC requires time derivatives ΔU = 1.0 -Ū = 1.0 -ϵ = (0.082Ū, 0.012Ū) +Ubar = 1.0 +ϵ = (0.082Ubar, 0.012Ubar) n = (0.4π, 0.3π) ω = (0.22, 0.11) -u_bc(x, y, t) = - x ≈ 0.0 ? - 1.0 + ΔU / 2 * tanh(2y) + sum(@. ϵ * (1 - tanh(y / 2)^2) * cos(n * y) * sin(ω * t)) : - 0.0 -v_bc(x, y, t) = 0.0 -dudt_bc(x, y, t) = - x ≈ 0.0 ? sum(@. ϵ * (1 - tanh(y / 2)^2) * cos(n * y) * ω * cos(ω * t)) : 0.0 -dvdt_bc(x, y, t) = 0.0 -bc_type = (; - u = (; x = (:dirichlet, :pressure), y = (:symmetric, :symmetric)), - v = (; x = (:dirichlet, :symmetric), y = (:pressure, :pressure)), +U(x, y, t) = 1.0 + ΔU / 2 * tanh(2y) + sum(@. ϵ * (1 - tanh(y / 2)^2) * cos(n * y) * sin(ω * t)) +V(x, y, t) = 0.0 +dUdt(x, y, t) = sum(@. ϵ * (1 - tanh(y / 2)^2) * cos(n * y) * ω * cos(ω * t)) +dVdt(x, y, t) = 0.0 +boundary_conditions = ( + (DirichletBC((U, V), (dUdt, dVdt)), PressureBC()), + (SymmetricBC(), SymmetricBC()) ) # A 2D grid is a Cartesian product of two vectors @@ -52,21 +48,20 @@ y = LinRange(-32.0, 32.0, n) plot_grid(x, y) # Build setup and assemble operators -setup = Setup(x, y; Re, u_bc, v_bc, dudt_bc, dvdt_bc, bc_type); +setup = Setup(x, y; U, boundary_conditions); # Time interval -t_start, t_end = tlims = (0.0, 100.0) +t_start, t_end = tlims = 0.0, 100.0 -# Initial conditions -initial_velocity_u(x, y) = u_bc(0.0, y, 0.0) -initial_velocity_v(x, y) = 0.0 -initial_pressure(x, y) = 0.0 -V₀, p₀ = create_initial_conditions( +# Initial conditions (exten inflow) +initial_velocity = ( + (x, y) -> U(x, y, 0.0), + (x, y) -> 0.0, +) +u₀, p₀ = create_initial_conditions( setup, - initial_velocity_u, - initial_velocity_v, + initial_velocity, t_start; - initial_pressure, ); # Iteration processors @@ -81,9 +76,9 @@ processors = ( ); # Solve unsteady problem -V, p, outputs = solve_unsteady( +u, p, outputs = solve_unsteady( setup, - V₀, + u₀, p₀, tlims; method = RK44P2(), @@ -91,23 +86,24 @@ V, p, outputs = solve_unsteady( processors, inplace = true, ); -#md current_figure() # ## Post-process # -# We may visualize or export the computed fields `(V, p)` +# We may visualize or export the computed fields `(u, p)` + +outputs[1] # Export to VTK -save_vtk(setup, V, p, t_end, "output/solution") +save_vtk(setup, u, p, "output/solution") # Plot pressure plot_pressure(setup, p) # Plot velocity -plot_velocity(setup, V, t_end) +plot_velocity(setup, u) # Plot vorticity -plot_vorticity(setup, V, t_end) +plot_vorticity(setup, u) # Plot streamfunction -plot_streamfunction(setup, V, t_end) +plot_streamfunction(setup, u) diff --git a/examples/PlaneJets2D.jl b/examples/PlaneJets2D.jl index 8e20af670..cf79989ff 100644 --- a/examples/PlaneJets2D.jl +++ b/examples/PlaneJets2D.jl @@ -28,12 +28,12 @@ name = "PlaneJets2D" # Floating point type T = Float64 -# For CPU -device = identity - -# For GPU (note that `cu` converts to `Float32`) -## using CUDA -## device = cu +# Array type +ArrayType = Array +## using CUDA; ArrayType = CuArray +## using AMDGPU; ArrayType = ROCArray +## using oneAPI; ArrayType = oneArray +## using Metal; ArrayType = MtlArray # Reynolds number Re = T(6_000) @@ -85,12 +85,13 @@ U(x, y) = (1 + T(0.1) * (rand(T) - T(0.5))) * U(y) n = 64 ## n = 128 ## n = 256 -x = LinRange(T(0), T(16), 4n), LinRange(-T(10), T(10), 5n) -plot_grid(x...) +x = LinRange(T(0), T(16), 4n) +y = LinRange(-T(10), T(10), 5n) +plot_grid(x, y) # Build setup and assemble operators -setup = device(Setup(x; Re)); -## setup = device(Setup(x, y; Re, boundary_conditions)); +setup = Setup(x, y; Re, ArrayType); +## setup = Setup(x, y; Re, boundary_conditions, ArrayType); # Since the grid is uniform and identical for x and y, we may use a specialized # spectral pressure solver @@ -102,7 +103,7 @@ t_start, t_end = tlims = T(0), T(1) # Initial conditions initial_velocity = ( (x, y) -> U(x, y), - (x, y) -> zero(T), + (x, y) -> zero(x), ) u₀, p₀ = create_initial_conditions( setup, diff --git a/examples/ShearLayer2D.jl b/examples/ShearLayer2D.jl index 8f30305b6..78a5635c3 100644 --- a/examples/ShearLayer2D.jl +++ b/examples/ShearLayer2D.jl @@ -25,12 +25,12 @@ name = "ShearLayer2D" # Floating point type T = Float64 -# For CPU -device = identity - -# For GPU (note that `cu` converts to `Float32`) -## using CUDA -## device = cu +# Array type +ArrayType = Array +## using CUDA; ArrayType = CuArray +## using AMDGPU; ArrayType = ROCArray +## using oneAPI; ArrayType = oneArray +## using Metal; ArrayType = MtlArray # Reynolds number Re = T(Inf) @@ -38,11 +38,12 @@ Re = T(Inf) # A 2D grid is a Cartesian product of two vectors n = 100 lims = T(0), T(2π) -x = LinRange(lims..., n + 1), LinRange(lims..., n + 1) +x = LinRange(lims..., n + 1) +y = LinRange(lims..., n + 1) plot_grid(x, y) # Build setup and assemble operators -setup = device(Setup(x; Re)); +setup = Setup(x, y; Re, ArrayType); # Time interval t_start, t_end = tlims = T(0), T(8) diff --git a/examples/TaylorGreenVortex2D.jl b/examples/TaylorGreenVortex2D.jl index dcec7f5ce..7e0ca8f10 100644 --- a/examples/TaylorGreenVortex2D.jl +++ b/examples/TaylorGreenVortex2D.jl @@ -23,14 +23,14 @@ using IncompressibleNavierStokes name = "TaylorGreenVortex2D" # Floating point type -T = Float32 +T = Float64 -# For CPU -device = identity - -# For GPU (note that `cu` converts to `Float32`) -## using CUDA -## device = cu +# Array type +ArrayType = Array +## using CUDA; ArrayType = CuArray +## using AMDGPU; ArrayType = ROCArray +## using oneAPI; ArrayType = oneArray +## using Metal; ArrayType = MtlArray # Reynolds number Re = T(2_000) @@ -49,7 +49,7 @@ setup = device(Setup(x; Re)); pressure_solver = SpectralPressureSolver(setup) # Time interval -t_start, t_end = tlims = T(0), T(1) +t_start, t_end = tlims = T(0), T(5) # Initial conditions initial_velocity = ( diff --git a/examples/TaylorGreenVortex3D.jl b/examples/TaylorGreenVortex3D.jl index f3aab4eda..4f044125f 100644 --- a/examples/TaylorGreenVortex3D.jl +++ b/examples/TaylorGreenVortex3D.jl @@ -25,24 +25,26 @@ name = "TaylorGreenVortex3D" # Floating point precision T = Float64 -# For CPU -device = identity - -# For GPU (note that `cu` converts to `Float32`) -## using CUDA -## device = cu - -# Viscosity model +# Array type +ArrayType = Array +## using CUDA; ArrayType = CuArray +## using AMDGPU; ArrayType = ROCArray +## using oneAPI; ArrayType = oneArray +## using Metal; ArrayType = MtlArray + +# Reynolds number Re = T(6_000) # A 3D grid is a Cartesian product of three vectors n = 32 lims = T(0), T(2π) -x = ntuple(α -> LinRange(lims..., n + 1), 3) -plot_grid(x...) +x = LinRange(lims..., n + 1) +y = LinRange(lims..., n + 1) +z = LinRange(lims..., n + 1) +plot_grid(x, y, z) # Build setup and assemble operators -setup = device(Setup(x; Re)); +setup = Setup(x, y, z; Re, ArrayType); # Since the grid is uniform and identical for x, y, and z, we may use a # specialized spectral pressure solver @@ -69,9 +71,9 @@ CUDA.reclaim() # Iteration processors processors = ( - field_plotter(setup; nupdate = 1), - ## energy_history_plotter(setup; nupdate = 1), - ## energy_spectrum_plotter(setup; nupdate = 100), + # field_plotter(setup; fieldname = :velocity, nupdate = 1), + # energy_history_plotter(setup; nupdate = 1), + energy_spectrum_plotter(setup; nupdate = 100), ## animator(setup, "vorticity.mp4"; nupdate = 4), ## vtk_writer(setup; nupdate = 10, dir = "output/$name", filename = "solution"), ## field_saver(setup; nupdate = 10), diff --git a/src/grid/grid.jl b/src/grid/grid.jl index 017bc50b7..93a3c304b 100644 --- a/src/grid/grid.jl +++ b/src/grid/grid.jl @@ -4,7 +4,7 @@ Create nonuniform Cartesian box mesh `x[1]` × ... × `x[d]` with boundary conditions `boundary_conditions`. """ -function Grid(x, boundary_conditions) +function Grid(x, boundary_conditions; ArrayType = Array) # Kill all LinRanges etc. x = Array.(x) xlims = extrema.(x) @@ -103,13 +103,13 @@ function Grid(x, boundary_conditions) Iu, Ip, xlims, - x, - xp, - Δ, - Δu, - Ω, - # Ωu, - # Ωω, - # Γu, + x = ArrayType.(x), + xp = ArrayType.(xp), + Δ = ArrayType.(Δ), + Δu = ArrayType.(Δu), + Ω = ArrayType(Ω), + # Ωu = ArrayType.(Ωu), + # Ωω = ArrayType(Ωω), + # Γu = ArrayType.(Γu), ) end diff --git a/src/setup.jl b/src/setup.jl index d9fccc73a..fee44032d 100644 --- a/src/setup.jl +++ b/src/setup.jl @@ -1,26 +1,28 @@ """ Setup( - x; + x...; boundary_conditions = ntuple(d -> (PeriodicBC(), PeriodicBC()), length(x)), Re = convert(eltype(x[1]), 1_000), viscosity_model = LaminarModel(), convection_model = NoRegConvectionModel(), bodyforce = nothing, closure_model = nothing, + ArrayType = Array, ) Create setup. """ function Setup( - x; + x...; boundary_conditions = ntuple(d -> (PeriodicBC(), PeriodicBC()), length(x)), Re = convert(eltype(x[1]), 1_000), viscosity_model = LaminarModel(), convection_model = NoRegConvectionModel(), bodyforce = nothing, closure_model = nothing, + ArrayType = Array, ) - grid = Grid(x, boundary_conditions) + grid = Grid(x, boundary_conditions; ArrayType) (; grid, boundary_conditions, @@ -29,5 +31,6 @@ function Setup( convection_model, bodyforce, closure_model, + ArrayType, ) end From 43e1e228bac4f8a69f5a25331195719a103259f5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Wed, 20 Sep 2023 11:09:36 +0200 Subject: [PATCH 068/379] Fix ticks --- src/boundary_conditions.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/boundary_conditions.jl b/src/boundary_conditions.jl index 09aa9ff5d..b24d77b7d 100644 --- a/src/boundary_conditions.jl +++ b/src/boundary_conditions.jl @@ -71,8 +71,8 @@ ghost_b!(::PressureBC, x) = push!(x, x[end], x[end] + (x[end] - x[end-1])) offset_u(bc, isnormal, atend) Number of non-DOF velocity components at boundary. -If ``isnormal``, then the velocity is normal to the boundary, else parallel. -If ``atend``, it is at the end/right/rear/top boundary, otherwise beginning. +If `isnormal`, then the velocity is normal to the boundary, else parallel. +If `atend`, it is at the end/right/rear/top boundary, otherwise beginning. """ function offset_u end From c66611d2bdd8d2163f2f030637b7d0662ba1290d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Wed, 20 Sep 2023 11:11:02 +0200 Subject: [PATCH 069/379] docs: Add docstrings --- src/operators.jl | 54 ++++++++++++++++++++++++++++++++++++++- src/time_steppers/step.jl | 4 +-- 2 files changed, 55 insertions(+), 3 deletions(-) diff --git a/src/operators.jl b/src/operators.jl index 22700b427..1e9883241 100644 --- a/src/operators.jl +++ b/src/operators.jl @@ -4,6 +4,11 @@ struct Offset{D} end (::Offset{D})(i) where {D} = CartesianIndex(ntuple(j -> j == i ? 1 : 0, D)) +""" + divergence!(M, u, setup) + +Compute divergence of velocity field (in-place version). +""" function divergence!(M, u, setup) (; boundary_conditions, grid) = setup (; Δ, Np, Ip, Ω) = grid @@ -26,12 +31,22 @@ function divergence!(M, u, setup) M end +""" + divergence(u, setup) + +Compute divergence of velocity field. +""" divergence(u, setup) = divergence!( KernelAbstractions.zeros(get_backend(u[1]), eltype(u[1]), setup.grid.N...), u, setup, ) +""" + vorticity(u, setup) + +Compute vorticity field. +""" vorticity(u, setup) = vorticity!( setup.grid.dimension() == 2 ? KernelAbstractions.zeros(get_backend(u[1]), eltype(u[1]), setup.grid.N) : @@ -42,6 +57,12 @@ vorticity(u, setup) = vorticity!( u, setup, ) + +""" + vorticity!(ω, u, setup) + +Compute vorticity field. +""" vorticity!(ω, u, setup) = vorticity!(setup.grid.dimension, ω, u, setup) function vorticity!(::Dimension{2}, ω, u, setup) @@ -84,6 +105,11 @@ function vorticity!(::Dimension{3}, ω, u, setup) ω end +""" + convection!(F, u, setup) + +Compute convective term. +""" function convection!(F, u, setup) (; boundary_conditions, grid, Re, bodyforce) = setup (; dimension, Δ, Δu, Nu, Iu) = grid @@ -110,8 +136,13 @@ function convection!(F, u, setup) F end -# Add ϵ in denominator for "infinitely thin" volumes +""" + diffusion!(F, u, setup; ϵ = eps(eltype(F[1]))) + +Compute diffusive term. +""" function diffusion!(F, u, setup; ϵ = eps(eltype(F[1]))) + # Add ϵ in denominator for "infinitely thin" volumes (; boundary_conditions, grid, Re, bodyforce) = setup (; dimension, Δ, Δu, Nu, Iu) = grid D = dimension() @@ -138,6 +169,11 @@ function diffusion!(F, u, setup; ϵ = eps(eltype(F[1]))) F end +""" + bodyforce!(F, u, setup; ϵ = eps(eltype(F[1]))) + +Compute body force. +""" function bodyforce!(F, u, t, setup) (; boundary_conditions, grid, Re, bodyforce) = setup (; dimension, Δ, Δu, Nu, Iu, x, xp) = grid @@ -164,6 +200,12 @@ function bodyforce!(F, u, t, setup) F end +""" + momentum!(F, u, t, setup) + +Right hand side of momentum equations, excluding pressure gradient. +Put the result in ``F``. +""" function momentum!(F, u, t, setup) (; grid, closure_model) = setup (; dimension) = grid @@ -178,6 +220,11 @@ function momentum!(F, u, t, setup) F end +""" + pressuregradient!(G, p, setup) + +Compute pressure gradient (in-place). +""" function pressuregradient!(G, p, setup) (; boundary_conditions, grid) = setup (; dimension, Δu, Np, Iu) = grid @@ -198,6 +245,11 @@ function pressuregradient!(G, p, setup) G end +""" + pressuregradient(p, setup) + +Compute pressure gradient. +""" pressuregradient(p, setup) = pressuregradient!( ntuple( α -> KernelAbstractions.zeros(get_backend(p), eltype(p), setup.grid.N), diff --git a/src/time_steppers/step.jl b/src/time_steppers/step.jl index e61fb4c08..de42789d5 100644 --- a/src/time_steppers/step.jl +++ b/src/time_steppers/step.jl @@ -5,7 +5,7 @@ Perform one time step. Non-mutating/allocating/out-of-place version. -See also [`step!`](@ref). +See also [`timestep!`](@ref). """ function timestep end # step(stepper, Δt; bc_vectors = nothing) = step(stepper.method, stepper, Δt; bc_vectors = nothing) @@ -17,7 +17,7 @@ Perform one time step> Mutating/non-allocating/in-place version. -See also [`step`](@ref). +See also [`timestep`](@ref). """ function timestep! end # step!(stepper, Δt; kwargs...) = step!(stepper.method, stepper, Δt; kwargs...) From b6fb2f009f9d6bc08b34494960ef9cdecaee1107 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Wed, 20 Sep 2023 11:11:46 +0200 Subject: [PATCH 070/379] Fix plots --- src/operators.jl | 18 +++++ src/processors/real_time_plot.jl | 119 +++++++++++++------------------ 2 files changed, 67 insertions(+), 70 deletions(-) diff --git a/src/operators.jl b/src/operators.jl index 1e9883241..966ba6784 100644 --- a/src/operators.jl +++ b/src/operators.jl @@ -337,3 +337,21 @@ function interpolate_ω_p!(::Dimension{3}, setup, ωp, ω) synchronize(get_backend(ωp[1])) ωp end + +""" + kinetic_energy(setup, u) + +Compute total kinetic energy. The velocity components are interpolated to the +volume centers and squared. +""" +function kinetic_energy(setup, u) + (; dimension, Ω, Ip) = setup.grid + D = dimension() + up = interpolate_u_p(setup, u) + E = zero(eltype(up[1])) + for α = 1:D + # E += sum(I -> Ω[I] * up[α][I]^2, Ip) + E += sum(Ω[Ip] .* up[α][Ip] .^ 2) + end + sqrt(E) +end diff --git a/src/processors/real_time_plot.jl b/src/processors/real_time_plot.jl index 9e37f68d3..f9a9257ce 100644 --- a/src/processors/real_time_plot.jl +++ b/src/processors/real_time_plot.jl @@ -46,12 +46,13 @@ function field_plot( displayfig = true, ) (; boundary_conditions, grid) = setup - (; xlims, x, xp) = grid + (; dimension, xlims, x, xp, Ip) = grid + D = dimension() if fieldname == :velocity xf = xp elseif fieldname == :vorticity - xf = Array.(x) + xf = ntuple(α -> Array(xp[α][Ip.indices[α]]), D) elseif fieldname == :streamfunction if boundary_conditions.u.x[1] == :periodic xf = x @@ -77,7 +78,7 @@ function field_plot( up, vp = get_velocity(setup, u, t) map((u, v) -> √sum(u^2 + v^2), up, vp) elseif fieldname == :vorticity - vorticity(u, setup) + interpolate_ω_p(setup, vorticity(u, setup))[Ip] elseif fieldname == :streamfunction get_streamfunction(setup, u, t) elseif fieldname == :pressure @@ -151,7 +152,7 @@ function field_plot( (; xlims, x, xp) = grid if fieldname == :velocity - xf, yf, zf = xp, yp, zp + xf = xp elseif fieldname == :vorticity xf = xp elseif fieldname == :streamfunction @@ -223,17 +224,15 @@ energy_history_plotter(setup; nupdate = 1, kwargs...) = processor(state -> energy_history_plot(setup, state; kwargs...); nupdate) function energy_history_plot(setup, state; displayfig = true) - (; Ωp) = setup.grid _points = Point2f[] points = @lift begin - (; V, p, t) = $state - vels = get_velocity(setup, V, t) - vels = reshape.(vels, :) - E = sum(vel -> sum(@. Ωp * vel^2), vels) + (; u, p, t) = $state + E = kinetic_energy(setup, u) push!(_points, Point2f(t, E)) end fig = lines(points; axis = (; xlabel = "t", ylabel = "Kinetic energy")) displayfig && display(fig) + on(_ -> autolimits!(fig.axis), points) fig end @@ -243,77 +242,57 @@ end Create energy spectrum plot, redrawn every time `step_observer` is updated. """ energy_spectrum_plotter(setup; nupdate = 1, kwargs...) = processor( - state -> energy_spectrum_plot(setup.grid.dimension, setup, state; kwargs...); + state -> energy_spectrum_plot(setup, state; kwargs...); nupdate, ) -function energy_spectrum_plot(::Dimension{2}, setup, state; displayfig = true) - (; xpp) = setup.grid - Kx, Ky = size(xpp) .÷ 2 - kx = 1:(Kx-1) - ky = 1:(Ky-1) - kk = reshape([sqrt(kx^2 + ky^2) for kx ∈ kx, ky ∈ ky], :) - ehat = @lift begin - (; V, p, t) = $state - up, vp = get_velocity(setup, V, t) - e = up .^ 2 .+ vp .^ 2 - reshape(abs.(fft(e)[kx.+1, ky.+1]), :) +function energy_spectrum_plot(setup, state; displayfig = true) + (; dimension, xp, Ip) = setup.grid + T = eltype(xp[1]) + D = dimension() + K = size(Ip) .÷ 2 + kx = ntuple(α -> 1:K[α]-1, D) + k = KernelAbstractions.zeros(get_backend(xp[1]), T, length.(kx)...) + for α = 1:D + kα = reshape(kx[α], ntuple(Returns(1), α - 1)..., :, ntuple(Returns(1), D - α)...) + k .+= kα .^ 2 end - espec = Figure() - ax = Axis(espec[1, 1]; xlabel = "k", ylabel = "e(k)", xscale = log10, yscale = log10) - ## ylims!(ax, (1e-20, 1)) - scatter!(ax, kk, ehat; label = "Kinetic energy") - krange = LinRange(extrema(kk)..., 100) - lines!(ax, krange, 1e7 * krange .^ (-3); label = "k⁻³", color = :red) - axislegend(ax) - displayfig && display(espec) - espec -end - -function energy_spectrum_plot( - ::Dimension{3}, - setup, - state, - displayfig = true, - checktime = 0.0001, -) - (; xpp) = setup.grid - Kx, Ky, Kz = size(xpp) .÷ 2 - kx = 1:(Kx-1) - ky = 1:(Ky-1) - kz = 1:(Ky-1) - kk = reshape([sqrt(kx^2 + ky^2 + kz^2) for kx ∈ kx, ky ∈ ky, kz ∈ kz], :) + k .= sqrt.(k) + k = Array(reshape(k, :)) ehat = @lift begin - (; V, p, t) = $state - V = Array(V) - up, vp, wp = get_velocity(setup, V, t) - e = @. up^2 + vp^2 + wp^2 - reshape(abs.(fft(e)[kx.+1, ky.+1, kz.+1]), :) + (; u, p, t) = $state + up = interpolate_u_p(setup, u) + e = sum(up -> up[Ip] .^ 2, up) + Array(reshape(abs.(fft(e)[ntuple(α -> kx[α].+1, D)...]), :)) end espec = Figure() ax = Axis(espec[1, 1]; xlabel = "k", ylabel = "e(k)", xscale = log10, yscale = log10) ## ylims!(ax, (1e-20, 1)) - scatter!(ax, kk, ehat; label = "Kinetic energy") - krange = LinRange(extrema(kk)..., 100) - lines!(ax, krange, 1e6 * krange .^ (-5 / 3); label = "\$k^{-5/3}\$", color = :red) + scatter!(ax, k, ehat; label = "Kinetic energy") + krange = LinRange(extrema(k)..., 100) + D == 2 && lines!(ax, krange, 1e7 * krange .^ (-3); label = "k⁻³", color = :red) + D == 3 && lines!(ax, krange, 1e6 * krange .^ (-5 / 3); label = "\$k^{-5/3}\$", color = :red) axislegend(ax) displayfig && display(espec) - return espec - - # Make sure the figure is fully rendered before allowing code to continue - if displayfig - render = display(espec) - done_rendering = Ref(false) - on(render.render_tic) do _ - done_rendering[] = true - end - on(state) do s - # State is updated, block code execution until GLMakie has rendered - # figure update - done_rendering[] = false - while !done_rendering[] - sleep(checktime) - end - end + on(ehat) do _ + autolimits!(ax) end + espec end + +# # Make sure the figure is fully rendered before allowing code to continue +# if displayfig +# render = display(espec) +# done_rendering = Ref(false) +# on(render.render_tic) do _ +# done_rendering[] = true +# end +# on(state) do s +# # State is updated, block code execution until GLMakie has rendered +# # figure update +# done_rendering[] = false +# while !done_rendering[] +# sleep(checktime) +# end +# end +# end From 9103b9c554bbe3bf6b5e5ba89e4ff2dc42ff25ed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Wed, 20 Sep 2023 11:12:20 +0200 Subject: [PATCH 071/379] Step DOFs only --- src/time_steppers/step_explicit_runge_kutta.jl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/time_steppers/step_explicit_runge_kutta.jl b/src/time_steppers/step_explicit_runge_kutta.jl index 6b1686efb..613baeb31 100644 --- a/src/time_steppers/step_explicit_runge_kutta.jl +++ b/src/time_steppers/step_explicit_runge_kutta.jl @@ -4,7 +4,7 @@ create_stepper(::ExplicitRungeKuttaMethod; setup, pressure_solver, u, p, t, n = function timestep!(method::ExplicitRungeKuttaMethod, stepper, Δt; cache) (; setup, pressure_solver, u, p, t, n) = stepper (; grid, boundary_conditions) = setup - (; dimension, Ip) = grid + (; dimension, Iu, Ip) = grid (; A, b, c, p_add_solve) = method (; u₀, ku, v, F, M, G) = cache @@ -42,7 +42,7 @@ function timestep!(method::ExplicitRungeKuttaMethod, stepper, Δt; cache) for α = 1:D v[α] .= u₀[α] for j = 1:i - @. v[α] = v[α] + Δt * A[i, j] * ku[j][α] + @. v[α][Iu[α]] = v[α][Iu[α]] + Δt * A[i, j] * ku[j][α][Iu[α]] end end @@ -64,7 +64,7 @@ function timestep!(method::ExplicitRungeKuttaMethod, stepper, Δt; cache) # Update velocity current stage, which is now divergence free for α = 1:D - @. u[α] = v[α] - c[i] * Δt * G[α] + @. u[α][Iu[α]] = v[α][Iu[α]] - c[i] * Δt * G[α][Iu[α]] end apply_bc_u!(u, t, setup) end From a6ae53c47c0269be861cee8e1540047f6bd50cec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Wed, 20 Sep 2023 11:14:08 +0200 Subject: [PATCH 072/379] Update BC --- src/boundary_conditions.jl | 37 ++++++++++++++++++------------------- src/operators.jl | 5 +++-- 2 files changed, 21 insertions(+), 21 deletions(-) diff --git a/src/boundary_conditions.jl b/src/boundary_conditions.jl index b24d77b7d..9ede7b56a 100644 --- a/src/boundary_conditions.jl +++ b/src/boundary_conditions.jl @@ -107,7 +107,7 @@ function apply_bc_u!(u, t, setup; kwargs...) end end -function apply_bc_p!(p, t, setup) +function apply_bc_p!(p, t, setup; kwargs...) (; boundary_conditions, grid) = setup (; dimension) = grid D = dimension() @@ -146,7 +146,7 @@ function apply_bc_u!(::PeriodicBC, u, β, t, setup; atend, kwargs...) end end -function apply_bc_p!(::PeriodicBC, p, β, t, setup; atend) +function apply_bc_p!(::PeriodicBC, p, β, t, setup; atend, kwargs...) (; grid) = setup (; dimension, Np, Ip) = grid D = dimension() @@ -177,30 +177,34 @@ function apply_bc_u!(bc::DirichletBC, u, β, t, setup; atend, dudt = false, kwar (; dimension, Nu, x, xp) = setup.grid D = dimension() δ = Offset{D}() + isnothing(bc.u) && return bcfunc = dudt ? bc.dudt : bc.u - @kernel function _bc_a(u, α, β) + @kernel function _bc_a(u, α, β, I0) I = @index(Global, Cartesian) - # u[i][I] = bcfunc[i](ntuple(k -> k == i [I + Nu[i][j] * δ(j)] - # TODO: Apply bcfunc + I = I + I0 + u[α][I] = bcfunc[α](ntuple(γ -> γ == α ? x[I[α] + 1] : xp[I[γ]], D)) end - @kernel function _bc_b(u, α, β; xΓ) + @kernel function _bc_b(u, α, β, I0) I = @index(Global, Cartesian) - # u[α][I] = bcfunc[α](ntuple(xp) - # TODO: Apply bcfunc + I = I + I0 + u[α][I] = bcfunc[α](ntuple(γ -> γ == α ? x[I[α] + 1] : xp[I[γ]], D)) end for α = 1:D - xΓ = (xp[1:β-1]..., xp[β+1:end]...) + Xu = (xp[1:β-1]..., x[β], xp[β+1:end]...) + I0 = first(Iu[α]) + I0 -= oneunit(I0) + ndrange = (Nu[α][1:β-1]..., 1, Nu[α][β+1:end]...) if atend - _bc_b(get_backend(u[1]), WORKGROUP)(u, α, β, I0) + _bc_b(get_backend(u[1]), WORKGROUP)(u, α, β, I0; ndrange) synchronize(get_backend(u[1])) else - _bc_a(get_backend(u[1]), WORKGROUP)(u, α, β, I0) + _bc_a(get_backend(u[1]), WORKGROUP)(u, α, β, I0; ndrange) synchronize(get_backend(u[1])) end end end -function apply_bc_p!(::DirichletBC, p, β, t, setup; atend) +function apply_bc_p!(::DirichletBC, p, β, t, setup; atend, kwargs...) nothing end @@ -209,16 +213,11 @@ function apply_bc_u!(::SymmetricBC, u, β, t, setup; atend, kwargs...) (; Nu, x, xp) = setup.grid D = dimension() δ = Offset{D}() - bcfunc = dudt ? bc.dudt : bc.u @kernel function _bc_a(u, α, β) I = @index(Global, Cartesian) - # u[i][I] = bcfunc[i](ntuple(k -> k == i [I + Nu[i][j] * δ(j)] - # TODO: Apply bcfunc end @kernel function _bc_b(u, α, β) I = @index(Global, Cartesian) - # u[α][I] = bcfunc[α](ntuple(xp) - # TODO: Apply bcfunc end for α = 1:D for β = 1:D @@ -234,7 +233,7 @@ function apply_bc_u!(::SymmetricBC, u, β, t, setup; atend, kwargs...) end end -function apply_bc_p!(::SymmetricBC, p, β, t, setup; atend) +function apply_bc_p!(::SymmetricBC, p, β, t, setup; atend, kwargs...) error("Not implemented") end @@ -242,6 +241,6 @@ function apply_bc_u!(bc::PressureBC, u, β, t, setup; atend, kwargs...) error("Not implemented") end -function apply_bc_p!(bc::PressureBC, p, β, t, setup; atend) +function apply_bc_p!(bc::PressureBC, p, β, t, setup; atend, kwargs...) error("Not implemented") end diff --git a/src/operators.jl b/src/operators.jl index 966ba6784..35f58631c 100644 --- a/src/operators.jl +++ b/src/operators.jl @@ -94,7 +94,8 @@ function vorticity!(::Dimension{3}, ω, u, setup) I = I + I0 β = mod1(α + 1, D) γ = mod1(α - 1, D) - ω[α][I] = -(u[β][I+δ(γ)] - u[β][I]) / Δu[γ][I[γ]] + (u[γ][I+δ(β)] - u[γ][I]) / Δu[β][I[β]] + ω[α][I] = + -(u[β][I+δ(γ)] - u[β][I]) / Δu[γ][I[γ]] + (u[γ][I+δ(β)] - u[γ][I]) / Δu[β][I[β]] end for α = 1:D I0 = CartesianIndex(ntuple(Returns(1), D)) @@ -155,7 +156,7 @@ function diffusion!(F, u, setup; ϵ = eps(eltype(F[1]))) F[α][I] += ν * ( (u[α][I+δ(β)] - u[α][I]) / ((β == α ? Δ : Δu)[β][I[β]] + ϵ) - - (u[α][I] - u[α][I-δ(β)]) / ((β == α ? Δ : Δu)[β][(I-δ(β))[β]] + ϵ) + (u[α][I] - u[α][I-δ(β)]) / ((β == α ? Δ : Δu)[β][I[β]-1] + ϵ) ) / Δuαβ[I[β]] end for α = 1:D From 884b97942cc4789c92da05444f98dc31c7e0c8d5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Fri, 22 Sep 2023 22:25:17 +0200 Subject: [PATCH 073/379] Update files --- docs/src/api/api.md | 34 +++- examples/Actuator2D.jl | 4 +- examples/BackwardFacingStep2D.jl | 4 +- examples/LidDrivenCavity2D.jl | 32 ++- examples/PlanarMixing2D.jl | 2 +- examples/ShearLayer2D.jl | 6 +- examples/TaylorGreenVortex2D.jl | 2 +- examples/TaylorGreenVortex3D.jl | 4 +- src/IncompressibleNavierStokes.jl | 9 +- src/boundary_conditions.jl | 190 +++++++++++------- src/create_initial_conditions.jl | 13 +- src/grid/grid.jl | 44 +++- src/operators.jl | 66 +++--- src/postprocess/plot_velocity.jl | 2 +- src/processors/real_time_plot.jl | 31 +-- .../pressure/pressure_additional_solve.jl | 20 +- src/solvers/pressure/pressure_poisson.jl | 80 +++++--- src/solvers/pressure/pressure_solvers.jl | 67 +++++- src/solvers/solve_unsteady.jl | 2 +- .../step_explicit_runge_kutta.jl | 8 +- src/utils/get_lims.jl | 2 +- 21 files changed, 395 insertions(+), 227 deletions(-) diff --git a/docs/src/api/api.md b/docs/src/api/api.md index 787487e8e..6f9c40850 100644 --- a/docs/src/api/api.md +++ b/docs/src/api/api.md @@ -121,16 +121,8 @@ lambda_diff_max nstage ode_method_cache runge_kutta_method -step -step! -``` - -## Filter -```@docs -create_top_hat_u -create_top_hat_v -create_top_hat_p -create_top_hat_velocity +timestep +timestep! ``` ## Utils @@ -139,3 +131,25 @@ create_top_hat_velocity get_lims plotmat ``` + +## Other + +```@docs +DirichletBC +SymmetricBC +PressureBC +mean_squared_error +cnn +relative_error +create_randloss +kinetic_energy +FourierLayer +create_callback +offset_p +momentum_allstage +momentum_allstage! +fno +offset_u +pressuregradient +pressuregradient! +``` diff --git a/examples/Actuator2D.jl b/examples/Actuator2D.jl index b58d41484..5c89c9022 100644 --- a/examples/Actuator2D.jl +++ b/examples/Actuator2D.jl @@ -63,8 +63,8 @@ D = T(1) # Disk diameter Cₜ = T(5e-4) # Thrust coefficient cₜ = Cₜ / (D * δ) inside(x, y) = abs(x - xc) ≤ δ / 2 && abs(y - yc) ≤ D / 2 -fu(x, y) = -cₜ * inside(x, y) -fv(x, y) = zero(x) +fu(x, y, t) = -cₜ * inside(x, y) +fv(x, y, t) = zero(x) # Build setup and assemble operators setup = Setup( diff --git a/examples/BackwardFacingStep2D.jl b/examples/BackwardFacingStep2D.jl index 5ba286b63..30aed52e5 100644 --- a/examples/BackwardFacingStep2D.jl +++ b/examples/BackwardFacingStep2D.jl @@ -36,10 +36,10 @@ ArrayType = Array ## using Metal; ArrayType = MtlArray # Reynolds number -Re = T(3000) +Re = T(3_000) # Boundary conditions: steady inflow on the top half -U(x, y, t) = y ≥ 0 ? 24y * (1 - y) / 2 : zero(x) +U(x, y, t::T) where {T} = y ≥ 0 ? 24y * (T(1 / 2) - y) : zero(x) V(x, y, t) = zero(x) dUdt(x, y, t) = zero(x) dVdt(x, y, t) = zero(x) diff --git a/examples/LidDrivenCavity2D.jl b/examples/LidDrivenCavity2D.jl index 409dd3b0b..3ef0dff80 100644 --- a/examples/LidDrivenCavity2D.jl +++ b/examples/LidDrivenCavity2D.jl @@ -57,14 +57,8 @@ Re = T(1_000) # Non-zero Dirichlet boundary conditions are specified as plain Julia functions. # Other possible BC types are `PeriodicBC()`, `SymmetricBC`, and `PressureBC`. -lidvel = ( - (x, y, t) -> one(x), - (x, y, t) -> zero(x), -) -dlidveldt = ( - (x, y, t) -> zero(x), - (x, y, t) -> zero(x), -) +lidvel = ((x, y, t) -> one(x), (x, y, t) -> zero(x)) +dlidveldt = ((x, y, t) -> zero(x), (x, y, t) -> zero(x)) boundary_conditions = ( ## x left, x right (DirichletBC(), DirichletBC()), @@ -76,7 +70,7 @@ boundary_conditions = ( # We create a two-dimensional domain with a box of size `[1, 1]`. The grid is # created as a Cartesian product between two vectors. We add a refinement near # the walls. -n = 40 +n = 32 lims = T(0), T(1) x = cosine_grid(lims..., n) y = cosine_grid(lims..., n) @@ -94,7 +88,7 @@ setup = Setup(x, y; boundary_conditions, Re, ArrayType); # - [`SpectralPressureSolver`](@ref) (only for periodic boundary conditions and # uniform grids) -pressure_solver = DirectPressureSolver(setup) +pressure_solver = CGPressureSolverManual(setup); # We will solve for a time interval of ten seconds. t_start, t_end = tlims = T(0), T(10) @@ -104,12 +98,7 @@ initial_velocity = ( (x, y) -> zero(x), (x, y) -> zero(x), ) -u₀, p₀ = create_initial_conditions( - setup, - initial_velocity, - t_start; - pressure_solver, -) +u₀, p₀ = create_initial_conditions(setup, initial_velocity, t_start; pressure_solver); # ## Solve problems # @@ -138,8 +127,15 @@ processors = ( # By default, a standard fourth order Runge-Kutta method is used. If we don't # provide the time step explicitly, an adaptive time step is used. -u, p, outputs = - solve_unsteady(setup, u₀, p₀, tlims; Δt = T(0.001), processors, pressure_solver, device); +u, p, outputs = solve_unsteady( + setup, + u₀, + p₀, + tlims; + Δt = T(0.001), + processors, + pressure_solver, +); # ## Post-process # diff --git a/examples/PlanarMixing2D.jl b/examples/PlanarMixing2D.jl index a047f7adc..544ad19ef 100644 --- a/examples/PlanarMixing2D.jl +++ b/examples/PlanarMixing2D.jl @@ -48,7 +48,7 @@ y = LinRange(-32.0, 32.0, n) plot_grid(x, y) # Build setup and assemble operators -setup = Setup(x, y; U, boundary_conditions); +setup = Setup(x, y; Re, boundary_conditions); # Time interval t_start, t_end = tlims = 0.0, 100.0 diff --git a/examples/ShearLayer2D.jl b/examples/ShearLayer2D.jl index 78a5635c3..2263d0f41 100644 --- a/examples/ShearLayer2D.jl +++ b/examples/ShearLayer2D.jl @@ -36,7 +36,7 @@ ArrayType = Array Re = T(Inf) # A 2D grid is a Cartesian product of two vectors -n = 100 +n = 128 lims = T(0), T(2π) x = LinRange(lims..., n + 1) y = LinRange(lims..., n + 1) @@ -45,6 +45,8 @@ plot_grid(x, y) # Build setup and assemble operators setup = Setup(x, y; Re, ArrayType); +pressure_solver = SpectralPressureSolver(setup) + # Time interval t_start, t_end = tlims = T(0), T(8) @@ -64,6 +66,7 @@ u₀, p₀ = create_initial_conditions( setup, initial_velocity, t_start; + pressure_solver, ); # Iteration processors @@ -87,6 +90,7 @@ u, p, outputs = solve_unsteady( Δt = T(0.01), processors, inplace = true, + pressure_solver, ); # ## Post-process diff --git a/examples/TaylorGreenVortex2D.jl b/examples/TaylorGreenVortex2D.jl index 7e0ca8f10..7585ec6a7 100644 --- a/examples/TaylorGreenVortex2D.jl +++ b/examples/TaylorGreenVortex2D.jl @@ -42,7 +42,7 @@ x = LinRange(lims..., n + 1), LinRange(lims..., n + 1) plot_grid(x...) # Build setup and assemble operators -setup = device(Setup(x; Re)); +setup = Setup(x...; Re, ArrayType); # Since the grid is uniform and identical for x and y, we may use a specialized # spectral pressure solver diff --git a/examples/TaylorGreenVortex3D.jl b/examples/TaylorGreenVortex3D.jl index 4f044125f..52359e141 100644 --- a/examples/TaylorGreenVortex3D.jl +++ b/examples/TaylorGreenVortex3D.jl @@ -86,8 +86,8 @@ t_start, t_end = tlims = T(0), T(5) # Solve unsteady problem u, p, outputs = solve_unsteady( setup, - # u₀, p₀, - u, p, + u₀, p₀, + # u, p, tlims; Δt = T(0.01), processors, diff --git a/src/IncompressibleNavierStokes.jl b/src/IncompressibleNavierStokes.jl index 7e549e0e5..c321df15d 100644 --- a/src/IncompressibleNavierStokes.jl +++ b/src/IncompressibleNavierStokes.jl @@ -96,7 +96,7 @@ include("closures/training.jl") include("closures/create_les_data.jl") # Boundary conditions -export PeriodicBC, DirichletBC, SymmetricBC, NeumannBC +export PeriodicBC, DirichletBC, SymmetricBC, PressureBC # Force export SteadyBodyForce @@ -117,13 +117,16 @@ export Setup export stretched_grid, cosine_grid # Pressure solvers -export DirectPressureSolver, CGPressureSolver, SpectralPressureSolver +export DirectPressureSolver, + CGPressureSolver, CGPressureSolverManual, SpectralPressureSolver export pressure_poisson, pressure_poisson!, pressure_additional_solve, pressure_additional_solve! +# Operators +export momentum, divergence, pressuregradient + # Problems export solve_unsteady, solve_steady_state -export momentum, momentum! export create_initial_conditions, random_field, get_velocity diff --git a/src/boundary_conditions.jl b/src/boundary_conditions.jl index 9ede7b56a..5e93bd202 100644 --- a/src/boundary_conditions.jl +++ b/src/boundary_conditions.jl @@ -63,9 +63,11 @@ ghost_b!(::DirichletBC, x) = push!(x, x[end]) ghost_a!(::SymmetricBC, x) = pushfirst!(x, x[1] + (x[2] - x[1])) ghost_b!(::SymmetricBC, x) = push!(x, x[end] + (x[end] - x[end-1])) -# Add infinitely thin boundary volume and then symmetric -ghost_a!(::PressureBC, x) = pushfirst!(x, x[1] - (x[2] - x[1]), x[1]) -ghost_b!(::PressureBC, x) = push!(x, x[end], x[end] + (x[end] - x[end-1])) +# Add infinitely thin boundary volume +# On the left, we need to add two ghost volumes to have a normal component at +# the left of the first ghost volume +ghost_a!(::PressureBC, x) = pushfirst!(x, x[1], x[1]) +ghost_b!(::PressureBC, x) = push!(x, x[end]) """ offset_u(bc, isnormal, atend) @@ -84,16 +86,16 @@ Number of non-DOF pressure components at boundary. function offset_p end offset_u(::PeriodicBC, isnormal, atend) = 1 -offset_p(::PeriodicBC) = 1 +offset_p(::PeriodicBC, atend) = 1 -offset_u(::DirichletBC, isnormal, atend) = 1 -offset_p(::DirichletBC) = 1 +offset_u(::DirichletBC, isnormal, atend) = 1 + isnormal * atend +offset_p(::DirichletBC, atend) = 1 -offset_u(::SymmetricBC, isnormal, atend) = 1 -offset_p(::SymmetricBC) = 1 +offset_u(::SymmetricBC, isnormal, atend) = 1 + isnormal * atend +offset_p(::SymmetricBC, atend) = 1 -offset_u(::PressureBC, isnormal, atend) = isnormal && atend ? 2 : 1 -offset_p(::PressureBC) = 2 +offset_u(::PressureBC, isnormal, atend) = 1 + !isnormal * !atend +offset_p(::PressureBC, atend) = 1 + !atend function apply_bc_u! end function apply_bc_p! end @@ -119,88 +121,107 @@ end function apply_bc_u!(::PeriodicBC, u, β, t, setup; atend, kwargs...) (; grid) = setup - (; dimension, Nu, Iu) = grid + (; dimension, N) = grid D = dimension() δ = Offset{D}() - @kernel function _bc_a!(u, α, β, I0) + @kernel function _bc_a!(u, α, β) I = @index(Global, Cartesian) - I = I + I0 - u[α][I] = u[α][I+Nu[α][β]*δ(β)] + u[α][I] = u[α][I+(N[β]-2)*δ(β)] end - @kernel function _bc_b!(u, α, β, I0) + @kernel function _bc_b!(u, α, β) I = @index(Global, Cartesian) - I = I + I0 - u[α][I+Nu[α][β]*δ(β)] = u[α][I] + u[α][I+(N[β]-1)*δ(β)] = u[α][I+δ(β)] end + ndrange = ntuple(γ -> γ == β ? 1 : N[γ], D) for α = 1:D - I0 = first(Iu[α]) - I0 -= oneunit(I0) - ndrange = (Nu[α][1:β-1]..., 1, Nu[α][β+1:end]...) if atend - _bc_b!(get_backend(u[1]), WORKGROUP)(u, α, β, I0; ndrange) - synchronize(get_backend(u[1])) + _bc_b!(get_backend(u[1]), WORKGROUP)(u, α, β; ndrange) else - _bc_a!(get_backend(u[1]), WORKGROUP)(u, α, β, I0 - δ(β); ndrange) - synchronize(get_backend(u[1])) + _bc_a!(get_backend(u[1]), WORKGROUP)(u, α, β; ndrange) end + synchronize(get_backend(u[1])) end end function apply_bc_p!(::PeriodicBC, p, β, t, setup; atend, kwargs...) (; grid) = setup - (; dimension, Np, Ip) = grid + (; dimension, N) = grid D = dimension() δ = Offset{D}() - @kernel function _bc_a(p, β, I0) + @kernel function _bc_a(p, β) I = @index(Global, Cartesian) - I = I + I0 - p[I] = p[I+Np[β]*δ(β)] + p[I] = p[I+(N[β]-2)*δ(β)] end - @kernel function _bc_b(p, β, I0) + @kernel function _bc_b(p, β) I = @index(Global, Cartesian) - I = I + I0 - p[I+Np[β]*δ(β)] = p[I] + p[I+(N[β]-1)*δ(β)] = p[I+δ(β)] end - I0 = first(Ip) - I0 -= oneunit(I0) - ndrange = (Np[1:β-1]..., 1, Np[β+1:end]...) + ndrange = ntuple(γ -> γ == β ? 1 : N[γ], D) if atend - _bc_b(get_backend(p), WORKGROUP)(p, β, I0; ndrange) - synchronize(get_backend(p)) + _bc_b(get_backend(p), WORKGROUP)(p, β; ndrange) else - _bc_a(get_backend(p), WORKGROUP)(p, β, I0 - δ(β); ndrange) - synchronize(get_backend(p)) + _bc_a(get_backend(p), WORKGROUP)(p, β; ndrange) end + synchronize(get_backend(p)) end +# function apply_bc_u!(bc::DirichletBC, u, β, t, setup; atend, dudt = false, kwargs...) +# (; dimension, Nu, x, xp, Iu) = setup.grid +# D = dimension() +# δ = Offset{D}() +# isnothing(bc.u) && return +# bcfunc = dudt ? bc.dudt : bc.u +# @kernel function _bc_a(u, α, β, I0) +# I = @index(Global, Cartesian) +# I = I + I0 +# u[α][I] = bcfunc[α](ntuple(γ -> γ == α ? x[γ][I[α]+1] : xp[γ][I[γ]], D)..., t) +# end +# @kernel function _bc_b(u, α, β, I0) +# I = @index(Global, Cartesian) +# I = I + I0 +# u[α][I] = bcfunc[α](ntuple(γ -> γ == α ? x[γ][I[α]+1] : xp[γ][I[γ]], D)..., t) +# end +# for α = 1:D +# Xu = (xp[1:β-1]..., x[β], xp[β+1:end]...) +# ndrange = (Nu[α][1:β-1]..., 1, Nu[α][β+1:end]...) +# if atend +# I0 = first(Iu[α]) +# I0 -= oneunit(I0) +# I0 += Nu[α][β] * δ(β) +# _bc_b(get_backend(u[1]), WORKGROUP)(u, α, β, I0; ndrange) +# synchronize(get_backend(u[1])) +# else +# I0 = first(Iu[α]) +# I0 -= oneunit(I0) +# I0 -= δ(β) +# _bc_a(get_backend(u[1]), WORKGROUP)(u, α, β, I0; ndrange) +# synchronize(get_backend(u[1])) +# end +# end +# end + function apply_bc_u!(bc::DirichletBC, u, β, t, setup; atend, dudt = false, kwargs...) - (; dimension, Nu, x, xp) = setup.grid + (; dimension, x, xp, N) = setup.grid D = dimension() δ = Offset{D}() isnothing(bc.u) && return bcfunc = dudt ? bc.dudt : bc.u - @kernel function _bc_a(u, α, β, I0) - I = @index(Global, Cartesian) - I = I + I0 - u[α][I] = bcfunc[α](ntuple(γ -> γ == α ? x[I[α] + 1] : xp[I[γ]], D)) - end - @kernel function _bc_b(u, α, β, I0) - I = @index(Global, Cartesian) - I = I + I0 - u[α][I] = bcfunc[α](ntuple(γ -> γ == α ? x[I[α] + 1] : xp[I[γ]], D)) - end for α = 1:D - Xu = (xp[1:β-1]..., x[β], xp[β+1:end]...) - I0 = first(Iu[α]) - I0 -= oneunit(I0) - ndrange = (Nu[α][1:β-1]..., 1, Nu[α][β+1:end]...) if atend - _bc_b(get_backend(u[1]), WORKGROUP)(u, α, β, I0; ndrange) - synchronize(get_backend(u[1])) + I = CartesianIndices(ntuple(γ -> γ == β ? isnormal ? (N[γ]-1:N[γ]-1) : (N[γ]:N[γ]) : (1:N[γ]), D)) else - _bc_a(get_backend(u[1]), WORKGROUP)(u, α, β, I0; ndrange) - synchronize(get_backend(u[1])) + I = CartesianIndices(ntuple(γ -> γ == β ? (1:1) : (1:N[γ]), D)) end + xI = ntuple( + γ -> reshape( + γ == α ? x[γ][I.indices[α].+1] : xp[γ][I.indices[γ]], + ntuple(Returns(1), γ - 1)..., + :, + ntuple(Returns(1), D - γ)..., + ), + D, + ) + u[α][I] .= bcfunc[α].(xI..., t) end end @@ -209,38 +230,59 @@ function apply_bc_p!(::DirichletBC, p, β, t, setup; atend, kwargs...) end function apply_bc_u!(::SymmetricBC, u, β, t, setup; atend, kwargs...) - error("Not implemented") - (; Nu, x, xp) = setup.grid + (; dimension, N) = setup.grid D = dimension() δ = Offset{D}() - @kernel function _bc_a(u, α, β) - I = @index(Global, Cartesian) - end - @kernel function _bc_b(u, α, β) - I = @index(Global, Cartesian) - end for α = 1:D - for β = 1:D - xΓ = (xp[1:β-1]..., xp[β+1:end]...) + if α != β if atend - _bc_b(get_backend(u[1]), WORKGROUP)(u, α, β, I0) - synchronize(get_backend(u[1])) + I = CartesianIndices(ntuple(γ -> γ == β ? (N[γ]:N[γ]) : (1:N[γ]), D)) + u[α][I] .= u[α][I.-δ(β)] else - _bc_a(get_backend(u[1]), WORKGROUP)(u, α, β, I0) - synchronize(get_backend(u[1])) + I = CartesianIndices(ntuple(γ -> γ == β ? (1:1) : (1:N[γ]), D)) + u[α][I] .= u[α][I.+δ(β)] end end end end function apply_bc_p!(::SymmetricBC, p, β, t, setup; atend, kwargs...) - error("Not implemented") + nothing end function apply_bc_u!(bc::PressureBC, u, β, t, setup; atend, kwargs...) - error("Not implemented") + (; grid) = setup + (; dimension, Nu, Iu) = grid + D = dimension() + δ = Offset{D}() + @kernel function _bc_a!(u, α, β, I0) + I = @index(Global, Cartesian) + I = I + I0 + u[α][I-δ(β)] = u[α][I] + end + @kernel function _bc_b!(u, α, β, I0) + I = @index(Global, Cartesian) + I = I + I0 + u[α][I+δ(β)] = u[α][I] + end + for α = 1:D + if atend + I0 = first(Iu[α]) + (Nu[α][β] - 1) * δ(β) + I0 -= oneunit(I0) + ndrange = (Nu[α][1:β-1]..., 1, Nu[α][β+1:end]...) + _bc_b!(get_backend(u[1]), WORKGROUP)(u, α, β, I0; ndrange) + synchronize(get_backend(u[1])) + else + I0 = first(Iu[α]) + I0 -= oneunit(I0) + ndrange = (Nu[α][1:β-1]..., 1, Nu[α][β+1:end]...) + _bc_a!(get_backend(u[1]), WORKGROUP)(u, α, β, I0; ndrange) + synchronize(get_backend(u[1])) + end + end end function apply_bc_p!(bc::PressureBC, p, β, t, setup; atend, kwargs...) - error("Not implemented") + # p is already zero at boundary + nothing end diff --git a/src/create_initial_conditions.jl b/src/create_initial_conditions.jl index d67441cf0..e05d1c60d 100644 --- a/src/create_initial_conditions.jl +++ b/src/create_initial_conditions.jl @@ -12,7 +12,7 @@ function create_initial_conditions( setup, initial_velocity, t; - pressure_solver = DirectPressureSolver(setup), + pressure_solver = CGPressureSolverManual(setup), ) (; grid) = setup (; dimension, N, Iu, Ip, x, xp) = grid @@ -34,7 +34,7 @@ function create_initial_conditions( # Kinetic energy and momentum of initial velocity field # Iteration 1 corresponds to t₀ = 0 (for unsteady simulations) - maxdiv = maximum(divergence(u, setup)) + maxdiv = maximum(abs, divergence(u, setup)) # TODO: Maybe eps(T)^(3//4) if maxdiv > 1e-12 @@ -43,8 +43,8 @@ function create_initial_conditions( # Make velocity field divergence free f = divergence(u, setup) - Δp = pressure_poisson(pressure_solver, f[Ip][:]) - p[Ip][:] .= Δp + Δp = pressure_poisson(pressure_solver, f) + p .= Δp apply_bc_p!(p, t, setup) G = pressuregradient(p, setup) for α = 1:D @@ -110,13 +110,12 @@ function random_field( backend = get_backend(x[1]) u = ntuple(α -> real.(ifft(create_spectrum(N; A, σ, s, backend))), D) + apply_bc_u!(u, t, setup) M = divergence(u, setup) p = zero(M) # Make velocity field divergence free - Min = view(M, Ip) - pin = view(p, Ip) - pressure_poisson!(pressure_solver, pin, Min) + pressure_poisson!(pressure_solver, p, M) apply_bc_p!(p, t, setup) G = pressuregradient(p, setup) for α = 1:D diff --git a/src/grid/grid.jl b/src/grid/grid.jl index 93a3c304b..04841648d 100644 --- a/src/grid/grid.jl +++ b/src/grid/grid.jl @@ -48,15 +48,15 @@ function Grid(x, boundary_conditions; ArrayType = Array) # Number of p DOFs in each dimension Np = ntuple(D) do α - na = offset_p(boundary_conditions[α][1]) - nb = offset_p(boundary_conditions[α][2]) + na = offset_p(boundary_conditions[α][1], false) + nb = offset_p(boundary_conditions[α][2], true) N[α] - na - nb end # Cartesian index range of pressure DOFs Ip = CartesianIndices(ntuple(D) do α - na = offset_p(boundary_conditions[α][1]) - nb = offset_p(boundary_conditions[α][2]) + na = offset_p(boundary_conditions[α][1], false) + nb = offset_p(boundary_conditions[α][2], true) 1+na:N[α]-nb end) @@ -67,32 +67,57 @@ function Grid(x, boundary_conditions; ArrayType = Array) Δu = ntuple(d -> push!(diff(xp[d]), Δ[d][end] / 2), D) # Reference volume sizes - Ω = KernelAbstractions.ones(get_backend(x[1]), T, N...) + Ω = ones(T, N...) for d = 1:D Ω .*= reshape(Δ[d], ntuple(Returns(1), d - 1)..., :) end # Velocity volume sizes - Ωu = ntuple(α -> KernelAbstractions.ones(get_backend(x[1]), T, N), D) + Ωu = ntuple(α -> ones(T, N), D) for α = 1:D, β = 1:D Ωu[α] .*= reshape((α == β ? Δu : Δ)[β], ntuple(Returns(1), β - 1)..., :) end # Vorticity volume sizes - Ωω = KernelAbstractions.ones(get_backend(x[1]), T, N) + Ωω = ones(T, N) for α = 1:D Ωω .*= reshape(Δu[α], ntuple(Returns(1), α - 1)..., :) end # Velocity volume mid-sections - Γu = ntuple(α -> ntuple(β -> KernelAbstractions.ones(get_backend(x[1]), T, N), D), D) + Γu = ntuple(α -> ntuple(β -> ones(T, N), D), D) for α = 1:D, β = 1:D, γ in ((1:β-1)..., (β+1:D)...) Γu[α][β] .*= reshape(γ == β ? 1 : γ == α ? Δu[γ] : Δ[γ], ntuple(Returns(1), γ - 1)..., :) end # # Velocity points - # Xu = ntuple(α -> KernelAbstractions.ones(get_backend(x[1]), T, N) + # Xu = ntuple(α -> ones(T, N)) + + # Interpolation weights from α-face centers x_I to x_{I + δ(β) / 2} + A = ntuple( + α -> ntuple( + β -> begin + if α == β + # Interpolation from face center to volume center + Aαβ1 = fill(T(1 / 2), N[α]) + Aαβ1[1] = 1 + Aαβ2 = fill(T(1 / 2), N[α]) + Aαβ2[end] = 1 + else + # Interpolation from α-face center to left (1) or right (2) α-face β-edge + ϵ = eps(T) + Aαβ1 = [(xp[β][i] - x[β][i]) / (Δu[β][i-1] + ϵ) for i = 2:N[β]] + Aαβ2 = 1 .- Aαβ1 + pushfirst!(Aαβ1, 1) + push!(Aαβ2, 1) + end + (ArrayType(Aαβ1), ArrayType(Aαβ2)) + end, + D, + ), + D, + ) # Grid quantities (; @@ -111,5 +136,6 @@ function Grid(x, boundary_conditions; ArrayType = Array) # Ωu = ArrayType.(Ωu), # Ωω = ArrayType(Ωω), # Γu = ArrayType.(Γu), + A, ) end diff --git a/src/operators.jl b/src/operators.jl index 35f58631c..70694701d 100644 --- a/src/operators.jl +++ b/src/operators.jl @@ -2,16 +2,16 @@ # for writing kernel loops struct Offset{D} end -(::Offset{D})(i) where {D} = CartesianIndex(ntuple(j -> j == i ? 1 : 0, D)) +(::Offset{D})(α) where {D} = CartesianIndex(ntuple(β -> β == α ? 1 : 0, D)) """ divergence!(M, u, setup) Compute divergence of velocity field (in-place version). """ -function divergence!(M, u, setup) +function divergence!(M, u, setup; ϵ = sqrt(eps(eltype(M)))) (; boundary_conditions, grid) = setup - (; Δ, Np, Ip, Ω) = grid + (; Δ, N, Ip, Ω) = grid D = length(u) δ = Offset{D}() @kernel function _divergence!(M, u, α, I0) @@ -19,13 +19,17 @@ function divergence!(M, u, setup) I = I + I0 # D = length(I) # δ = Offset{D}() - M[I] += Ω[I] / Δ[α][I[α]] * (u[α][I] - u[α][I-δ(α)]) + M[I] += Ω[I] / (Δ[α][I[α]] + ϵ) * (u[α][I] - u[α][I-δ(α)]) end M .= 0 - I0 = first(Ip) + # All volumes have a right velocity + # All volumes have a left velocity except the first one + # Start at second volume + ndrange = N .- 1 + I0 = 2 * oneunit(first(Ip)) I0 -= oneunit(I0) for α = 1:D - _divergence!(get_backend(M), WORKGROUP)(M, u, α, I0; ndrange = Np) + _divergence!(get_backend(M), WORKGROUP)(M, u, α, I0; ndrange) synchronize(get_backend(M)) end M @@ -112,19 +116,19 @@ end Compute convective term. """ function convection!(F, u, setup) - (; boundary_conditions, grid, Re, bodyforce) = setup - (; dimension, Δ, Δu, Nu, Iu) = grid + (; boundary_conditions, grid, Re) = setup + (; dimension, Δ, Δu, Nu, Iu, A) = grid D = dimension() δ = Offset{D}() @kernel function _convection!(F, u, α, β, I0) I = @index(Global, Cartesian) I = I + I0 - Δuαβ = (α == β ? Δu[β] : Δ[β]) - F[α][I] -= - ( - (u[α][I] + u[α][I+δ(β)]) / 2 * (u[β][I] + u[β][I+δ(α)]) / 2 - - (u[α][I-δ(β)] + u[α][I]) / 2 * (u[β][I-δ(β)] + u[β][I-δ(β)+δ(α)]) / 2 - ) / Δuαβ[I[β]] + Δuαβ = α == β ? Δu[β] : Δ[β] + uαβ1 = A[α][β][2][I[β]-1] * u[α][I-δ(β)] + A[α][β][1][I[β]] * u[α][I] + uαβ2 = A[α][β][2][I[β]] * u[α][I] + A[α][β][1][I[β]+1] * u[α][I+δ(β)] + uβα1 = A[β][α][2][I[α]-1] * u[β][I-δ(β)] + A[β][α][1][I[α]+1] * u[β][I-δ(β)+δ(α)] + uβα2 = A[β][α][2][I[α]] * u[β][I] + A[β][α][1][I[α]] * u[β][I+δ(α)] + F[α][I] -= (uαβ2 * uβα2 - uαβ1 * uβα1) / Δuαβ[I[β]] end for α = 1:D I0 = first(Iu[α]) @@ -143,8 +147,9 @@ end Compute diffusive term. """ function diffusion!(F, u, setup; ϵ = eps(eltype(F[1]))) + # ϵ = 1 # Add ϵ in denominator for "infinitely thin" volumes - (; boundary_conditions, grid, Re, bodyforce) = setup + (; boundary_conditions, grid, Re) = setup (; dimension, Δ, Δu, Nu, Iu) = grid D = dimension() δ = Offset{D}() @@ -155,8 +160,8 @@ function diffusion!(F, u, setup; ϵ = eps(eltype(F[1]))) Δuαβ = (α == β ? Δu[β] : Δ[β]) F[α][I] += ν * ( - (u[α][I+δ(β)] - u[α][I]) / ((β == α ? Δ : Δu)[β][I[β]] + ϵ) - - (u[α][I] - u[α][I-δ(β)]) / ((β == α ? Δ : Δu)[β][I[β]-1] + ϵ) + (u[α][I+δ(β)] - u[α][I]) / ((β == α ? Δ[β][I[β]+1] : Δu[β][I[β]]) + ϵ) - + (u[α][I] - u[α][I-δ(β)]) / ((β == α ? Δ[β][I[β]] : Δu[β][I[β]-1]) + ϵ) ) / Δuαβ[I[β]] end for α = 1:D @@ -214,33 +219,48 @@ function momentum!(F, u, t, setup) for α = 1:D F[α] .= 0 end - convection!(F, u, setup) diffusion!(F, u, setup) + convection!(F, u, setup) bodyforce!(F, u, t, setup) isnothing(closure_model) || (F .+= closure_model(u)) F end +""" + momentum(u, t, setup) + +Right hand side of momentum equations, excluding pressure gradient. +""" +momentum(u, t, setup) = momentum!( + ntuple( + α -> KernelAbstractions.zeros(get_backend(u[1]), typeof(t), setup.grid.N), + length(u), + ), + u, + t, + setup, +) + """ pressuregradient!(G, p, setup) Compute pressure gradient (in-place). """ -function pressuregradient!(G, p, setup) +function pressuregradient!(G, p, setup; ϵ = sqrt(eps(eltype(p)))) (; boundary_conditions, grid) = setup - (; dimension, Δu, Np, Iu) = grid + (; dimension, Δu, Nu, Iu) = grid D = dimension() δ = Offset{D}() @kernel function _pressuregradient!(G, p, α, I0) I = @index(Global, Cartesian) I = I0 + I - G[α][I] = (p[I+δ(α)] - p[I]) / Δu[α][I[α]] + G[α][I] = (p[I+δ(α)] - p[I]) / (Δu[α][I[α]] + ϵ) end D = dimension() for α = 1:D I0 = first(Iu[α]) I0 -= oneunit(I0) - _pressuregradient!(get_backend(G[1]), WORKGROUP)(G, p, α, I0; ndrange = Np) + _pressuregradient!(get_backend(G[1]), WORKGROUP)(G, p, α, I0; ndrange = Nu[α]) synchronize(get_backend(G[1])) end G @@ -319,7 +339,7 @@ function interpolate_ω_p!(::Dimension{2}, setup, ωp, ω) end function interpolate_ω_p!(::Dimension{3}, setup, ωp, ω) - (; boundary_conditions, grid, Re, bodyforce) = setup + (; boundary_conditions, grid, Re) = setup (; dimension, Np, Ip) = grid D = dimension() δ = Offset{D}() diff --git a/src/postprocess/plot_velocity.jl b/src/postprocess/plot_velocity.jl index b67e56541..b757e393c 100644 --- a/src/postprocess/plot_velocity.jl +++ b/src/postprocess/plot_velocity.jl @@ -20,7 +20,7 @@ function plot_velocity(::Dimension{2}, setup, u; kwargs...) # Levels μ, σ = mean(qp), std(qp) - # ≈(μ + σ, μ; rtol = 1e-8, atol = 1e-8) && (σ = 1e-4) + ≈(μ + σ, μ; rtol = sqrt(eps(T)), atol = sqrt(eps(T))) && (σ = sqrt(sqrt(eps(T)))) levels = LinRange(μ - T(1.5) * σ, μ + T(1.5) * σ, 10) fig = Figure() diff --git a/src/processors/real_time_plot.jl b/src/processors/real_time_plot.jl index f9a9257ce..c8a826538 100644 --- a/src/processors/real_time_plot.jl +++ b/src/processors/real_time_plot.jl @@ -49,41 +49,20 @@ function field_plot( (; dimension, xlims, x, xp, Ip) = grid D = dimension() - if fieldname == :velocity - xf = xp - elseif fieldname == :vorticity - xf = ntuple(α -> Array(xp[α][Ip.indices[α]]), D) - elseif fieldname == :streamfunction - if boundary_conditions.u.x[1] == :periodic - xf = x - else - xf = x[2:(end-1)] - end - if boundary_conditions.v.y[1] == :periodic - yf = y - else - yf = y[2:(end-1)] - end - elseif fieldname == :pressure - error("Not implemented") - xf = xp - else - error("Unknown fieldname") - end + xf = ntuple(α -> Array(xp[α][Ip.indices[α]]), D) field = @lift begin isnothing(sleeptime) || sleep(sleeptime) (; u, p, t) = $state f = if fieldname == :velocity - up, vp = get_velocity(setup, u, t) - map((u, v) -> √sum(u^2 + v^2), up, vp) + up = interpolate_u_p(setup, u) + map((u, v) -> √sum(u^2 + v^2), up...)[Ip] elseif fieldname == :vorticity interpolate_ω_p(setup, vorticity(u, setup))[Ip] elseif fieldname == :streamfunction - get_streamfunction(setup, u, t) + get_streamfunction(setup, u, t)[Ip] elseif fieldname == :pressure - error("Not implemented") - reshape(p, length(xp), length(yp)) + p[Ip] end Array(f) end diff --git a/src/solvers/pressure/pressure_additional_solve.jl b/src/solvers/pressure/pressure_additional_solve.jl index a6d8aa19f..3808bf00c 100644 --- a/src/solvers/pressure/pressure_additional_solve.jl +++ b/src/solvers/pressure/pressure_additional_solve.jl @@ -4,17 +4,24 @@ Do additional pressure solve. This makes the pressure compatible with the velocity field, resulting in same order pressure as velocity. """ -function pressure_additional_solve!(pressure_solver, u, p, t, setup, F, M) +function pressure_additional_solve!(pressure_solver, u, p, t, setup, F, G, M) (; grid) = setup - (; Ip) = grid + (; dimension, Iu, Ip) = grid + D = dimension() momentum!(F, u, t, setup) + apply_bc_u!(F, t, setup; dudt = true) + + pressuregradient!(G, p, setup) + for α = 1:D + F[α][Iu[α]] .-= G[α][Iu[α]] + end divergence!(M, F, setup) - Min = view(M, Ip) - pin = view(p, Ip) - pressure_poisson!(pressure_solver, pin, Min) + pressure_poisson!(pressure_solver, p, M) + # dp = pressure_poisson(pressure_solver, M) + # p .+= dp apply_bc_p!(p, t, setup) p end @@ -29,6 +36,7 @@ function pressure_additional_solve(pressure_solver, u, t, setup) D = setup.grid.dimension() p = KernelAbstractions.zeros(get_backend(u[1]), eltype(u[1]), setup.grid.N) F = ntuple(α -> KernelAbstractions.zeros(get_backend(u[1]), eltype(u[1]), setup.grid.N), D) + G = ntuple(α -> KernelAbstractions.zeros(get_backend(u[1]), eltype(u[1]), setup.grid.N), D) M = KernelAbstractions.zeros(get_backend(u[1]), eltype(u[1]), setup.grid.N) - pressure_additional_solve!(pressure_solver, u, p, t, setup, F, M) + pressure_additional_solve!(pressure_solver, u, p, t, setup, F, G, M) end diff --git a/src/solvers/pressure/pressure_poisson.jl b/src/solvers/pressure/pressure_poisson.jl index 6add28b5c..1b0089aad 100644 --- a/src/solvers/pressure/pressure_poisson.jl +++ b/src/solvers/pressure/pressure_poisson.jl @@ -10,34 +10,11 @@ See also [`pressure_poisson!`](@ref). """ function pressure_poisson end -function pressure_poisson(solver::DirectPressureSolver, f) - # Assume the Laplace matrix is known (A) and is possibly factorized - - # Use pre-determined decomposition - solver.A_fact \ f -end - -function pressure_poisson(solver::CGPressureSolver, f) - (; A, abstol, reltol, maxiter) = solver - cg(A, f; abstol, reltol, maxiter) -end - -function pressure_poisson(solver::SpectralPressureSolver, f) - (; Ahat) = solver - - f = reshape(f, size(Ahat)) - - # Fourier transform of right hand side - fhat = fft(f) - - # Solve for coefficients in Fourier space - phat = @. -fhat / Ahat - - # Transform back - p = ifft(phat) - - reshape(real.(p), :) -end +pressure_poisson(solver, f) = pressure_poisson!( + solver, + KernelAbstractions.zeros(get_backend(f), typeof(solver.setup.Re), solver.setup.grid.N), + f, +) """ pressure_poisson!(solver, p, f) @@ -68,8 +45,51 @@ function pressure_poisson!(solver::CGPressureSolver, p, f) cg!(p, A, f; abstol, reltol, maxiter) end +function pressure_poisson!(solver::CGPressureSolverManual, p, f) + (; setup, abstol, reltol, maxiter, r, G, M, q) = solver + (; Ip) = setup.grid + T = typeof(reltol) + + # Initial residual + pressuregradient!(G, p, setup) + divergence!(M, G, setup) + + # Intialize + q .= 0 + r .= f .- M + residual = norm(r[Ip]) + prev_residual = one(residual) + tolerance = max(reltol * residual, abstol) + iteration = 0 + + while iteration < maxiter && residual > tolerance + β = residual^2 / prev_residual^2 + q .= r .+ β .* q + + pressuregradient!(G, q, setup) + divergence!(M, G, setup) + α = residual^2 / sum(q[Ip] .* M[Ip]) + + p .+= α .* q + r .-= α .* M + + # Periodic paddding (maybe) + apply_bc_p!(p, T(0), setup) + + prev_residual = residual + residual = norm(r[Ip]) + + iteration += 1 + end + + p +end + function pressure_poisson!(solver::SpectralPressureSolver, p, f) - (; Ahat, fhat, phat) = solver + (; setup, Ahat, fhat, phat) = solver + (; Ip) = setup.grid + + f = @view f[Ip] phat .= complex.(f) @@ -81,7 +101,7 @@ function pressure_poisson!(solver::SpectralPressureSolver, p, f) # Transform back ifft!(phat) - @. p = real(phat) + @. p[Ip] = real(phat) p end diff --git a/src/solvers/pressure/pressure_solvers.jl b/src/solvers/pressure/pressure_solvers.jl index 952191c16..0e55c6194 100644 --- a/src/solvers/pressure/pressure_solvers.jl +++ b/src/solvers/pressure/pressure_solvers.jl @@ -59,7 +59,59 @@ Adapt.adapt_structure(to, s::CGPressureSolver) = CGPressureSolver( adapt(to, s.maxiter), ) -struct SpectralPressureSolver{T,A<:AbstractArray{Complex{T}}} <: AbstractPressureSolver{T} +""" + CGPressureSolverManual(setup; [abstol], [reltol], [maxiter]) + +Conjugate gradients iterative pressure solver. +""" +struct CGPressureSolverManual{T,S,A,AT} <: AbstractPressureSolver{T} + setup::S + abstol::T + reltol::T + maxiter::Int + r::A + G::AT + M::A + q::A +end + +CGPressureSolverManual( + setup; + abstol = zero(eltype(setup.grid.x[1])), + reltol = sqrt(eps(eltype(setup.grid.x[1]))), + maxiter = prod(setup.grid.Np), +) = CGPressureSolverManual( + setup, + abstol, + reltol, + maxiter, + KernelAbstractions.zeros( + get_backend(setup.grid.x[1]), + eltype(setup.grid.x[1]), + setup.grid.N, + ), + ntuple( + α -> KernelAbstractions.zeros( + get_backend(setup.grid.x[1]), + eltype(setup.grid.x[1]), + setup.grid.N, + ), + setup.grid.dimension(), + ), + KernelAbstractions.zeros( + get_backend(setup.grid.x[1]), + eltype(setup.grid.x[1]), + setup.grid.N, + ), + KernelAbstractions.zeros( + get_backend(setup.grid.x[1]), + eltype(setup.grid.x[1]), + setup.grid.N, + ), +) + +struct SpectralPressureSolver{T,A<:AbstractArray{Complex{T}},S} <: AbstractPressureSolver{T} + setup::S Ahat::A phat::A fhat::A @@ -94,11 +146,18 @@ function SpectralPressureSolver(setup) "SpectralPressureSolver requires uniform grid along each dimension", ) - # Fourier transform of the discretization # Assuming uniform grid, although Δx[1] and Δx[2] do not need to be the same - k = ntuple(d -> reshape(0:Np[d]-1, ntuple(Returns(1), d - 1)..., :, ntuple(Returns(1), D - d)...), D) + k = ntuple( + d -> reshape( + 0:Np[d]-1, + ntuple(Returns(1), d - 1)..., + :, + ntuple(Returns(1), D - d)..., + ), + D, + ) Ahat = KernelAbstractions.zeros(get_backend(x[1]), Complex{T}, Np...) Tπ = T(π) # CUDA doesn't like pi @@ -117,5 +176,5 @@ function SpectralPressureSolver(setup) phat = zero(Ahat) fhat = zero(Ahat) - SpectralPressureSolver{T,typeof(Ahat)}(Ahat, phat, fhat) + SpectralPressureSolver{T,typeof(Ahat),typeof(setup)}(setup, Ahat, phat, fhat) end diff --git a/src/solvers/solve_unsteady.jl b/src/solvers/solve_unsteady.jl index 31e3f3c61..e82620368 100644 --- a/src/solvers/solve_unsteady.jl +++ b/src/solvers/solve_unsteady.jl @@ -48,7 +48,7 @@ function solve_unsteady( p₀, tlims; method = RK44(; T = eltype(u₀[1])), - pressure_solver = DirectPressureSolver(setup), + pressure_solver = CGPressureSolverManual(setup), Δt = zero(eltype(u₀[1])), cfl = 1, n_adapt_Δt = 1, diff --git a/src/time_steppers/step_explicit_runge_kutta.jl b/src/time_steppers/step_explicit_runge_kutta.jl index 613baeb31..06798a33d 100644 --- a/src/time_steppers/step_explicit_runge_kutta.jl +++ b/src/time_steppers/step_explicit_runge_kutta.jl @@ -42,7 +42,7 @@ function timestep!(method::ExplicitRungeKuttaMethod, stepper, Δt; cache) for α = 1:D v[α] .= u₀[α] for j = 1:i - @. v[α][Iu[α]] = v[α][Iu[α]] + Δt * A[i, j] * ku[j][α][Iu[α]] + @. v[α][Iu[α]] .+= Δt * A[i, j] * ku[j][α][Iu[α]] end end @@ -54,9 +54,7 @@ function timestep!(method::ExplicitRungeKuttaMethod, stepper, Δt; cache) @. M = M / (c[i] * Δt) # Solve the Poisson equation - Min = view(M, Ip) - pin = view(p, Ip) - pressure_poisson!(pressure_solver, pin, Min) + pressure_poisson!(pressure_solver, p, M) apply_bc_p!(p, t, setup) # Compute pressure correction term @@ -73,7 +71,7 @@ function timestep!(method::ExplicitRungeKuttaMethod, stepper, Δt; cache) t = t₀ + Δt # Do additional pressure solve to avoid first order pressure - p_add_solve && pressure_additional_solve!(pressure_solver, u, p, t, setup, F, M) + p_add_solve && pressure_additional_solve!(pressure_solver, u, p, t, setup, F, G, M) create_stepper(method; setup, pressure_solver, u, p, t, n = n + 1) end diff --git a/src/utils/get_lims.jl b/src/utils/get_lims.jl index 5ba772d8a..8e67c4470 100644 --- a/src/utils/get_lims.jl +++ b/src/utils/get_lims.jl @@ -9,6 +9,6 @@ function get_lims(x, n = 1.5) T = eltype(x) μ = mean(x) σ = std(x) - # ≈(μ + σ, μ; rtol = sqrt(eps(T)), atol = sqrt(eps(T))) && (σ = sqrt(sqrt(eps(T)))) + ≈(μ + σ, μ; rtol = sqrt(eps(T)), atol = sqrt(eps(T))) && (σ = sqrt(sqrt(eps(T)))) (μ - n * σ, μ + n * σ) end From 3b1511503aea311fa9e87e8311e3b52700b24a86 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Tue, 3 Oct 2023 11:26:30 +0200 Subject: [PATCH 074/379] Add missing API --- docs/src/api/api.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/src/api/api.md b/docs/src/api/api.md index 6f9c40850..be7ade4c6 100644 --- a/docs/src/api/api.md +++ b/docs/src/api/api.md @@ -98,6 +98,7 @@ solve_steady_state AbstractPressureSolver DirectPressureSolver CGPressureSolver +CGPressureSolverManual SpectralPressureSolver pressure_additional_solve pressure_additional_solve! @@ -152,4 +153,5 @@ fno offset_u pressuregradient pressuregradient! +momentum ``` From d3abcc1fc6037e8367268f6388409373fa994edd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Wed, 18 Oct 2023 10:35:11 +0200 Subject: [PATCH 075/379] Hide broken examples for now --- docs/make.jl | 48 +++++++++++++------------ docs/src/equations/spatial.md | 4 +-- docs/src/generated/LidDrivenCavity2D.md | 1 + 3 files changed, 29 insertions(+), 24 deletions(-) create mode 100644 docs/src/generated/LidDrivenCavity2D.md diff --git a/docs/make.jl b/docs/make.jl index dee3b0015..a53f96b0f 100644 --- a/docs/make.jl +++ b/docs/make.jl @@ -19,30 +19,33 @@ DocMeta.setdocmeta!( bib = CitationBibliography(joinpath(@__DIR__, "references.bib")) -# Generate examples -examples = [ - "Tutorial: Lid-Driven Cavity (2D)" => "LidDrivenCavity2D", - "Actuator (2D)" => "Actuator2D", - # "Actuator (3D)" => "Actuator3D", - "Backward Facing Step (2D)" => "BackwardFacingStep2D", - # "Backward Facing Step (3D)" => "BackwardFacingStep3D", - "Decaying Turbulunce (2D)" => "DecayingTurbulence2D", - # "Decaying Turbulunce (3D)" => "DecayingTurbulence3D", - # "Lid-Driven Cavity (3D)" => "LidDrivenCavity3D", - "Planar Mixing (2D)" => "PlanarMixing2D", - "Shear Layer (2D)" => "ShearLayer2D", - # "Taylor-Green Vortex (2D)" => "TaylorGreenVortex2D", - # "Taylor-Green Vortex (3D)" => "TaylorGreenVortex3D", -] +# # Generate examples +# examples = [ +# "Tutorial: Lid-Driven Cavity (2D)" => "LidDrivenCavity2D", +# "Actuator (2D)" => "Actuator2D", +# # "Actuator (3D)" => "Actuator3D", +# "Backward Facing Step (2D)" => "BackwardFacingStep2D", +# # "Backward Facing Step (3D)" => "BackwardFacingStep3D", +# "Decaying Turbulunce (2D)" => "DecayingTurbulence2D", +# # "Decaying Turbulunce (3D)" => "DecayingTurbulence3D", +# # "Lid-Driven Cavity (3D)" => "LidDrivenCavity3D", +# "Planar Mixing (2D)" => "PlanarMixing2D", +# "Shear Layer (2D)" => "ShearLayer2D", +# # "Taylor-Green Vortex (2D)" => "TaylorGreenVortex2D", +# # "Taylor-Green Vortex (3D)" => "TaylorGreenVortex3D", +# ] +# +# output = "generated" +# for e ∈ examples +# e = joinpath(@__DIR__, "..", "examples", "$(e.second).jl") +# o = joinpath(@__DIR__, "src", output) +# Literate.markdown(e, o) +# # Literate.notebook(e, o) +# # Literate.script(e, o) +# end output = "generated" -for e ∈ examples - e = joinpath(@__DIR__, "..", "examples", "$(e.second).jl") - o = joinpath(@__DIR__, "src", output) - Literate.markdown(e, o) - # Literate.notebook(e, o) - # Literate.script(e, o) -end +examples = ["Tutorial: Lid-Driven Cavity (2D)" => "LidDrivenCavity2D"] makedocs(; modules = [IncompressibleNavierStokes], @@ -55,6 +58,7 @@ makedocs(; canonical = "https://agdestein.github.io/IncompressibleNavierStokes.jl", assets = String[], ), + pagesonly = true, pages = [ "Home" => "index.md", "Getting Started" => "getting_started.md", diff --git a/docs/src/equations/spatial.md b/docs/src/equations/spatial.md index 96456a202..ee274bfd6 100644 --- a/docs/src/equations/spatial.md +++ b/docs/src/equations/spatial.md @@ -395,7 +395,7 @@ The discrete momentum equations become \begin{split} \frac{\mathrm{d} u_h}{\mathrm{d} t} & = -C(u_h) + \nu (D u_h + y_D) + f_h - (G p_h + y_G) \\ - & = F(V_h) - (G p_h + y_G), + & = F(u_h) - (G p_h + y_G), \end{split} ``` @@ -423,7 +423,7 @@ discrete divergence operator ``M`` to the discrete momentum equations yields the discrete pressure Poisson equation ```math -L p_h = M (F(V_h) - y_G) + \frac{\mathrm{d} y_M}{\mathrm{d} t}, +L p_h = M (F(u_h) - y_G) + \frac{\mathrm{d} y_M}{\mathrm{d} t}, ``` where ``L = M G`` is a discrete Laplace operator. It is positive diff --git a/docs/src/generated/LidDrivenCavity2D.md b/docs/src/generated/LidDrivenCavity2D.md new file mode 100644 index 000000000..5042c9155 --- /dev/null +++ b/docs/src/generated/LidDrivenCavity2D.md @@ -0,0 +1 @@ +# LidDrivenCavity2D From bfddc1fde3b1d1b985fd944ba4a5971037268bbf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Thu, 19 Oct 2023 14:10:57 +0200 Subject: [PATCH 076/379] Make spatial directions static --- src/boundary_conditions.jl | 31 ++++++++++++--------- src/operators.jl | 56 ++++++++++++++++++++------------------ 2 files changed, 48 insertions(+), 39 deletions(-) diff --git a/src/boundary_conditions.jl b/src/boundary_conditions.jl index 5e93bd202..4336db2c7 100644 --- a/src/boundary_conditions.jl +++ b/src/boundary_conditions.jl @@ -124,20 +124,20 @@ function apply_bc_u!(::PeriodicBC, u, β, t, setup; atend, kwargs...) (; dimension, N) = grid D = dimension() δ = Offset{D}() - @kernel function _bc_a!(u, α, β) + @kernel function _bc_a!(u, ::Val{α}, ::Val{β}) where {α,β} I = @index(Global, Cartesian) u[α][I] = u[α][I+(N[β]-2)*δ(β)] end - @kernel function _bc_b!(u, α, β) + @kernel function _bc_b!(u, ::Val{α}, ::Val{β}) where {α,β} I = @index(Global, Cartesian) u[α][I+(N[β]-1)*δ(β)] = u[α][I+δ(β)] end ndrange = ntuple(γ -> γ == β ? 1 : N[γ], D) for α = 1:D if atend - _bc_b!(get_backend(u[1]), WORKGROUP)(u, α, β; ndrange) + _bc_b!(get_backend(u[1]), WORKGROUP)(u, Val(α), Val(β); ndrange) else - _bc_a!(get_backend(u[1]), WORKGROUP)(u, α, β; ndrange) + _bc_a!(get_backend(u[1]), WORKGROUP)(u, Val(α), Val(β); ndrange) end synchronize(get_backend(u[1])) end @@ -148,19 +148,19 @@ function apply_bc_p!(::PeriodicBC, p, β, t, setup; atend, kwargs...) (; dimension, N) = grid D = dimension() δ = Offset{D}() - @kernel function _bc_a(p, β) + @kernel function _bc_a(p, ::Val{β}) where {β} I = @index(Global, Cartesian) p[I] = p[I+(N[β]-2)*δ(β)] end - @kernel function _bc_b(p, β) + @kernel function _bc_b(p, ::Val{β}) where {β} I = @index(Global, Cartesian) p[I+(N[β]-1)*δ(β)] = p[I+δ(β)] end ndrange = ntuple(γ -> γ == β ? 1 : N[γ], D) if atend - _bc_b(get_backend(p), WORKGROUP)(p, β; ndrange) + _bc_b(get_backend(p), WORKGROUP)(p, Val(β); ndrange) else - _bc_a(get_backend(p), WORKGROUP)(p, β; ndrange) + _bc_a(get_backend(p), WORKGROUP)(p, Val(β); ndrange) end synchronize(get_backend(p)) end @@ -208,7 +208,12 @@ function apply_bc_u!(bc::DirichletBC, u, β, t, setup; atend, dudt = false, kwar bcfunc = dudt ? bc.dudt : bc.u for α = 1:D if atend - I = CartesianIndices(ntuple(γ -> γ == β ? isnormal ? (N[γ]-1:N[γ]-1) : (N[γ]:N[γ]) : (1:N[γ]), D)) + I = CartesianIndices( + ntuple( + γ -> γ == β ? isnormal ? (N[γ]-1:N[γ]-1) : (N[γ]:N[γ]) : (1:N[γ]), + D, + ), + ) else I = CartesianIndices(ntuple(γ -> γ == β ? (1:1) : (1:N[γ]), D)) end @@ -255,12 +260,12 @@ function apply_bc_u!(bc::PressureBC, u, β, t, setup; atend, kwargs...) (; dimension, Nu, Iu) = grid D = dimension() δ = Offset{D}() - @kernel function _bc_a!(u, α, β, I0) + @kernel function _bc_a!(u, ::Val{α}, ::Val{β}, I0) where {α,β} I = @index(Global, Cartesian) I = I + I0 u[α][I-δ(β)] = u[α][I] end - @kernel function _bc_b!(u, α, β, I0) + @kernel function _bc_b!(u, ::Val{α}, ::Val{β}, I0) where {α,β} I = @index(Global, Cartesian) I = I + I0 u[α][I+δ(β)] = u[α][I] @@ -270,13 +275,13 @@ function apply_bc_u!(bc::PressureBC, u, β, t, setup; atend, kwargs...) I0 = first(Iu[α]) + (Nu[α][β] - 1) * δ(β) I0 -= oneunit(I0) ndrange = (Nu[α][1:β-1]..., 1, Nu[α][β+1:end]...) - _bc_b!(get_backend(u[1]), WORKGROUP)(u, α, β, I0; ndrange) + _bc_b!(get_backend(u[1]), WORKGROUP)(u, Val(α), Val(β), I0; ndrange) synchronize(get_backend(u[1])) else I0 = first(Iu[α]) I0 -= oneunit(I0) ndrange = (Nu[α][1:β-1]..., 1, Nu[α][β+1:end]...) - _bc_a!(get_backend(u[1]), WORKGROUP)(u, α, β, I0; ndrange) + _bc_a!(get_backend(u[1]), WORKGROUP)(u, Val(α), Val(β), I0; ndrange) synchronize(get_backend(u[1])) end end diff --git a/src/operators.jl b/src/operators.jl index 70694701d..2d0fe87da 100644 --- a/src/operators.jl +++ b/src/operators.jl @@ -14,7 +14,7 @@ function divergence!(M, u, setup; ϵ = sqrt(eps(eltype(M)))) (; Δ, N, Ip, Ω) = grid D = length(u) δ = Offset{D}() - @kernel function _divergence!(M, u, α, I0) + @kernel function _divergence!(M, u, ::Val{α}, I0) where {α} I = @index(Global, Cartesian) I = I + I0 # D = length(I) @@ -29,8 +29,7 @@ function divergence!(M, u, setup; ϵ = sqrt(eps(eltype(M)))) I0 = 2 * oneunit(first(Ip)) I0 -= oneunit(I0) for α = 1:D - _divergence!(get_backend(M), WORKGROUP)(M, u, α, I0; ndrange) - synchronize(get_backend(M)) + _divergence!(get_backend(M), WORKGROUP)(M, u, Val(α), I0; ndrange) end M end @@ -83,7 +82,6 @@ function vorticity!(::Dimension{2}, ω, u, setup) I0 = CartesianIndex(ntuple(Returns(1), D)) I0 -= oneunit(I0) _vorticity!(get_backend(ω), WORKGROUP)(ω, u, I0; ndrange = N .- 1) - synchronize(get_backend(ω)) ω end @@ -92,7 +90,7 @@ function vorticity!(::Dimension{3}, ω, u, setup) (; dimension, Δu, N) = grid D = dimension() δ = Offset{D}() - @kernel function _vorticity!(ω, u, α, I0) + @kernel function _vorticity!(ω, u, ::Val{α}, I0) where {α} T = eltype(ω) I = @index(Global, Cartesian) I = I + I0 @@ -104,8 +102,7 @@ function vorticity!(::Dimension{3}, ω, u, setup) for α = 1:D I0 = CartesianIndex(ntuple(Returns(1), D)) I0 -= oneunit(I0) - _vorticity!(get_backend(ω[1]), WORKGROUP)(ω, u, α, I0; ndrange = N .- 1) - synchronize(get_backend(ω[1])) + _vorticity!(get_backend(ω[1]), WORKGROUP)(ω, u, Val(α), I0; ndrange = N .- 1) end ω end @@ -120,7 +117,7 @@ function convection!(F, u, setup) (; dimension, Δ, Δu, Nu, Iu, A) = grid D = dimension() δ = Offset{D}() - @kernel function _convection!(F, u, α, β, I0) + @kernel function _convection!(F, u, ::Val{α}, ::Val{β}, I0) where {α,β} I = @index(Global, Cartesian) I = I + I0 Δuαβ = α == β ? Δu[β] : Δ[β] @@ -134,8 +131,14 @@ function convection!(F, u, setup) I0 = first(Iu[α]) I0 -= oneunit(I0) for β = 1:D - _convection!(get_backend(F[1]), WORKGROUP)(F, u, α, β, I0; ndrange = Nu[α]) - synchronize(get_backend(F[1])) + _convection!(get_backend(F[1]), WORKGROUP)( + F, + u, + Val(α), + Val(β), + I0; + ndrange = Nu[α], + ) end end F @@ -154,7 +157,7 @@ function diffusion!(F, u, setup; ϵ = eps(eltype(F[1]))) D = dimension() δ = Offset{D}() ν = 1 / Re - @kernel function _diffusion!(F, u, α, β, I0) + @kernel function _diffusion!(F, u, ::Val{α}, ::Val{β}, I0) where {α,β} I = @index(Global, Cartesian) I = I + I0 Δuαβ = (α == β ? Δu[β] : Δ[β]) @@ -168,8 +171,14 @@ function diffusion!(F, u, setup; ϵ = eps(eltype(F[1]))) I0 = first(Iu[α]) I0 -= oneunit(I0) for β = 1:D - _diffusion!(get_backend(F[1]), WORKGROUP)(F, u, α, β, I0; ndrange = Nu[α]) - synchronize(get_backend(F[1])) + _diffusion!(get_backend(F[1]), WORKGROUP)( + F, + u, + Val(α), + Val(β), + I0; + ndrange = Nu[α], + ) end end F @@ -185,7 +194,7 @@ function bodyforce!(F, u, t, setup) (; dimension, Δ, Δu, Nu, Iu, x, xp) = grid D = dimension() δ = Offset{D}() - @kernel function _bodyforce!(F, force, α, t, I0) + @kernel function _bodyforce!(F, force, ::Val{α}, t, I0) where {α} I = @index(Global, Cartesian) I = I + I0 F[α][I] += force[α](ntuple(β -> α == β ? x[β][1+I[β]] : xp[β][I[β]], D)..., t) @@ -196,12 +205,11 @@ function bodyforce!(F, u, t, setup) isnothing(bodyforce) || _bodyforce!(get_backend(F[1]), WORKGROUP)( F, bodyforce, - α, + Val(α), t, I0; ndrange = Nu[α], ) - synchronize(get_backend(F[1])) end F end @@ -251,7 +259,7 @@ function pressuregradient!(G, p, setup; ϵ = sqrt(eps(eltype(p)))) (; dimension, Δu, Nu, Iu) = grid D = dimension() δ = Offset{D}() - @kernel function _pressuregradient!(G, p, α, I0) + @kernel function _pressuregradient!(G, p, ::Val{α}, I0) where {α} I = @index(Global, Cartesian) I = I0 + I G[α][I] = (p[I+δ(α)] - p[I]) / (Δu[α][I[α]] + ϵ) @@ -260,8 +268,7 @@ function pressuregradient!(G, p, setup; ϵ = sqrt(eps(eltype(p)))) for α = 1:D I0 = first(Iu[α]) I0 -= oneunit(I0) - _pressuregradient!(get_backend(G[1]), WORKGROUP)(G, p, α, I0; ndrange = Nu[α]) - synchronize(get_backend(G[1])) + _pressuregradient!(get_backend(G[1]), WORKGROUP)(G, p, Val(α), I0; ndrange = Nu[α]) end G end @@ -294,7 +301,7 @@ function interpolate_u_p!(setup, up, u) (; dimension, Np, Ip) = grid D = dimension() δ = Offset{D}() - @kernel function _interpolate_u_p!(up, u, α, I0) + @kernel function _interpolate_u_p!(up, u, ::Val{α}, I0) where {α} I = @index(Global, Cartesian) I = I + I0 up[α][I] = (u[α][I-δ(α)] + u[α][I]) / 2 @@ -302,8 +309,7 @@ function interpolate_u_p!(setup, up, u) for α = 1:D I0 = first(Ip) I0 -= oneunit(I0) - _interpolate_u_p!(get_backend(up[1]), WORKGROUP)(up, u, α, I0; ndrange = Np) - synchronize(get_backend(up[1])) + _interpolate_u_p!(get_backend(up[1]), WORKGROUP)(up, u, Val(α), I0; ndrange = Np) end up end @@ -334,7 +340,6 @@ function interpolate_ω_p!(::Dimension{2}, setup, ωp, ω) I0 = first(Ip) I0 -= oneunit(I0) _interpolate_ω_p!(get_backend(ωp), WORKGROUP)(ωp, ω, I0; ndrange = Np) - synchronize(get_backend(ωp)) ωp end @@ -343,7 +348,7 @@ function interpolate_ω_p!(::Dimension{3}, setup, ωp, ω) (; dimension, Np, Ip) = grid D = dimension() δ = Offset{D}() - @kernel function _interpolate_ω_p!(ωp, ω, α, I0) + @kernel function _interpolate_ω_p!(ωp, ω, ::Val{α}, I0) where {α} I = @index(Global, Cartesian) I = I + I0 β = mod1(α + 1, D) @@ -353,9 +358,8 @@ function interpolate_ω_p!(::Dimension{3}, setup, ωp, ω) I0 = first(Ip) I0 -= oneunit(I0) for α = 1:D - _interpolate_ω_p!(get_backend(ωp[1]), WORKGROUP)(ωp, ω, α, I0; ndrange = Np) + _interpolate_ω_p!(get_backend(ωp[1]), WORKGROUP)(ωp, ω, Val(α), I0; ndrange = Np) end - synchronize(get_backend(ωp[1])) ωp end From 6a6583d106c06847751de19e8bfa773461958497 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Mon, 23 Oct 2023 14:42:27 +0200 Subject: [PATCH 077/379] Put eps in grid size --- src/grid/grid.jl | 12 ++++++++++-- src/operators.jl | 20 +++++++++----------- 2 files changed, 19 insertions(+), 13 deletions(-) diff --git a/src/grid/grid.jl b/src/grid/grid.jl index 04841648d..20c10c590 100644 --- a/src/grid/grid.jl +++ b/src/grid/grid.jl @@ -63,8 +63,16 @@ function Grid(x, boundary_conditions; ArrayType = Array) xp = ntuple(d -> (x[d][1:end-1] .+ x[d][2:end]) ./ 2, D) # Volume widths - Δ = ntuple(d -> diff(x[d]), D) - Δu = ntuple(d -> push!(diff(xp[d]), Δ[d][end] / 2), D) + Δ = ntuple(D) do d + Δ = diff(x[d]) + Δ[Δ .== 0] .= eps(eltype(Δ)) + Δ + end + Δu = ntuple(D) do d + Δu = push!(diff(xp[d]), Δ[d][end] / 2) + Δu[Δu .== 0] .= eps(eltype(Δu)) + Δu + end # Reference volume sizes Ω = ones(T, N...) diff --git a/src/operators.jl b/src/operators.jl index 2d0fe87da..2798675c7 100644 --- a/src/operators.jl +++ b/src/operators.jl @@ -9,7 +9,7 @@ struct Offset{D} end Compute divergence of velocity field (in-place version). """ -function divergence!(M, u, setup; ϵ = sqrt(eps(eltype(M)))) +function divergence!(M, u, setup) (; boundary_conditions, grid) = setup (; Δ, N, Ip, Ω) = grid D = length(u) @@ -19,7 +19,7 @@ function divergence!(M, u, setup; ϵ = sqrt(eps(eltype(M)))) I = I + I0 # D = length(I) # δ = Offset{D}() - M[I] += Ω[I] / (Δ[α][I[α]] + ϵ) * (u[α][I] - u[α][I-δ(α)]) + M[I] += Ω[I] / Δ[α][I[α]] * (u[α][I] - u[α][I-δ(α)]) end M .= 0 # All volumes have a right velocity @@ -145,13 +145,11 @@ function convection!(F, u, setup) end """ - diffusion!(F, u, setup; ϵ = eps(eltype(F[1]))) + diffusion!(F, u, setup) Compute diffusive term. """ -function diffusion!(F, u, setup; ϵ = eps(eltype(F[1]))) - # ϵ = 1 - # Add ϵ in denominator for "infinitely thin" volumes +function diffusion!(F, u, setup) (; boundary_conditions, grid, Re) = setup (; dimension, Δ, Δu, Nu, Iu) = grid D = dimension() @@ -163,8 +161,8 @@ function diffusion!(F, u, setup; ϵ = eps(eltype(F[1]))) Δuαβ = (α == β ? Δu[β] : Δ[β]) F[α][I] += ν * ( - (u[α][I+δ(β)] - u[α][I]) / ((β == α ? Δ[β][I[β]+1] : Δu[β][I[β]]) + ϵ) - - (u[α][I] - u[α][I-δ(β)]) / ((β == α ? Δ[β][I[β]] : Δu[β][I[β]-1]) + ϵ) + (u[α][I+δ(β)] - u[α][I]) / (β == α ? Δ[β][I[β]+1] : Δu[β][I[β]]) - + (u[α][I] - u[α][I-δ(β)]) / (β == α ? Δ[β][I[β]] : Δu[β][I[β]-1]) ) / Δuαβ[I[β]] end for α = 1:D @@ -185,7 +183,7 @@ function diffusion!(F, u, setup; ϵ = eps(eltype(F[1]))) end """ - bodyforce!(F, u, setup; ϵ = eps(eltype(F[1]))) + bodyforce!(F, u, setup) Compute body force. """ @@ -254,7 +252,7 @@ momentum(u, t, setup) = momentum!( Compute pressure gradient (in-place). """ -function pressuregradient!(G, p, setup; ϵ = sqrt(eps(eltype(p)))) +function pressuregradient!(G, p, setup) (; boundary_conditions, grid) = setup (; dimension, Δu, Nu, Iu) = grid D = dimension() @@ -262,7 +260,7 @@ function pressuregradient!(G, p, setup; ϵ = sqrt(eps(eltype(p)))) @kernel function _pressuregradient!(G, p, ::Val{α}, I0) where {α} I = @index(Global, Cartesian) I = I0 + I - G[α][I] = (p[I+δ(α)] - p[I]) / (Δu[α][I[α]] + ϵ) + G[α][I] = (p[I+δ(α)] - p[I]) / Δu[α][I[α]] end D = dimension() for α = 1:D From 3fe5811926a4ca91a2a7e26135bd672d60aef4ed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Mon, 23 Oct 2023 14:43:58 +0200 Subject: [PATCH 078/379] Reduce copies --- src/solvers/pressure/pressure_additional_solve.jl | 3 ++- src/solvers/solve_unsteady.jl | 4 ++-- src/time_steppers/step_explicit_runge_kutta.jl | 6 ++++-- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/src/solvers/pressure/pressure_additional_solve.jl b/src/solvers/pressure/pressure_additional_solve.jl index 3808bf00c..66c15c5c5 100644 --- a/src/solvers/pressure/pressure_additional_solve.jl +++ b/src/solvers/pressure/pressure_additional_solve.jl @@ -15,7 +15,8 @@ function pressure_additional_solve!(pressure_solver, u, p, t, setup, F, G, M) pressuregradient!(G, p, setup) for α = 1:D - F[α][Iu[α]] .-= G[α][Iu[α]] + F[α] .-= G[α] + # F[α][Iu[α]] .-= G[α][Iu[α]] end divergence!(M, F, setup) diff --git a/src/solvers/solve_unsteady.jl b/src/solvers/solve_unsteady.jl index e82620368..2f3cd2ae1 100644 --- a/src/solvers/solve_unsteady.jl +++ b/src/solvers/solve_unsteady.jl @@ -71,8 +71,8 @@ function solve_unsteady( method; setup, pressure_solver, - u = copy.(u₀), - p = copy(p₀), + u = u₀, + p = p₀, t = t_start, ) diff --git a/src/time_steppers/step_explicit_runge_kutta.jl b/src/time_steppers/step_explicit_runge_kutta.jl index 06798a33d..1fcfe02cf 100644 --- a/src/time_steppers/step_explicit_runge_kutta.jl +++ b/src/time_steppers/step_explicit_runge_kutta.jl @@ -42,7 +42,8 @@ function timestep!(method::ExplicitRungeKuttaMethod, stepper, Δt; cache) for α = 1:D v[α] .= u₀[α] for j = 1:i - @. v[α][Iu[α]] .+= Δt * A[i, j] * ku[j][α][Iu[α]] + @. v[α] += Δt * A[i, j] * ku[j][α] + # @. v[α][Iu[α]] += Δt * A[i, j] * ku[j][α][Iu[α]] end end @@ -62,7 +63,8 @@ function timestep!(method::ExplicitRungeKuttaMethod, stepper, Δt; cache) # Update velocity current stage, which is now divergence free for α = 1:D - @. u[α][Iu[α]] = v[α][Iu[α]] - c[i] * Δt * G[α][Iu[α]] + @. u[α] = v[α] - c[i] * Δt * G[α] + # @. u[α][Iu[α]] = v[α][Iu[α]] - c[i] * Δt * G[α][Iu[α]] end apply_bc_u!(u, t, setup) end From 608b769b26cca4d98698dafec534e6a5b8a082d4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Mon, 23 Oct 2023 14:44:08 +0200 Subject: [PATCH 079/379] fix: Missing semicolon --- src/processors/animator.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/processors/animator.jl b/src/processors/animator.jl index 614690762..de83a3af1 100644 --- a/src/processors/animator.jl +++ b/src/processors/animator.jl @@ -17,7 +17,7 @@ animator(setup, path; nupdate = 1, plotter = field_plotter(setup), kwargs...) = function (state) _state = Observable(state[]) fig = plotter.initialize(_state) - stream = VideoStream(fig, kwargs...) + stream = VideoStream(fig; kwargs...) @lift begin _state[] = $state recordframe!(stream) From beb0129091d3cecf2c094050d73a20a512dbe130 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Mon, 23 Oct 2023 17:16:51 +0200 Subject: [PATCH 080/379] Update README --- README.md | 104 +++++++++++++++++++---------------------- examples/Actuator2D.jl | 72 ++++++++-------------------- 2 files changed, 66 insertions(+), 110 deletions(-) diff --git a/README.md b/README.md index b36f42b8e..c2cc5b3c2 100644 --- a/README.md +++ b/README.md @@ -8,8 +8,10 @@ This package implements energy-conserving solvers for the incompressible Navier-Stokes equations on a staggered Cartesian grid. It is based on the Matlab package -[INS2D](https://github.com/bsanderse/INS2D)/[INS3D](https://github.com/bsanderse/INS3D). +[INS2D](https://github.com/bsanderse/INS2D)/[INS3D](https://github.com/bsanderse/INS3D). The simulations can be run on the single/multithreaded CPUs or Nvidia GPUs. +This package also provides experimental support for neural closure models for +large eddy simulation. ## Installation @@ -36,10 +38,10 @@ The velocity and pressure fields may be visualized in a live session using snapshot files using the `save_vtk` function, or the full time series using the `VTKWriter` processor. -| ![](assets/examples/Actuator2D.png) | ![](assets/examples/BackwardFacingStep2D.png) | ![](assets/examples/DecayingTurbulence2D.png) | ![](assets/examples/TaylorGreenVortex2D.png) | -|:---------------------------------------:|:-------------------------------------------------------------:|:------------------------------------------------------------:|:-----------------------------------------------------------:| +| ![](assets/examples/Actuator2D.png) | ![](assets/examples/BackwardFacingStep2D.png) | ![](assets/examples/DecayingTurbulence2D.png) | ![](assets/examples/TaylorGreenVortex2D.png) | +|:-:|:-:|:-:|:-:| | [Actuator (2D)](examples/Actuator2D.jl) | [Backward Facing Step (2D)](examples/BackwardFacingStep2D.jl) | [Decaying Turbulence (2D)](examples/DecayingTurbulence2D.jl) | [Taylor-Green Vortex (2D)](examples/TaylorGreenVortex2D.jl) | -| ![](assets/examples/Actuator3D.png) | ![](assets/examples/BackwardFacingStep3D.png) | ![](assets/examples/DecayingTurbulence3D.png) | ![](assets/examples/TaylorGreenVortex3D.png) | +| ![](assets/examples/Actuator3D.png) | ![](assets/examples/BackwardFacingStep3D.png) | ![](assets/examples/DecayingTurbulence3D.png) | ![](assets/examples/TaylorGreenVortex3D.png) | | [Actuator (3D)](examples/Actuator3D.jl) | [Backward Facing Step (3D)](examples/BackwardFacingStep3D.jl) | [Decaying Turbulence (3D)](examples/DecayingTurbulence3D.jl) | [Taylor-Green Vortex (3D)](examples/TaylorGreenVortex3D.jl) | @@ -53,23 +55,24 @@ wind conditions. using GLMakie using IncompressibleNavierStokes -# Reynolds number -Re = 100.0 +# A 2D grid is a Cartesian product of two vectors +n = 40 +x = LinRange(0.0, 10.0, 5n + 1) +y = LinRange(-2.0, 2.0, 2n + 1) # Boundary conditions: Unsteady BC requires time derivatives -u_bc(x, y, t) = x ≈ 0.0 ? cos(π / 6 * sin(π / 6 * t)) : 0.0 -v_bc(x, y, t) = x ≈ 0.0 ? sin(π / 6 * sin(π / 6 * t)) : 0.0 -dudt_bc(x, y, t) = x ≈ 0.0 ? -(π / 6)^2 * cos(π / 6 * t) * sin(π / 6 * sin(π / 6 * t)) : 0.0 -dvdt_bc(x, y, t) = x ≈ 0.0 ? (π / 6)^2 * cos(π / 6 * t) * cos(π / 6 * sin(π / 6 * t)) : 0.0 -bc_type = (; - u = (; x = (:dirichlet, :pressure), y = (:symmetric, :symmetric)), - v = (; x = (:dirichlet, :symmetric), y = (:pressure, :pressure)), +U(x, y, t) = cos(π / 6 * sin(π / 6 * t)) +V(x, y, t) = sin(π / 6 * sin(π / 6 * t)) +dUdt(x, y, t) = -(π / 6)^2 * cos(π / 6 * t) * sin(π / 6 * sin(π / 6 * t)) +dVdt(x, y, t) = (π / 6)^2 * cos(π / 6 * t) * cos(π / 6 * sin(π / 6 * t)) +boundary_conditions = ( + # Inlet, outlet + (DirichletBC((U, V), (dUdt, dVdt)), PressureBC()), + + # Sides + (SymmetricBC(), SymmetricBC()), ) -# A 2D grid is a Cartesian product of two vectors -x = LinRange(0.0, 10.0, 200) -y = LinRange(-2.0, 2.0, 80) - # Actuator body force: A thrust coefficient `Cₜ` distributed over a thin rectangle xc, yc = 2.0, 0.0 # Disk center D = 1.0 # Disk diameter @@ -77,56 +80,43 @@ D = 1.0 # Disk diameter Cₜ = 5e-4 # Thrust coefficient cₜ = Cₜ / (D * δ) inside(x, y) = abs(x - xc) ≤ δ / 2 && abs(y - yc) ≤ D / 2 -bodyforce_u(x, y) = -cₜ * inside(x, y) -bodyforce_v(x, y) = 0.0 +fu(x, y, t) = -cₜ * inside(x, y) +fv(x, y, t) = 0.0 # Build setup and assemble operators -setup = Setup( - x, - y; - Re, - u_bc, - v_bc, - dudt_bc, - dvdt_bc, - bc_type, - bodyforce_u, - bodyforce_v, -); - -# Time interval -t_start, t_end = tlims = (0.0, 40.0) +setup = Setup(x, y; Re = 100.0, boundary_conditions, bodyforce = (fu, fv)); # Initial conditions (extend inflow) -initial_velocity_u(x, y) = 1.0 -initial_velocity_v(x, y) = 0.0 -initial_pressure(x, y) = 0.0 -V₀, p₀ = create_initial_conditions( - setup, - initial_velocity_u, - initial_velocity_v, - t_start; - initial_pressure, -); - -# Time step processors -processors = ( - # Record solution every fourth time step - animator(setup, "vorticity.mp4"; nupdate = 4), - - # Log time step information - step_logger(), -) +u₀, p₀ = create_initial_conditions(setup, ((x, y) -> 1.0, (x, y) -> 0.0)); # Solve unsteady Navier-Stokes equations -V, p, outputs = solve_unsteady( - setup, V₀, p₀, tlims; - method = RK44P2(), +u, p, outputs = solve_unsteady( + setup, u₀, p₀, (0.0, 12.0); Δt = 0.05, - processors, + processors = ( + animator(setup, "vorticity.mp4"; nupdate = 4), + step_logger(), + ), ) ``` The resulting animation is shown below. https://github.com/agdestein/IncompressibleNavierStokes.jl/assets/40632532/6ee09a03-1674-46e0-843c-000f0b9b9527 + +## Similar projects + +- [WaterLily.jl](https://github.com/weymouth/WaterLily.jl/) + Incompressible solver with immersed boundaries +- [Oceananigans.jl](https://github.com/CliMA/Oceananigans.jl): + Ocean simulations +- [ClimaCore.jl](https://github.com/CliMA/ClimaCore.jl): + Atmospheric simulations +- [Trixi.jl](https://github.com/trixi-framework/Trixi.jl): + High order solvers for various hyperbolic equations +- [Ferrite.jl](https://github.com/Ferrite-FEM/Ferrite.jl): + Finite element discretizations +- [Gridap.jl](https://github.com/gridap/Gridap.jl): + Finite element discretizations +- [FourierFlows.jl](https://github.com/FourierFlows/FourierFlows.jl): + Pseudo-spectral discretizations diff --git a/examples/Actuator2D.jl b/examples/Actuator2D.jl index 5c89c9022..60144bbaa 100644 --- a/examples/Actuator2D.jl +++ b/examples/Actuator2D.jl @@ -24,19 +24,6 @@ using IncompressibleNavierStokes # Case name for saving results name = "Actuator2D" -# Floating point type -T = Float64 - -# Array type -ArrayType = Array -## using CUDA; ArrayType = CuArray -## using AMDGPU; ArrayType = ROCArray -## using oneAPI; ArrayType = oneArray -## using Metal; ArrayType = MtlArray - -# Reynolds number -Re = T(100) - # A 2D grid is a Cartesian product of two vectors n = 40 x = LinRange(0.0, 10.0, 5n + 1) @@ -47,7 +34,7 @@ plot_grid(x, y) U(x, y, t) = cos(π / 6 * sin(π / 6 * t)) V(x, y, t) = sin(π / 6 * sin(π / 6 * t)) dUdt(x, y, t) = -(π / 6)^2 * cos(π / 6 * t) * sin(π / 6 * sin(π / 6 * t)) -dVdt(x, y, t) = (π / 6)^2 * cos(π / 6 * t) * cos(π / 6 * sin(π / 6 * t)) +dVdt(x, y, t) = (π / 6)^2 * cos(π / 6 * t) * cos(π / 6 * sin(π / 6 * t)) boundary_conditions = ( ## x left, x right (DirichletBC((U, V), (dUdt, dVdt)), PressureBC()), @@ -57,59 +44,38 @@ boundary_conditions = ( ) # Actuator body force: A thrust coefficient `Cₜ` distributed over a thin rectangle -xc, yc = T(2), T(0) # Disk center -D = T(1) # Disk diameter -δ = T(0.11) # Disk thickness -Cₜ = T(5e-4) # Thrust coefficient +xc, yc = 2.0, 0.0 # Disk center +D = 1.0 # Disk diameter +δ = 0.11 # Disk thickness +Cₜ = 5e-4 # Thrust coefficient cₜ = Cₜ / (D * δ) inside(x, y) = abs(x - xc) ≤ δ / 2 && abs(y - yc) ≤ D / 2 fu(x, y, t) = -cₜ * inside(x, y) -fv(x, y, t) = zero(x) +fv(x, y, t) = 0.0 # Build setup and assemble operators -setup = Setup( - x, y; - Re, - boundary_conditions, - bodyforce = (fu, fv), - ArrayType, -); - -# Time interval -t_start, t_end = tlims = T(0), T(12) +setup = Setup(x, y; Re = 100.0, boundary_conditions, bodyforce = (fu, fv)); # Initial conditions (extend inflow) -initial_velocity = ( - (x, y) -> one(x), - (x, y) -> zero(x), -) -u₀, p₀ = create_initial_conditions( - setup, - initial_velocity, - t_start; -); - -# Iteration processors -processors = ( - field_plotter(setup; nupdate = 1), - ## energy_history_plotter(setup; nupdate = 10), - ## energy_spectrum_plotter(setup; nupdate = 10), - ## animator(setup, "vorticity.mkv"; nupdate = 4), - ## vtk_writer(setup; nupdate = 10, dir = "output/$name", filename = "solution"), - ## field_saver(setup; nupdate = 10), - step_logger(; nupdate = 1), -); +u₀, p₀ = create_initial_conditions(setup, ((x, y) -> 1.0, (x, y) -> 0.0)); # Solve unsteady problem u, p, outputs = solve_unsteady( setup, u₀, p₀, - tlims; + (0.0, 12.0); method = RK44P2(), - Δt = T(0.05), - processors, - inplace = true, + Δt = 0.05, + processors = ( + field_plotter(setup; nupdate = 1), + ## energy_history_plotter(setup; nupdate = 10), + ## energy_spectrum_plotter(setup; nupdate = 10), + ## animator(setup, "vorticity.mkv"; nupdate = 4), + ## vtk_writer(setup; nupdate = 10, dir = "output/$name", filename = "solution"), + ## field_saver(setup; nupdate = 10), + step_logger(; nupdate = 1), + ), ); # ## Post-process From 0a680000effb7f68a65d5e0ba1a1f043b1f79e8a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Mon, 23 Oct 2023 19:18:35 +0200 Subject: [PATCH 081/379] Split volumes --- docs/src/equations/spatial.md | 109 +++++++++--------- docs/src/equations/time.md | 108 ++++++++--------- src/create_initial_conditions.jl | 18 +-- src/grid/grid.jl | 4 +- src/operators.jl | 23 ++-- .../pressure/pressure_additional_solve.jl | 3 +- src/solvers/pressure/pressure_poisson.jl | 6 +- .../step_explicit_runge_kutta.jl | 4 +- 8 files changed, 140 insertions(+), 135 deletions(-) diff --git a/docs/src/equations/spatial.md b/docs/src/equations/spatial.md index ee274bfd6..f2a1fb814 100644 --- a/docs/src/equations/spatial.md +++ b/docs/src/equations/spatial.md @@ -168,10 +168,12 @@ This yields the discrete mass equation ```math \sum_{\alpha = 1}^d -\left| \Gamma^\alpha_I \right| \left( u^\alpha_{I + \delta(\alpha) / 2} - -u^\alpha_{I - \delta(\alpha) / 2} \right) = 0. +\frac{u^\alpha_{I + \delta(\alpha) / 2} - +u^\alpha_{I - \delta(\alpha) / 2}}{\Delta^\alpha_I} = 0, ``` +where we have divided by the volume sizes ``\Omega_I``. + !!! note "Approximation error" For the mass equation, the only approximation we have performed is quadrature. No interpolation or finite difference error is present. @@ -267,32 +269,30 @@ Finally, the discrete ``\alpha``-momentum equations are given by ```math \begin{split} - \left| \Omega_{I + \delta(\alpha) / 2} \right| & \frac{\mathrm{d} }{\mathrm{d} t} u^\alpha_{I + \delta(\alpha) / 2} = \\ - - & \sum_{\beta = 1}^d \left| \Gamma^\beta_{I + \delta(\alpha) / 2} \right| - \left( - (u^\alpha - u^\beta)_{I + \delta(\alpha) / 2 + \delta(\beta) / 2} + - & \sum_{\beta = 1}^d + \frac{ + (u^\alpha u^\beta)_{I + \delta(\alpha) / 2 + \delta(\beta) / 2} - - (u^\alpha - u^\beta )_{I + \delta(\alpha) / 2 - \delta(\beta) / 2} - \right) \\ - - & \left| \Gamma^\alpha_{I + \delta(\alpha) / 2} \right| - \left( p_{I + \delta(\alpha)} - p_{I} \right) \\ - + & \nu \sum_{\beta = 1}^d \left| \Gamma^\beta_{I + \delta(\alpha) / 2} - \right| + (u^\alpha u^\beta )_{I + \delta(\alpha) / 2 - \delta(\beta) / 2} + }{\Delta^\beta_{I(\beta) + \delta_{\alpha \beta} / 2}} \\ + - & \frac{p_{I + \delta(\alpha)} - p_{I}}{\Delta^\alpha_{I(\alpha) + 1 / 2}} \\ + + & \nu \sum_{\beta = 1}^d + \frac{1}{\Delta^\beta_{I(\beta)+ \delta_{\alpha \beta} / 2}} \left( \frac{u^\alpha_{I + \delta(\alpha) / 2 + \delta(\beta)} - u^\alpha_{I + - \delta(\alpha) / 2}}{x^\beta_{I(\beta) + 1} - x^\beta_{I(\beta)}} - - \frac{u^\alpha_{I + \delta(\alpha) / 2} - u^\alpha_{I + - \delta(\alpha) / 2 - \delta(\beta)}}{x^\beta_{I(\beta)} - - x^\beta_{I(\beta) - 1}} + \delta(\alpha) / 2}}{\Delta^\beta_{I(\beta) + 1 / 2}} + - \frac{ + u^\alpha_{I + \delta(\alpha) / 2} - + u^\alpha_{I + \delta(\alpha) / 2 - \delta(\beta)} + }{\Delta^\beta_{I(\beta) - 1 / 2}} \right) \\ - + & \left| \Omega_{I + \delta(\alpha) / 2} \right| f^\alpha(x_{I + - \delta(\alpha) / 2}). + + & f^\alpha(x_{I + \delta(\alpha) / 2}), \end{split} ``` +where we have divided each equation by the volume size +``| \Omega_{I + \delta(\alpha) / 2} |``. ## Boundary conditions @@ -401,17 +401,21 @@ The discrete momentum equations become where ``C`` is the convection operator (including boundary contributions), ``D`` is the diffusion operator, ``y_D`` is boundary contribution to the -diffusion term, ``G = \Omega_h^{-1} M^\mathsf{T}`` is the pressure gradient -operator, ``y_G`` contains the boundary contribution of the pressure to the -pressure gradient (only non-zero for pressure boundary conditions), and -``\Omega_h`` is a diagonal matrix containing the velocity volumes. The term -``F`` refers to all the forces except for the pressure gradient. +diffusion term, +``G = W_u^{-1} M^\mathsf{T} W`` is the pressure gradient +operator, +``y_G`` contains the boundary contribution of the pressure to the +pressure gradient (only non-zero for pressure boundary conditions), +``W_u`` is a diagonal matrix containing the velocity volume sizes +``| \Omega_{I + \delta(\alpha) / 2} |``, and ``W`` is a diagonal matrix +containing the reference volume sizes ``| \Omega_I |``. +The term ``F`` refers to all the forces except for the pressure gradient. !!! note "Volume normalization" - All the operators (except for ``M``) have been divided by the velocity - volume sizes ``\Omega_h``. As a result, the operators have the same units - as their continuous counterparts. + All the operators have been divided by the velocity volume sizes. + As a result, the operators have the same units as their + continuous counterparts. ## Discrete pressure Poisson equation @@ -423,11 +427,11 @@ discrete divergence operator ``M`` to the discrete momentum equations yields the discrete pressure Poisson equation ```math -L p_h = M (F(u_h) - y_G) + \frac{\mathrm{d} y_M}{\mathrm{d} t}, +L p_h = W M (F(u_h) - y_G) + W \frac{\mathrm{d} y_M}{\mathrm{d} t}, ``` -where ``L = M G`` is a discrete Laplace operator. It is positive -symmetric since ``G = \Omega_h^{-1} M^\mathsf{T}``. +where ``L = W M G = W M W_u^{-1} M^\mathsf{T} W`` is a discrete Laplace +operator. It is positive symmetric. !!! note "Unsteady Dirichlet boundary conditions" @@ -455,17 +459,17 @@ symmetric since ``G = \Omega_h^{-1} M^\mathsf{T}``. discrete Poisson equation: ```math - p_h = L^{-1} M (F(u_h) - y_G) + L^{-1} \frac{\mathrm{d} y_M}{\mathrm{d} t}. + p_h = L^{-1} W M (F(u_h) - y_G) + L^{-1} W \frac{\mathrm{d} y_M}{\mathrm{d} t}. ``` The momentum equations then become ```math - \frac{\mathrm{d} u_h}{\mathrm{d} t} = (I - G L^{-1} M) - (F(u_h) - y_G) - G L^{-1} \frac{\mathrm{d} y_M}{\mathrm{d} t}. + \frac{\mathrm{d} u_h}{\mathrm{d} t} = (I - G L^{-1} W M) + (F(u_h) - y_G) - G L^{-1} W \frac{\mathrm{d} y_M}{\mathrm{d} t}. ``` - The matrix ``(I - G L^{-1} M)`` is a projector onto the space + The matrix ``(I - G L^{-1} W M)`` is a projector onto the space of discretely divergence free velocities. However, using this formulation would require an efficient way to perform the projection without assembling the operator matrix ``L^{-1}``, which would be very costly. @@ -500,19 +504,16 @@ the vorticity volume ``\Omega_{I + \delta(1) / 2 + \delta(2) / 2}`` gives \end{split}. ``` -Using quadrature, the discrete vorticity in the corner is given by +Using quadrature, and dividing by the vorticity volume +``| \Omega_{I + \delta(1) / 2 + \delta(2) / 2} |``, +the discrete vorticity in the corner is given by ```math -\begin{split} -\left| \Omega_{I + \delta(1) / 2 + \delta(2) / 2} \right| \omega_{I + \delta(1) / 2 + \delta(2) / 2} = -& - \left| \Gamma^2_{I + \delta(1) / 2 + \delta(2) / 2} \right| -(u^1_{I + \delta(1) / 2 + \delta(2)} -- u^1_{I + \delta(1) / 2}) \\ -& + \left| \Gamma^1_{I + \delta(1) / 2 + \delta(2) / 2} \right| -(u^2_{I + \delta(1) + \delta(2) / 2} -- u^2_{I + \delta(2) / 2}) -\end{split}. +- \frac{u^1_{I + \delta(1) / 2 + \delta(2)} - +u^1_{I + \delta(1) / 2}}{\Delta^2_{I(2) + 1 / 2}} ++ \frac{u^2_{I + \delta(1) + \delta(2) / 2} - +u^2_{I + \delta(2) / 2}}{\Delta^1_{I(1) + 1 / 2}}. ``` The 3D vorticity is a vector field ``(\omega^1, \omega^2, \omega^3)``. @@ -534,17 +535,17 @@ through \end{split}. ``` +Using quadrature, and dividing by the vorticity volume +``| \Omega_{I + \delta(\alpha^+) / 2 + \delta(\alpha^-) / 2} |``, +the discrete vorticity around the ``\alpha``-edge is given by + ```math -\begin{split} -\left| \Omega_{I + \delta(\alpha^+) / 2 + \delta(\alpha^-) / 2} \right| \omega_{I + \delta(\alpha^+) / 2 + \delta(\alpha^-) / 2} = -& - \left| \Gamma^{\alpha^-}_{I + \delta(\alpha^+) / 2 + \delta(\alpha^-) / 2} \right| -(u^{\alpha^+}_{I + \delta(\alpha^+) / 2 + \delta(\alpha^-)} -- u^{\alpha^+}_{I + \delta(\alpha^+) / 2}) \\ -& + \left| \Gamma^{\alpha^+}_{I + \delta(\alpha^+) / 2 + \delta(\alpha^-) / 2} \right| -(u^{\alpha^-}_{I + \delta(\alpha^+) + \delta(\alpha^-) / 2} -- u^{\alpha^-}_{I + \delta(\alpha^-) / 2}) -\end{split}. +- \frac{u^{\alpha^+}_{I + \delta(\alpha^+) / 2 + \delta(\alpha^-)} - +u^{\alpha^+}_{I + \delta(\alpha^+) / 2}}{\Delta^{\alpha^-}_{I(\alpha^-) + 1 / 2}} ++ \frac{u^{\alpha^-}_{I + \delta(\alpha^+) + \delta(\alpha^-) / 2} - +u^{\alpha^-}_{I + \delta(\alpha^-) / 2}}{\Delta^{\alpha^+}_{I(\alpha^+) + 1 / +2}}. ``` ## Stream function diff --git a/docs/src/equations/time.md b/docs/src/equations/time.md index ae39eac7c..183ce33c2 100644 --- a/docs/src/equations/time.md +++ b/docs/src/equations/time.md @@ -4,13 +4,13 @@ The spatially discretized Navier-Stokes equations form a differential-algebraic system, with an ODE for the velocity ```math -\frac{\mathrm{d} u_h}{\mathrm{d} t} = F(u_h, t) - (G p_h + y_G) +\frac{\mathrm{d} u}{\mathrm{d} t} = F(u, t) - (G p + y_G) ``` subject to the algebraic constraint formed by the mass equation ```math -M u_h + y_M = 0. +M u + y_M = 0. ``` In the end of the [previous section](spatial.md), we differentiated the mass @@ -18,7 +18,7 @@ equation in time to obtain a discrete pressure Poisson equation. This equation includes the term ``\frac{\mathrm{d} y_M}{\mathrm{d} t}``, which is non-zero if an unsteady flow of mass is added to the domain (Dirichlet boundary conditions). This term ensures that the time-continuous discrete velocity field -``u_h(t)`` stays divergence free (conserves mass). However, if we directly +``u(t)`` stays divergence free (conserves mass). However, if we directly discretize this system in time, the mass preservation may actually not be respected. For this, we will change the definition of the pressure such that the time-discretized velocity field is divergence free at each time step and @@ -27,10 +27,10 @@ each time sub-step (to be defined in the following). Consider the interval ``[0, T]`` for some simulation time ``T``. We will divide it into ``N`` sub-intervals ``[t^n, t^{n + 1}]`` for ``n = 0, \dots, N - 1``, with ``t^0 = 0``, ``t^N = T``, and increment ``\Delta t^n = t^{n + 1} - t^n``. -We define ``U^n \approx u_h(t^n)`` as an approximation to the exact discrete -velocity field ``u_h(t^n)``, with ``U^0 = u_h(0)`` starting from the exact +We define ``u^n \approx u(t^n)`` as an approximation to the exact discrete +velocity field ``u(t^n)``, with ``u^0 = u(0)`` starting from the exact initial conditions. We say that the time integration scheme (definition of -``U^n``) is accurate to the order ``r`` if ``U^n = u_h(t^n) + +``u^n``) is accurate to the order ``r`` if ``u^n = u(t^n) + \mathcal{O}(\Delta t^r)`` for all ``n``. @@ -38,42 +38,42 @@ initial conditions. We say that the time integration scheme (definition of See Sanderse [Sanderse2012](@cite). -Consider the velocity field ``U_0`` at a certain time ``t_0``. We will now +Consider the velocity field ``u_0`` at a certain time ``t_0``. We will now perform one time step to ``t = t_0 + \Delta t``. For explicit Runge-Kutta methods, this time step is divided into ``s`` sub-steps ``t_i = t_0 + \Delta t_i`` with increment ``\Delta t_i = c_i \Delta t``. The final substep performs the full time step ``\Delta t_s = \Delta t`` such that ``t_s = t``. -For ``i = 1, \dots, s``, the intermediate velocity ``U_i`` and pressure ``P_i`` +For ``i = 1, \dots, s``, the intermediate velocity ``u_i`` and pressure ``p_i`` are computed as follows: ```math \begin{split} -F_i & = F(U_{i - 1}, t_{i - 1}) - y_G(t_{i - 1}) \\ -V_i & = U_0 + \Delta t \sum_{j = 1}^i a_{i j} F_j \\ -L P_i & = \frac{1}{c_i} \sum_{j = 1}^i a_{i j} F_j + -\frac{y_M(t_i) - y_M(t_0)}{\Delta t_i} \\ -& = \frac{(M V_i + y_M(t_i)) - (M U_0 + y_M(t_0))}{\Delta t_i^n} \\ -& = \frac{M V_i + y_M(t_i)}{\Delta t_i^n} \\ -U_i & = V_i - \Delta t_i G P_i, +k_i & = F(u_{i - 1}, t_{i - 1}) - y_G(t_{i - 1}) \\ +v_i & = u_0 + \Delta t \sum_{j = 1}^i a_{i j} k_j \\ +L p_i & = W M \frac{1}{c_i} \sum_{j = 1}^i a_{i j} k_j + +W \frac{y_M(t_i) - y_M(t_0)}{\Delta t_i} \\ +& = W \frac{(M v_i + y_M(t_i)) - (M u_0 + y_M(t_0))}{\Delta t_i^n} \\ +& = W \frac{M v_i + y_M(t_i)}{\Delta t_i^n} \\ +u_i & = v_i - \Delta t_i G p_i, \end{split} ``` where ``(a_{i j})_{i j}`` are the Butcher tableau coefficients of the RK-method, with the convention ``c_i = \sum_{j = 1}^i a_{i j}``. -Finally, we set ``U = U_s``. If ``U_0 = u_h(t_0)``, we get the accuracy ``U = -u_h(t) + \mathcal{O}(\Delta t^{r + 1})``, where ``r`` is the order of the +Finally, we return ``u_s``. If ``u_0 = u(t_0)``, we get the accuracy ``u_s = +u(t) + \mathcal{O}(\Delta t^{r + 1})``, where ``r`` is the order of the RK-method. If we perform ``n`` RK time steps instead of one, starting at exact -initial conditions ``U^0 = u_h(0)``, then ``U^n = u_h(t^n) + \mathcal{O}(\Delta -t^r)`` for all ``n \in \{1, \dots, N\}``. Note that for a given ``U``, the -corresponding pressure ``P`` can be calculated to the same accuracy as ``U`` by +initial conditions ``u^0 = u(0)``, then ``u^n = u(t^n) + \mathcal{O}(\Delta +t^r)`` for all ``n \in \{1, \dots, N\}``. Note that for a given ``u``, the +corresponding pressure ``p`` can be calculated to the same accuracy as ``u`` by doing an additional pressure projection after each outer time step ``\Delta t`` (if we know ``\frac{\mathrm{d} y_M}{\mathrm{d} t}(t)``), or to first order -accuracy by simply taking ``P = P_s``. +accuracy by simply returning ``p_s``. -Note that each of the sub-step velocities ``U_i`` is divergence free, after -projecting the tentative velocities ``V_i``. This is ensured due to the +Note that each of the sub-step velocities ``u_i`` is divergence free, after +projecting the tentative velocities ``v_i``. This is ensured due to the judiciously chosen replacement of ``\frac{\mathrm{d} y_M}{\mathrm{d} t}(t_i)`` with ``(y_M(t_i) - y_M(t_0)) / \Delta t_i``. The space-discrete divergence-freeness is thus perfectly preserved, even though the time @@ -89,17 +89,17 @@ See Sanderse [Sanderse2013](@cite). We here require that the time step ``\Delta t`` is constant. This methods uses Adams-Bashforth for the convective terms and Crank-Nicolson stepping for the -diffusion and body force terms. Given the velocity field ``U_0 = u_h(t_0)`` at -a time ``t_0`` and its previous value ``U_{-1} = u_h(t_0 - \Delta t)`` at the -previous time ``t_{-1} = t_0 - \Delta t``, the predicted velocity field ``U`` +diffusion and body force terms. Given the velocity field ``u_0 = u(t_0)`` at +a time ``t_0`` and its previous value ``u_{-1} = u(t_0 - \Delta t)`` at the +previous time ``t_{-1} = t_0 - \Delta t``, the predicted velocity field ``u`` at the time ``t = t_0 + \Delta t`` is defined by first computing a tentative velocity: ```math \begin{split} -\frac{V - U_0}{\Delta t} -& = - (\alpha_0 C(U_0, t_0) + \alpha_{-1} C(U_{-1}, t_{-1})) \\ -& + \theta (D U_0 + y_D(t_0)) + (1 - \theta) (D V + y_D(t)) \\ +\frac{v - u_0}{\Delta t} +& = - (\alpha_0 C(u_0, t_0) + \alpha_{-1} C(u_{-1}, t_{-1})) \\ +& + \theta (D u_0 + y_D(t_0)) + (1 - \theta) (D v + y_D(t)) \\ & + \theta f(t_0) + (1 - \theta) f(t) \\ & - (G p_0 + y_G(t_0)), \end{split} @@ -108,41 +108,41 @@ velocity: where ``\theta \in [0, 1]`` is the Crank-Nicolson parameter (``\theta = \frac{1}{2}`` for second order convergence), ``(\alpha_0, \alpha_{-1}) = \left( \frac{3}{2}, -\frac{1}{2} \right)`` are the Adams-Bashforth coefficients, and -``V`` is a tentative velocity yet to be made divergence free. We can group the -terms containing ``V`` on the left hand side, to obtain +``v`` is a tentative velocity yet to be made divergence free. We can group the +terms containing ``v`` on the left hand side, to obtain ```math \begin{split} -\left( \frac{1}{\Delta t} I - (1 - \theta) D \right) V -& = \left(\frac{1}{\Delta t} I - \theta D \right) U_0 \\ -& - (\alpha_0 C(U_0, t_0) + \alpha_{-1} C(U_{-1}, t_{-1})) \\ +\left( \frac{1}{\Delta t} I - (1 - \theta) D \right) v +& = \left(\frac{1}{\Delta t} I - \theta D \right) u_0 \\ +& - (\alpha_0 C(u_0, t_0) + \alpha_{-1} C(u_{-1}, t_{-1})) \\ & + \theta y_D(t_0) + (1 - \theta) y_D(t) \\ & + \theta f(t_0) + (1 - \theta) f(t) \\ -& - (G P_0 + y_G(t_0)). +& - (G p_0 + y_G(t_0)). \end{split} ``` -We can compute ``V`` by inverting the positive definite matrix ``\left( +We can compute ``v`` by inverting the positive definite matrix ``\left( \frac{1}{\Delta t} I - \theta D \right)`` for the given right hand side using a suitable linear solver. Assuming ``\Delta t`` is constant, we can precompute a Cholesky factorization of this matrix before starting time stepping. -We then compute the pressure difference ``\Delta P`` by solving +We then compute the pressure difference ``\Delta p`` by solving ```math -L \Delta P = \frac{M V + y_M(t)}{\Delta t} - M (y_G(t) - y_G(t_0)), +L \Delta p = W \frac{M v + y_M(t)}{\Delta t} - W M (y_G(t) - y_G(t_0)), ``` -after which a divergence free velocity ``U`` can be enforced: +after which a divergence free velocity ``u`` can be enforced: ```math -U = V - \Delta t (G \Delta P + y_G(t) - y_G(t_0)). +u = v - \Delta t (G \Delta p + y_G(t) - y_G(t_0)). ``` -A first order accurate prediction of the corresponding pressure is ``P = P_0 + -\Delta P``. However, since this pressure is reused in the next time step, we +A first order accurate prediction of the corresponding pressure is ``p = p_0 + +\Delta p``. However, since this pressure is reused in the next time step, we perform an additional pressure solve to avoid accumulating first order errors. -The resulting pressure ``P`` is then accurate to the same order as ``U``. +The resulting pressure ``p`` is then accurate to the same order as ``u``. ## One-leg beta method @@ -150,32 +150,32 @@ The resulting pressure ``P`` is then accurate to the same order as ``U``. See Verstappen and Veldman [Verstappen2003](@cite) [Verstappen1997](@cite). We here require that the time step ``\Delta t`` is constant. Given the velocity -``U_0`` and pressure ``P_0`` at the current time ``t_0`` and their previous -values ``U_{-1}`` and ``P_{-1}`` at the time ``t_{-1} = t_0 - \Delta t``, we -start by computing the "offstep" values ``V = (1 + \beta) V_0 - \beta V_{-1}`` -and ``Q = (1 + \beta) P_0 - \beta P_{-1}`` for some ``\beta = \frac{1}{2}``. +``u_0`` and pressure ``p_0`` at the current time ``t_0`` and their previous +values ``u_{-1}`` and ``p_{-1}`` at the time ``t_{-1} = t_0 - \Delta t``, we +start by computing the "offstep" values ``v = (1 + \beta) v_0 - \beta v_{-1}`` +and ``Q = (1 + \beta) p_0 - \beta p_{-1}`` for some ``\beta = \frac{1}{2}``. -A tentative velocity field ``W`` is then computed as follows: +A tentative velocity field ``\tilde{v}`` is then computed as follows: ```math -W = \frac{1}{\beta + \frac{1}{2}} \left( 2 \beta U_0 - \left( \beta - -\frac{1}{2} \right) U_{-1} + \Delta t F(V, t) - \Delta t +\tilde{v} = \frac{1}{\beta + \frac{1}{2}} \left( 2 \beta u_0 - \left( \beta - +\frac{1}{2} \right) u_{-1} + \Delta t F(v, t) - \Delta t (G Q + y_G(t)) \right). ``` -A pressure correction ``\Delta P `` is obtained by solving the Poisson equation +A pressure correction ``\Delta p `` is obtained by solving the Poisson equation ```math -L \Delta P = \frac{\beta + \frac{1}{2}}{\Delta t} (M W + y_M(t)). +L \Delta p = \frac{\beta + \frac{1}{2}}{\Delta t} W (M \tilde{v} + y_M(t)). ``` Finally, the divergence free velocity field is given by ```math -U = W - \frac{\Delta t}{\beta + \frac{1}{2}} G \Delta P, +u = \tilde{v} - \frac{\Delta t}{\beta + \frac{1}{2}} G \Delta p, ``` while the second order accurate pressure is given by ```math -P = 2 P_0 - P_{-1} + \frac{4}{3} \Delta P. +p = 2 p_0 - p_{-1} + \frac{4}{3} \Delta p. ``` diff --git a/src/create_initial_conditions.jl b/src/create_initial_conditions.jl index e05d1c60d..be7ba187a 100644 --- a/src/create_initial_conditions.jl +++ b/src/create_initial_conditions.jl @@ -2,7 +2,7 @@ create_initial_conditions( setup, initial_velocity, - t; + t = 0; pressure_solver = DirectPressureSolver(setup), ) @@ -11,11 +11,11 @@ Create initial vectors `(u, p)` at starting time `t`. function create_initial_conditions( setup, initial_velocity, - t; + t = convert(eltype(setup.grid.x[1]), 0); pressure_solver = CGPressureSolverManual(setup), ) (; grid) = setup - (; dimension, N, Iu, Ip, x, xp) = grid + (; dimension, N, Iu, Ip, x, xp, Ω) = grid T = eltype(x[1]) D = dimension() @@ -43,6 +43,7 @@ function create_initial_conditions( # Make velocity field divergence free f = divergence(u, setup) + @. f *= Ω Δp = pressure_poisson(pressure_solver, f) p .= Δp apply_bc_p!(p, t, setup) @@ -83,7 +84,7 @@ end """ random_field( setup, t; - A = 1_000_000, + A = setup.grid.N[1] * 10_000, σ = 30, s = 5, pressure_solver = DirectPressureSolver(setup), @@ -98,12 +99,12 @@ Create random field. """ function random_field( setup, t; - A = convert(eltype(setup.grid.x), 1_000_000), - σ = convert(eltype(setup.grid.x), 30), - s = convert(eltype(setup.grid.x), 5), + A = convert(eltype(setup.grid.x[1]), setup.grid.N[1] * 7_500), + σ = convert(eltype(setup.grid.x[1]), 30), + s = convert(eltype(setup.grid.x[1]), 5), pressure_solver = DirectPressureSolver(setup), ) - (; dimension, x, N, Ip) = setup.grid + (; dimension, x, N, Ip, Ω) = setup.grid D = dimension() T = eltype(x[1]) @@ -112,6 +113,7 @@ function random_field( u = ntuple(α -> real.(ifft(create_spectrum(N; A, σ, s, backend))), D) apply_bc_u!(u, t, setup) M = divergence(u, setup) + @. M *= Ω p = zero(M) # Make velocity field divergence free diff --git a/src/grid/grid.jl b/src/grid/grid.jl index 20c10c590..acb983a9a 100644 --- a/src/grid/grid.jl +++ b/src/grid/grid.jl @@ -63,6 +63,7 @@ function Grid(x, boundary_conditions; ArrayType = Array) xp = ntuple(d -> (x[d][1:end-1] .+ x[d][2:end]) ./ 2, D) # Volume widths + # Infinitely thin widths are set to `eps(T)` to avoid division by zero Δ = ntuple(D) do d Δ = diff(x[d]) Δ[Δ .== 0] .= eps(eltype(Δ)) @@ -114,8 +115,7 @@ function Grid(x, boundary_conditions; ArrayType = Array) Aαβ2[end] = 1 else # Interpolation from α-face center to left (1) or right (2) α-face β-edge - ϵ = eps(T) - Aαβ1 = [(xp[β][i] - x[β][i]) / (Δu[β][i-1] + ϵ) for i = 2:N[β]] + Aαβ1 = [(xp[β][i] - x[β][i]) / Δu[β][i-1] for i = 2:N[β]] Aαβ2 = 1 .- Aαβ1 pushfirst!(Aαβ1, 1) push!(Aαβ2, 1) diff --git a/src/operators.jl b/src/operators.jl index 2798675c7..0b91ad9d0 100644 --- a/src/operators.jl +++ b/src/operators.jl @@ -11,15 +11,13 @@ Compute divergence of velocity field (in-place version). """ function divergence!(M, u, setup) (; boundary_conditions, grid) = setup - (; Δ, N, Ip, Ω) = grid + (; Δ, N, Ip) = grid D = length(u) δ = Offset{D}() @kernel function _divergence!(M, u, ::Val{α}, I0) where {α} I = @index(Global, Cartesian) I = I + I0 - # D = length(I) - # δ = Offset{D}() - M[I] += Ω[I] / Δ[α][I[α]] * (u[α][I] - u[α][I-δ(α)]) + M[I] += (u[α][I] - u[α][I-δ(α)]) / Δ[α][I[α]] end M .= 0 # All volumes have a right velocity @@ -94,14 +92,15 @@ function vorticity!(::Dimension{3}, ω, u, setup) T = eltype(ω) I = @index(Global, Cartesian) I = I + I0 - β = mod1(α + 1, D) - γ = mod1(α - 1, D) + α₊ = mod1(α + 1, D) + α₋ = mod1(α - 1, D) ω[α][I] = - -(u[β][I+δ(γ)] - u[β][I]) / Δu[γ][I[γ]] + (u[γ][I+δ(β)] - u[γ][I]) / Δu[β][I[β]] + (u[α₋][I+δ(α₊)] - u[α₋][I]) / Δu[α₊][I[α₊]] - + (u[α₊][I+δ(α₋)] - u[α₊][I]) / Δu[α₋][I[α₋]] end + I0 = CartesianIndex(ntuple(Returns(1), D)) + I0 -= oneunit(I0) for α = 1:D - I0 = CartesianIndex(ntuple(Returns(1), D)) - I0 -= oneunit(I0) _vorticity!(get_backend(ω[1]), WORKGROUP)(ω, u, Val(α), I0; ndrange = N .- 1) end ω @@ -349,9 +348,9 @@ function interpolate_ω_p!(::Dimension{3}, setup, ωp, ω) @kernel function _interpolate_ω_p!(ωp, ω, ::Val{α}, I0) where {α} I = @index(Global, Cartesian) I = I + I0 - β = mod1(α + 1, D) - γ = mod1(α - 1, D) - ωp[α][I] = (ω[α][I-δ(β)-δ(γ)] + ω[α][I]) / 2 + α₊ = mod1(α + 1, D) + α₋ = mod1(α - 1, D) + ωp[α][I] = (ω[α][I-δ(α₊)-δ(α₋)] + ω[α][I]) / 2 end I0 = first(Ip) I0 -= oneunit(I0) diff --git a/src/solvers/pressure/pressure_additional_solve.jl b/src/solvers/pressure/pressure_additional_solve.jl index 66c15c5c5..f6664f1ed 100644 --- a/src/solvers/pressure/pressure_additional_solve.jl +++ b/src/solvers/pressure/pressure_additional_solve.jl @@ -6,7 +6,7 @@ field, resulting in same order pressure as velocity. """ function pressure_additional_solve!(pressure_solver, u, p, t, setup, F, G, M) (; grid) = setup - (; dimension, Iu, Ip) = grid + (; dimension, Iu, Ip, Ω) = grid D = dimension() momentum!(F, u, t, setup) @@ -19,6 +19,7 @@ function pressure_additional_solve!(pressure_solver, u, p, t, setup, F, G, M) # F[α][Iu[α]] .-= G[α][Iu[α]] end divergence!(M, F, setup) + @. M *= Ω pressure_poisson!(pressure_solver, p, M) # dp = pressure_poisson(pressure_solver, M) diff --git a/src/solvers/pressure/pressure_poisson.jl b/src/solvers/pressure/pressure_poisson.jl index 1b0089aad..089ce54eb 100644 --- a/src/solvers/pressure/pressure_poisson.jl +++ b/src/solvers/pressure/pressure_poisson.jl @@ -47,12 +47,13 @@ end function pressure_poisson!(solver::CGPressureSolverManual, p, f) (; setup, abstol, reltol, maxiter, r, G, M, q) = solver - (; Ip) = setup.grid + (; Ip, Ω) = setup.grid T = typeof(reltol) - + # Initial residual pressuregradient!(G, p, setup) divergence!(M, G, setup) + @. M *= Ω # Intialize q .= 0 @@ -68,6 +69,7 @@ function pressure_poisson!(solver::CGPressureSolverManual, p, f) pressuregradient!(G, q, setup) divergence!(M, G, setup) + @. M *= Ω α = residual^2 / sum(q[Ip] .* M[Ip]) p .+= α .* q diff --git a/src/time_steppers/step_explicit_runge_kutta.jl b/src/time_steppers/step_explicit_runge_kutta.jl index 1fcfe02cf..8d2f288af 100644 --- a/src/time_steppers/step_explicit_runge_kutta.jl +++ b/src/time_steppers/step_explicit_runge_kutta.jl @@ -4,7 +4,7 @@ create_stepper(::ExplicitRungeKuttaMethod; setup, pressure_solver, u, p, t, n = function timestep!(method::ExplicitRungeKuttaMethod, stepper, Δt; cache) (; setup, pressure_solver, u, p, t, n) = stepper (; grid, boundary_conditions) = setup - (; dimension, Iu, Ip) = grid + (; dimension, Iu, Ip, Ω) = grid (; A, b, c, p_add_solve) = method (; u₀, ku, v, F, M, G) = cache @@ -52,7 +52,7 @@ function timestep!(method::ExplicitRungeKuttaMethod, stepper, Δt; cache) # Divergence of tentative velocity field divergence!(M, v, setup) - @. M = M / (c[i] * Δt) + @. M *= Ω / (c[i] * Δt) # Solve the Poisson equation pressure_poisson!(pressure_solver, p, M) From eb14b13e7796c5a731c950aa77967043500f1bf6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Mon, 23 Oct 2023 19:36:23 +0200 Subject: [PATCH 082/379] Add plot fields --- docs/references.bib | 26 ++++++- docs/src/api/api.md | 3 + src/IncompressibleNavierStokes.jl | 2 +- src/operators.jl | 105 +++++++++++++++++++++++++- src/postprocess/plot_vorticity.jl | 4 +- src/processors/real_time_plot.jl | 120 +++++++++++++++++++----------- 6 files changed, 211 insertions(+), 49 deletions(-) diff --git a/docs/references.bib b/docs/references.bib index 513805881..0eaf9143b 100644 --- a/docs/references.bib +++ b/docs/references.bib @@ -50,6 +50,18 @@ @article{Harlow1965 URL = {https://aip.scitation.org/doi/abs/10.1063/1.1761178}, eprint = {https://aip.scitation.org/doi/pdf/10.1063/1.1761178}, } +@article{Jeong1995, + doi = {10.1017/s0022112095000462}, + url = {https://doi.org/10.1017%2Fs0022112095000462}, + year = {1995}, + month = {feb}, + publisher = {Cambridge University Press ({CUP})}, + volume = {285}, + pages = {69--94}, + author = {Jinhee Jeong and Fazle Hussain}, + title = {On the identification of a vortex}, + journal = {J. Fluid. Mech.} +} @article{Kochkov2021, author = {Dmitrii Kochkov and Jamie A. Smith and Ayya Alieva and Qing Wang and Michael P. Brenner and Stephan Hoyer}, doi = {10.1073/pnas.2101784118}, @@ -68,10 +80,22 @@ @article{Kurz2022 author = {Kurz, Marius and Offenhäuser, Philipp and Beck, Andrea}, keywords = {Fluid Dynamics (physics.flu-dyn), Artificial Intelligence (cs.AI), Computational Engineering, Finance, and Science (cs.CE), Machine Learning (cs.LG), FOS: Physical sciences, FOS: Physical sciences, FOS: Computer and information sciences, FOS: Computer and information sciences}, title = {Deep Reinforcement Learning for Turbulence Modeling in Large Eddy Simulations}, - publisher = {arXiv}, + publisher = {arXiv:2206.11038}, + journal = {arXiv}, year = {2022}, copyright = {Creative Commons Attribution Non Commercial No Derivatives 4.0 International} } +@article{LiJiajia2019, + Author = {Jiajia Li and Pablo M. Carrica}, + Title = {A simple approach for vortex core visualization}, + journal = {arXiv:1910.06998}, + Eprint = {1910.06998}, + ArchivePrefix = {arXiv}, + PrimaryClass = {physics.flu-dyn}, + Year = {2019}, + Month = {Oct}, + Url = {http://arxiv.org/abs/1910.06998}, +} @article{List2022, author = {List, Björn and Chen, Li-Wei and Thuerey, Nils}, copyright = {arXiv.org perpetual, non-exclusive license}, diff --git a/docs/src/api/api.md b/docs/src/api/api.md index be7ade4c6..a8b14326e 100644 --- a/docs/src/api/api.md +++ b/docs/src/api/api.md @@ -154,4 +154,7 @@ offset_u pressuregradient pressuregradient! momentum +Dfield! +Qfield! +Offset ``` diff --git a/src/IncompressibleNavierStokes.jl b/src/IncompressibleNavierStokes.jl index c321df15d..f516291bf 100644 --- a/src/IncompressibleNavierStokes.jl +++ b/src/IncompressibleNavierStokes.jl @@ -123,7 +123,7 @@ export pressure_poisson, pressure_poisson!, pressure_additional_solve, pressure_additional_solve! # Operators -export momentum, divergence, pressuregradient +export momentum, divergence, pressuregradient, Dfield!, Qfield! # Problems export solve_unsteady, solve_steady_state diff --git a/src/operators.jl b/src/operators.jl index 0b91ad9d0..01b1f0795 100644 --- a/src/operators.jl +++ b/src/operators.jl @@ -1,7 +1,15 @@ -# See https://b-fg.github.io/2023/05/07/waterlily-on-gpu.html -# for writing kernel loops +""" + δ = Offset{D}() +Carsesian index unit vector in `D = 2` or `D = 3` dimensions. +Calling `δ(α)` returns a Cartesian index with `1` in the dimension `α` and zeros +elsewhere. + +See +for writing kernel loops using Cartesian indices. +""" struct Offset{D} end + (::Offset{D})(α) where {D} = CartesianIndex(ntuple(β -> β == α ? 1 : 0, D)) """ @@ -360,6 +368,99 @@ function interpolate_ω_p!(::Dimension{3}, setup, ωp, ω) ωp end +""" + Dfield!(d, G, p, setup; ϵ = eps(eltype(p))) + +Compute the ``D``-field [LiJiajia2019](@cite) given by + +```math +D = \\frac{2 | \\nabla p |}{\\nabla^2 p}. +``` +""" +function Dfield!(d, G, p, setup; ϵ = eps(eltype(p))) + (; boundary_conditions, grid) = setup + (; dimension, Np, Ip, Δ) = grid + T = eltype(p) + D = dimension() + δ = Offset{D}() + @kernel function _Dfield!(d, G, p, I0) + I = @index(Global, Cartesian) + I = I + I0 + g = zero(eltype(p)) + for α = 1:D + g += (G[α][I-δ(α)] + G[α][I])^2 + end + lap = zero(eltype(p)) + # for α = 1:D + # lap += (G[α][I] - G[α][I-δ(α)]) / Δ[α][I[α]] + # end + if D == 2 + lap += (G[1][I] - G[1][I-δ(1)]) / Δ[1][I[1]] + lap += (G[2][I] - G[2][I-δ(2)]) / Δ[2][I[2]] + elseif D == 3 + lap += (G[1][I] - G[1][I-δ(1)]) / Δ[1][I[1]] + lap += (G[2][I] - G[2][I-δ(2)]) / Δ[2][I[2]] + lap += (G[3][I] - G[3][I-δ(3)]) / Δ[3][I[3]] + end + lap = lap > 0 ? max(lap, ϵ) : min(lap, -ϵ) + # lap = abs(lap) + d[I] = sqrt(g) / 2 / lap + end + pressuregradient!(G, p, setup) + I0 = first(Ip) + I0 -= oneunit(I0) + _Dfield!(get_backend(p), WORKGROUP)(d, G, p, I0; ndrange = Np) + d +end + +Dfield(p, setup) = Dfield!( + KernelAbstractions.zeros(get_backend(p), eltype(p), setup.grid.N), + ntuple( + α -> KernelAbstractions.zeros(get_backend(p), eltype(p), setup.grid.N), + setup.grid.dimension(), + ), + p, + setup, +) + +""" + Qfield!(Q, u, setup; ϵ = eps(eltype(Q))) + +Compute ``Q``-field [Jeong1995](@cite) given by + +```math +Q = - \\frac{1}{2} \\sum_{α, β} \\frac{\\partial u^α}{\\partial x^β} +\\frac{\\partial u^β}{\\partial x^α}. +``` +""" +function Qfield!(Q, u, setup; ϵ = eps(eltype(Q))) + (; boundary_conditions, grid) = setup + (; dimension, Np, Ip, Δ) = grid + D = dimension() + δ = Offset{D}() + @kernel function _Qfield!(Q, u, I0) + I = @index(Global, Cartesian) + I = I + I0 + q = zero(eltype(Q)) + for α = 1:D, β = 1:D + q -= + (u[α][I] - u[α][I-δ(β)]) / Δ[β][I[β]] * (u[β][I] - u[β][I-δ(α)]) / + Δ[α][I[α]] / 2 + end + Q[I] = q + end + I0 = first(Ip) + I0 -= oneunit(I0) + _Qfield!(get_backend(u[1]), WORKGROUP)(Q, u, I0; ndrange = Np) + Q +end + +Qfield(u, setup) = Qfield!( + KernelAbstractions.zeros(get_backend(u[1]), eltype(u[1]), setup.grid.N), + u, + setup, +) + """ kinetic_energy(setup, u) diff --git a/src/postprocess/plot_vorticity.jl b/src/postprocess/plot_vorticity.jl index 2cac6b6b0..813c1d0d3 100644 --- a/src/postprocess/plot_vorticity.jl +++ b/src/postprocess/plot_vorticity.jl @@ -33,8 +33,8 @@ function plot_vorticity(::Dimension{2}, setup, u; kwargs...) ylabel = "y", ) limits!(ax, xlims[1]..., xlims[2]...) - cf = contourf!(ax, xp..., ω; extendlow = :auto, extendhigh = :auto, levels, kwargs...) - # cf = heatmap!(ax, xp..., ωp; kwargs...) + # cf = contourf!(ax, xp..., ω; extendlow = :auto, extendhigh = :auto, levels, kwargs...) + cf = heatmap!(ax, Array.(xp)..., Array(ωp); kwargs...) Colorbar(fig[1, 2], cf) # save("output/vorticity.png", fig, pt_per_unit = 2) diff --git a/src/processors/real_time_plot.jl b/src/processors/real_time_plot.jl index c8a826538..1de7761c7 100644 --- a/src/processors/real_time_plot.jl +++ b/src/processors/real_time_plot.jl @@ -44,27 +44,42 @@ function field_plot( sleeptime = 0.001, equal_axis = true, displayfig = true, + docolorbar = true, + resolution = (800, 600), + kwargs..., ) (; boundary_conditions, grid) = setup (; dimension, xlims, x, xp, Ip) = grid D = dimension() - xf = ntuple(α -> Array(xp[α][Ip.indices[α]]), D) + xf = Array.(getindex.(setup.grid.xp, Ip.indices)) + (; u, p, t) = state[] + if fieldname == :velocity + up = interpolate_u_p(setup, u) + elseif fieldname == :vorticity + ω = vorticity(u, setup) + ωp = interpolate_ω_p(setup, ω) + elseif fieldname == :streamfunction + ψ = get_streamfunction(setup, u, t) + elseif fieldname == :pressure + end field = @lift begin isnothing(sleeptime) || sleep(sleeptime) (; u, p, t) = $state f = if fieldname == :velocity - up = interpolate_u_p(setup, u) - map((u, v) -> √sum(u^2 + v^2), up...)[Ip] + interpolate_u_p!(setup, up, u) + map((u, v) -> √sum(u^2 + v^2), up...) elseif fieldname == :vorticity - interpolate_ω_p(setup, vorticity(u, setup))[Ip] + apply_bc_u!(u, t, setup) + vorticity!(ω, u, setup) + interpolate_ω_p!(setup, ωp, ω) elseif fieldname == :streamfunction - get_streamfunction(setup, u, t)[Ip] + get_streamfunction!(setup, ψ, u, t) elseif fieldname == :pressure - p[Ip] + p end - Array(f) + Array(f)[Ip] end lims = @lift begin @@ -86,10 +101,16 @@ function field_plot( lims end - fig = Figure() + fig = Figure(; resolution) if type ∈ (heatmap, image) - ax, hm = type(fig[1, 1], xf..., field; colormap = :viridis, colorrange = lims) + ax, hm = type( + fig[1, 1], + xf..., + field; + colorrange = lims, + kwargs..., + ) elseif type ∈ (contour, contourf) ax, hm = type( fig[1, 1], @@ -99,6 +120,7 @@ function field_plot( extendhigh = :auto, levels = @lift(LinRange($(lims)..., 10)), colorrange = lims, + kwargs..., ) else error("Unknown plot type") @@ -109,7 +131,7 @@ function field_plot( ax.xlabel = "x" ax.ylabel = "y" limits!(ax, xlims[1]..., xlims[2]...) - Colorbar(fig[1, 2], hm) + docolorbar && Colorbar(fig[1, 2], hm) displayfig && display(fig) @@ -120,33 +142,35 @@ function field_plot( ::Dimension{3}, setup, state; - fieldname = :vorticity, + fieldname = :Dfield, sleeptime = 0.001, - alpha = 0.05, + alpha = convert(eltype(setup.grid.x[1]), 0.1), + isorange = convert(eltype(setup.grid.x[1]), 0.5), equal_axis = true, levels = 3, displayfig = true, + docolorbar = true, + resolution = (800, 600), + kwargs..., ) (; boundary_conditions, grid) = setup - (; xlims, x, xp) = grid + (; xlims, x, xp, Ip) = grid + xf = Array.(getindex.(setup.grid.xp, Ip.indices)) if fieldname == :velocity - xf = xp elseif fieldname == :vorticity - xf = xp elseif fieldname == :streamfunction - if boundary_conditions.u.x[1] == :periodic - xf = x - else - xf = x[2:(end-1)] - end - if boundary_conditions.v.y[1] == :periodic - yf = y - else - yf = y[2:(end-1)] - end elseif fieldname == :pressure - xf, yf, zf = xp, yp, zp + elseif fieldname == :Dfield + p = state[].p + d = KernelAbstractions.zeros(get_backend(p), eltype(p), setup.grid.N) + G = ntuple( + α -> KernelAbstractions.zeros(get_backend(p), eltype(p), setup.grid.N), + setup.grid.dimension(), + ) + elseif fieldname == :Qfield + u = state[].u + Q = KernelAbstractions.zeros(get_backend(u[1]), eltype(u[1]), setup.grid.N) else error("Unknown fieldname") end @@ -164,30 +188,41 @@ function field_plot( get_streamfunction(setup, u, t) elseif fieldname == :pressure reshape(copy(p), length(xp), length(yp), length(zp)) + elseif fieldname == :Dfield + Dfield!(d, G, p, setup) + d + elseif fieldname == :Qfield + Qfield!(Q, u, setup) + Q end - Array(f) + Array(f)[Ip] end - lims = @lift get_lims($field) + # lims = @lift get_lims($field) + lims = isnothing(levels) ? @lift(get_lims($field)) : extrema(levels) isnothing(levels) && (levels = @lift(LinRange($(lims)..., 10))) aspect = equal_axis ? (; aspect = :data) : (;) - fig = Figure() - ax = Axis3(fig[1, 1]; title = titlecase(string(fieldname)), aspect...) - hm = contour!( - ax, + fig = Figure(; resolution) + # ax = Axis3(fig[1, 1]; title = titlecase(string(fieldname)), aspect...) + hm = contour( + fig[1,1], + # ax, xf..., field; - # levels, - # colorrange = lims, + levels, + colorrange = lims, + # colorrange = extrema(levels), shading = false, alpha, - highclip = :red, - lowclip = :red, + isorange, + # highclip = :red, + # lowclip = :red, + kwargs..., ) - Colorbar(fig[1, 2], hm) + docolorbar && Colorbar(fig[1, 2], hm) displayfig && display(fig) @@ -220,10 +255,8 @@ end Create energy spectrum plot, redrawn every time `step_observer` is updated. """ -energy_spectrum_plotter(setup; nupdate = 1, kwargs...) = processor( - state -> energy_spectrum_plot(setup, state; kwargs...); - nupdate, -) +energy_spectrum_plotter(setup; nupdate = 1, kwargs...) = + processor(state -> energy_spectrum_plot(setup, state; kwargs...); nupdate) function energy_spectrum_plot(setup, state; displayfig = true) (; dimension, xp, Ip) = setup.grid @@ -242,7 +275,7 @@ function energy_spectrum_plot(setup, state; displayfig = true) (; u, p, t) = $state up = interpolate_u_p(setup, u) e = sum(up -> up[Ip] .^ 2, up) - Array(reshape(abs.(fft(e)[ntuple(α -> kx[α].+1, D)...]), :)) + Array(reshape(abs.(fft(e)[ntuple(α -> kx[α] .+ 1, D)...]), :)) end espec = Figure() ax = Axis(espec[1, 1]; xlabel = "k", ylabel = "e(k)", xscale = log10, yscale = log10) @@ -250,7 +283,8 @@ function energy_spectrum_plot(setup, state; displayfig = true) scatter!(ax, k, ehat; label = "Kinetic energy") krange = LinRange(extrema(k)..., 100) D == 2 && lines!(ax, krange, 1e7 * krange .^ (-3); label = "k⁻³", color = :red) - D == 3 && lines!(ax, krange, 1e6 * krange .^ (-5 / 3); label = "\$k^{-5/3}\$", color = :red) + D == 3 && + lines!(ax, krange, 1e6 * krange .^ (-5 / 3); label = "\$k^{-5/3}\$", color = :red) axislegend(ax) displayfig && display(espec) on(ehat) do _ From 37126b2cd36e976d0115124f2fd3e1e79b711ca0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Mon, 23 Oct 2023 20:10:32 +0200 Subject: [PATCH 083/379] docs: Add neural closures --- docs/make.jl | 1 + docs/src/api/api.md | 6 --- docs/src/features/closure.md | 73 ++++++++++++++++++++++++++++++++++++ src/closures/cnn.jl | 4 +- src/closures/training.jl | 4 +- 5 files changed, 78 insertions(+), 10 deletions(-) create mode 100644 docs/src/features/closure.md diff --git a/docs/make.jl b/docs/make.jl index a53f96b0f..a758d2390 100644 --- a/docs/make.jl +++ b/docs/make.jl @@ -72,6 +72,7 @@ makedocs(; "Floating point precision" => "features/precision.md", "GPU Support" => "features/gpu.md", "Large eddy simulation" => "features/les.md", + "Neural closure models" => "features/closure.md", "Pressure solvers" => "features/pressure.md", "Boundary conditions" => "features/bc.md", "Time steppers" => "features/steppers.md", diff --git a/docs/src/api/api.md b/docs/src/api/api.md index a8b14326e..86c8a4135 100644 --- a/docs/src/api/api.md +++ b/docs/src/api/api.md @@ -139,17 +139,11 @@ plotmat DirichletBC SymmetricBC PressureBC -mean_squared_error -cnn -relative_error -create_randloss kinetic_energy FourierLayer -create_callback offset_p momentum_allstage momentum_allstage! -fno offset_u pressuregradient pressuregradient! diff --git a/docs/src/features/closure.md b/docs/src/features/closure.md new file mode 100644 index 000000000..232b9b63c --- /dev/null +++ b/docs/src/features/closure.md @@ -0,0 +1,73 @@ +# Neural closure models + +For [large eddy simulation (LES)](../features/les.md), a closure model is +required. With IncompressibleNavierStokes, a neural closure model can be traine +d on filtered DNS data. The discrete DNS equations are given by + +```math +\begin{split} +M u & = 0, \\ +\frac{\mathrm{d} u}{\mathrm{d} t} & = F(u) - G p. +\end{split} +``` + +Applying a spatial filter ``\Phi``, the extracted large scale components ``\bar{u} = \Phi u`` are governed by the equation + +```math +\begin{split} +M \bar{u} & = 0, \\ +\frac{\mathrm{d} \bar{u}}{\mathrm{d} t} & = F(\bar{u}) + c - G \bar{p}, +\end{split} +``` +where the discretizations ``M``, ``F``, and ``G`` are adapted to the size of +their inputs and ``c = \overline{F(u)} - F(\bar{u})`` is a commutator error. +Replacing ``c`` with a parameterized closure model ``m(\bar{u}, \theta) \approx +c`` gives the LES equations for the approximate large scale velocity ``\bar{v} +\approx \bar{u}`` + +```math +\begin{split} +M \bar{v} & = 0, \\ +\frac{\mathrm{d} \bar{v}}{\mathrm{d} t} & = F(\bar{v}) + m(\bar{v}, \theta) - G \bar{q}. +\end{split} +``` + +## Training + +To improve the model parameters, we exploit exact filtered DNS data ``\bar{u}`` +and exact commutator errors ``c`` obtained through DNS. The model is trained by +minimizing the a priori loss function + +```math +L^\text{prior}(\theta) = \| m(\bar{u}, \theta) - c \|^2, +``` + +or the a posteriori loss function + +```math +L^\text{post}(\theta) = \| \bar{v}_\theta - \bar{u} \|^2, +``` + +where ``\bar{v}_\theta`` is the solution to the LES equation for the given +parameters ``\theta``. The prior loss is easy to evaluate and easy to +differentiate, as it does not involve solving the ODE. However, minimizing +``L^\text{prior}`` does not take into account the effect of the prediction +error on the LES solution error. The posterior loss does, but has a longer +computational chain involving solving the LES ODE. + +```@docs +train +create_randloss +mean_squared_error +relative_error +create_callback +``` + +## Neural architectures + +We provide two neural architectures: A convolutional neural network (CNN) and a Fourier neural operator (FNO). + +```@docs +cnn +fno +``` diff --git a/src/closures/cnn.jl b/src/closures/cnn.jl index 196e9afd2..a7148a8b9 100644 --- a/src/closures/cnn.jl +++ b/src/closures/cnn.jl @@ -1,8 +1,8 @@ """ cnn(setup, r, c, σ, b; kwargs...) -Create CNN closure model. Return a tuple `(closure, Θ)` where `Θ` are the initial -parameters and `closure(V, Θ)` predicts the commutator error. +Create CNN closure model. Return a tuple `(closure, θ)` where `θ` are the initial +parameters and `closure(V, θ)` predicts the commutator error. """ cnn(setup, r, c, σ, b; kwargs...) = cnn(setup.grid.dimension, setup, r, c, σ, b; kwargs...) diff --git a/src/closures/training.jl b/src/closures/training.jl index b82a3e630..d7cda16a7 100644 --- a/src/closures/training.jl +++ b/src/closures/training.jl @@ -1,11 +1,11 @@ -raw""" +""" train( loss, opt, θ; niter = 100, ncallback = 1, - callback = (i, θ) -> println("Iteration $i of $niter"), + callback = (i, θ) -> println("Iteration \$i of \$niter"), ) Update parameters `θ` to minimize `loss(θ)` using the optimiser `opt` for From 0e6502f4b1b1437088033e9bd1797b3c92c5d44c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Mon, 23 Oct 2023 20:10:43 +0200 Subject: [PATCH 084/379] Disable tests for now --- test/runtests.jl | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/test/runtests.jl b/test/runtests.jl index 6580b9a1d..8424d0077 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -13,14 +13,14 @@ using Statistics using Test @testset "IncompressibleNavierStokes" begin - include("grid.jl") - include("pressure_solvers.jl") - include("models.jl") - include("solvers.jl") - include("simulation2D.jl") - include("simulation3D.jl") - include("postprocess2D.jl") - include("postprocess3D.jl") + # include("grid.jl") + # include("pressure_solvers.jl") + # include("models.jl") + # include("solvers.jl") + # include("simulation2D.jl") + # include("simulation3D.jl") + # include("postprocess2D.jl") + # include("postprocess3D.jl") @testset "Aqua" begin @info "Testing code with Aqua" From e2f12fb2820e2d93ac6e1b8342fe75a650022712 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Mon, 23 Oct 2023 20:16:35 +0200 Subject: [PATCH 085/379] fix: vorticity normalization --- src/operators.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/operators.jl b/src/operators.jl index 01b1f0795..1753586d2 100644 --- a/src/operators.jl +++ b/src/operators.jl @@ -83,7 +83,7 @@ function vorticity!(::Dimension{2}, ω, u, setup) I = @index(Global, Cartesian) I = I + I0 ω[I] = - -Δu[1][I[1]] * (u[1][I+δ(2)] - u[1][I]) + Δu[2][I[2]] * (u[2][I+δ(1)] - u[2][I]) + -(u[1][I+δ(2)] - u[1][I]) / Δu[2][I[2]] + (u[2][I+δ(1)] - u[2][I]) / Δu[1][I[1]] end I0 = CartesianIndex(ntuple(Returns(1), D)) I0 -= oneunit(I0) From 6ad9b47bbb267b1ce20e85bd15200f90ba9dc4c1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Mon, 23 Oct 2023 20:20:08 +0200 Subject: [PATCH 086/379] Simplify setup --- src/operators.jl | 2 +- src/setup.jl | 23 ++++++++++------------- 2 files changed, 11 insertions(+), 14 deletions(-) diff --git a/src/operators.jl b/src/operators.jl index 1753586d2..ae6a463ae 100644 --- a/src/operators.jl +++ b/src/operators.jl @@ -83,7 +83,7 @@ function vorticity!(::Dimension{2}, ω, u, setup) I = @index(Global, Cartesian) I = I + I0 ω[I] = - -(u[1][I+δ(2)] - u[1][I]) / Δu[2][I[2]] + (u[2][I+δ(1)] - u[2][I]) / Δu[1][I[1]] + (u[2][I+δ(1)] - u[2][I]) / Δu[1][I[1]] - (u[1][I+δ(2)] - u[1][I]) / Δu[2][I[2]] end I0 = CartesianIndex(ntuple(Returns(1), D)) I0 -= oneunit(I0) diff --git a/src/setup.jl b/src/setup.jl index fee44032d..ed1ec6fc0 100644 --- a/src/setup.jl +++ b/src/setup.jl @@ -12,7 +12,7 @@ Create setup. """ -function Setup( +Setup( x...; boundary_conditions = ntuple(d -> (PeriodicBC(), PeriodicBC()), length(x)), Re = convert(eltype(x[1]), 1_000), @@ -21,16 +21,13 @@ function Setup( bodyforce = nothing, closure_model = nothing, ArrayType = Array, +) = (; + grid = Grid(x, boundary_conditions; ArrayType), + boundary_conditions, + Re, + viscosity_model, + convection_model, + bodyforce, + closure_model, + ArrayType, ) - grid = Grid(x, boundary_conditions; ArrayType) - (; - grid, - boundary_conditions, - Re, - viscosity_model, - convection_model, - bodyforce, - closure_model, - ArrayType, - ) -end From 345d4daf641f1516bb7d8a181d5552134f4d69e5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Tue, 24 Oct 2023 08:53:49 +0200 Subject: [PATCH 087/379] Update docs --- docs/make.jl | 6 +- docs/src/api/api.md | 77 ------------------- docs/src/equations/time.md | 38 +++++++++ docs/src/features/bc.md | 19 +++-- docs/src/features/closure.md | 1 + docs/src/features/gpu.md | 24 ++---- docs/src/features/les.md | 14 ++-- docs/src/features/pressure.md | 23 +++--- docs/src/features/steppers.md | 9 --- src/IncompressibleNavierStokes.jl | 8 +- src/boundary_conditions.jl | 7 +- src/models/viscosity_models.jl | 4 +- src/operators.jl | 30 ++++++++ .../step_implicit_runge_kutta.jl | 41 +++------- 14 files changed, 135 insertions(+), 166 deletions(-) delete mode 100644 docs/src/features/steppers.md diff --git a/docs/make.jl b/docs/make.jl index a758d2390..e8d5a0573 100644 --- a/docs/make.jl +++ b/docs/make.jl @@ -69,13 +69,13 @@ makedocs(; "Time discretization" => "equations/time.md", ], "Features" => [ + "Boundary conditions" => "features/bc.md", + "Pressure solvers" => "features/pressure.md", "Floating point precision" => "features/precision.md", "GPU Support" => "features/gpu.md", + "Operators" => "features/operators.md", "Large eddy simulation" => "features/les.md", "Neural closure models" => "features/closure.md", - "Pressure solvers" => "features/pressure.md", - "Boundary conditions" => "features/bc.md", - "Time steppers" => "features/steppers.md", ], "API Reference" => ["API" => "api/api.md", "Runge-Kutta methods" => "api/tableaux.md"], diff --git a/docs/src/api/api.md b/docs/src/api/api.md index 86c8a4135..f0ee272f6 100644 --- a/docs/src/api/api.md +++ b/docs/src/api/api.md @@ -20,16 +20,6 @@ max_size stretched_grid ``` -## Visocosity Models - -```@docs -AbstractViscosityModel -LaminarModel -MixingLengthModel -SmagorinskyModel -QRModel -``` - ## Convection Models ```@docs @@ -40,19 +30,6 @@ C4ConvectionModel LerayConvectionModel ``` -## Momentum - -```@docs -divergence -divergence! -vorticity -vorticity! -convection! -diffusion! -bodyforce! -momentum! -``` - ## Postprocess ```@docs @@ -92,63 +69,9 @@ solve_unsteady solve_steady_state ``` -### Pressure solvers - -```@docs -AbstractPressureSolver -DirectPressureSolver -CGPressureSolver -CGPressureSolverManual -SpectralPressureSolver -pressure_additional_solve -pressure_additional_solve! -pressure_poisson -pressure_poisson! -``` - -## Time steppers - -```@docs -AbstractODEMethod -AbstractRungeKuttaMethod -AdamsBashforthCrankNicolsonMethod -OneLegMethod -ExplicitRungeKuttaMethod -ImplicitRungeKuttaMethod - -isexplicit -lambda_conv_max -lambda_diff_max -nstage -ode_method_cache -runge_kutta_method -timestep -timestep! -``` - ## Utils ```@docs get_lims plotmat ``` - -## Other - -```@docs -DirichletBC -SymmetricBC -PressureBC -kinetic_energy -FourierLayer -offset_p -momentum_allstage -momentum_allstage! -offset_u -pressuregradient -pressuregradient! -momentum -Dfield! -Qfield! -Offset -``` diff --git a/docs/src/equations/time.md b/docs/src/equations/time.md index 183ce33c2..a8eef08d8 100644 --- a/docs/src/equations/time.md +++ b/docs/src/equations/time.md @@ -1,3 +1,7 @@ +```@meta +CurrentModule = IncompressibleNavierStokes +``` + # Time discretization The spatially discretized Navier-Stokes equations form a differential-algebraic @@ -34,6 +38,27 @@ initial conditions. We say that the time integration scheme (definition of \mathcal{O}(\Delta t^r)`` for all ``n``. +IncompressibleNavierStokes provides a collection of explicit and implicit +[Runge-Kutta methods](../api/tableaux.md), in addition to Adams-Bashforth +Crank-Nicolson and one-leg beta method time steppers. + +The code is currently not adapted to time steppers from +[DifferentialEquations.jl](https://docs.sciml.ai/DiffEqDocs/stable/solvers/dae_solve/), +but they may be integrated in the future. + +```@docs +AbstractODEMethod +AbstractRungeKuttaMethod +isexplicit +lambda_conv_max +lambda_diff_max +ode_method_cache +nstage +runge_kutta_method +timestep +timestep! +``` + ## Explicit Runge-Kutta methods See Sanderse [Sanderse2012](@cite). @@ -79,11 +104,17 @@ with ``(y_M(t_i) - y_M(t_0)) / \Delta t_i``. The space-discrete divergence-freeness is thus perfectly preserved, even though the time discretization introduces other errors. +```@docs +ExplicitRungeKuttaMethod +``` ## Implicit Runge-Kutta methods See Sanderse [Sanderse2013](@cite). +```@docs +ImplicitRungeKuttaMethod +``` ## Adams-Bashforth Crank-Nicolson method @@ -144,6 +175,9 @@ A first order accurate prediction of the corresponding pressure is ``p = p_0 + perform an additional pressure solve to avoid accumulating first order errors. The resulting pressure ``p`` is then accurate to the same order as ``u``. +```@docs +AdamsBashforthCrankNicolsonMethod +``` ## One-leg beta method @@ -179,3 +213,7 @@ while the second order accurate pressure is given by ```math p = 2 p_0 - p_{-1} + \frac{4}{3} \Delta p. ``` + +```@docs +OneLegMethod +``` diff --git a/docs/src/features/bc.md b/docs/src/features/bc.md index 9ee4f12b5..15e65b990 100644 --- a/docs/src/features/bc.md +++ b/docs/src/features/bc.md @@ -1,8 +1,17 @@ # Boundary conditions -Various boundary conditions are supported. They are indicated by the symbols +Each boundary has exactly one type of boundary conditions. For periodic +boundary conditions, the opposite boundary must also be periodic. +The available boundary conditions are given below. -- `:periodic` -- `:dirichlet` -- `:symmetric` -- `:pressure` +```@docs +PeriodicBC +DirichletBC +SymmetricBC +PressureBC +``` + +```@docs +IncompressibleNavierStokes.offset_p +IncompressibleNavierStokes.offset_u +``` diff --git a/docs/src/features/closure.md b/docs/src/features/closure.md index 232b9b63c..474de01bb 100644 --- a/docs/src/features/closure.md +++ b/docs/src/features/closure.md @@ -70,4 +70,5 @@ We provide two neural architectures: A convolutional neural network (CNN) and a ```@docs cnn fno +FourierLayer ``` diff --git a/docs/src/features/gpu.md b/docs/src/features/gpu.md index d6a586769..5b0dc0ed9 100644 --- a/docs/src/features/gpu.md +++ b/docs/src/features/gpu.md @@ -1,29 +1,15 @@ # GPU Support -If an Nvidia GPU is available, the default CPU solve call - -```julia -solve_unsteady(setup, V₀, p₀, tlims; kwargs...) -``` - -can now be replaced with the following: - -```julia -using CUDA -solve_unsteady( - setup, V₀, p₀, tlims; - device = cu, - kwargs... -) -``` - -This moves the arrays and sparse operators to the GPU, outsourcing all array operations to the GPU. +IncompressibleNavierStokes supports various array types. The desired array type +only has to be passed to the [`Setup`](@ref) function. All operators have been +made are backend agnostic by using +[KernelAbstractions.jl](https://github.com/JuliaGPU/KernelAbstractions.jl/). +Even if a GPU is not available, the operators are multithreaded if Julia is started with multiple threads (e.g. `julia -t 4`) Limitations: - [`DirectPressureSolver`](@ref) is currently not supported on the GPU. Use [`CGPressureSolver`](@ref) instead. - Unsteady boundary conditions are currently not supported on the GPU. -- The code uses sparse matrices for discretization. For finer grids, these can take up a lot of memory on the GPU. - This has not been tested with other GPU interfaces, such as - [AMDGPU.jl](https://github.com/JuliaGPU/AMDGPU.jl) - [Metal.jl](https://github.com/JuliaGPU/Metal.jl) diff --git a/docs/src/features/les.md b/docs/src/features/les.md index 952b22cc0..5495ef225 100644 --- a/docs/src/features/les.md +++ b/docs/src/features/les.md @@ -30,10 +30,10 @@ computed from the local velocity field. The following eddy viscosity models are available: -- [`SmagorinskyModel`](@ref) -- [`QRModel`](@ref) -- [`MixingLengthModel`](@ref) - -In addition, the default [`LaminarModel`](@ref) assumes that there are no -sub-grid stresses. It can be used if the grid is sufficiently refined for the -given flow. It has the advantage of having a constant diffusion operator. +```@docs +AbstractViscosityModel +LaminarModel +SmagorinskyModel +QRModel +MixingLengthModel +``` diff --git a/docs/src/features/pressure.md b/docs/src/features/pressure.md index 80453f5d9..44b1e5bdb 100644 --- a/docs/src/features/pressure.md +++ b/docs/src/features/pressure.md @@ -2,14 +2,19 @@ The discrete pressure Poisson equation ```math -A p_h = g +L p = W M F(u) ``` -enforces divergence freeness. There are three options for solving this system: +enforces divergence freeness. There are multiple options for solving this +system. -- [`DirectPressureSolver`](@ref) factorizes the Laplace matrix ``A`` such that - the system can be solved for different right hand sides. This currently - only works for double precision on the CPU. -- [`CGPressureSolver`](@ref) uses conjugate gradients to solve the system for - different ``f``. -- [`SpectralPressureSolver`](@ref) solves the system in Fourier space, but only - in the case of a uniform grid with periodic boundary conditions. +```@docs +AbstractPressureSolver +DirectPressureSolver +CGPressureSolver +CGPressureSolverManual +SpectralPressureSolver +pressure_additional_solve +pressure_additional_solve! +pressure_poisson +pressure_poisson! +``` diff --git a/docs/src/features/steppers.md b/docs/src/features/steppers.md deleted file mode 100644 index d9d322fc4..000000000 --- a/docs/src/features/steppers.md +++ /dev/null @@ -1,9 +0,0 @@ -# Time steppers - -IncompressibleNavierStokes provides a collection of explicit and implicit -[Runge-Kutta methods](../api/tableaux.md), in addition to Adams-Bashforth -Crank-Nicolson and one-leg beta method time steppers. - -The code is currently not adapted to time steppers from -[DifferentialEquations.jl](https://docs.sciml.ai/DiffEqDocs/stable/solvers/dae_solve/), -but they may be integrated in the future. diff --git a/src/IncompressibleNavierStokes.jl b/src/IncompressibleNavierStokes.jl index f516291bf..449ac3c72 100644 --- a/src/IncompressibleNavierStokes.jl +++ b/src/IncompressibleNavierStokes.jl @@ -102,7 +102,7 @@ export PeriodicBC, DirichletBC, SymmetricBC, PressureBC export SteadyBodyForce # Models -export LaminarModel, MixingLengthModel, SmagorinskyModel, QRModel +export AbstractViscosityModel, LaminarModel, MixingLengthModel, SmagorinskyModel, QRModel export NoRegConvectionModel, C2ConvectionModel, C4ConvectionModel, LerayConvectionModel # Processors @@ -117,8 +117,8 @@ export Setup export stretched_grid, cosine_grid # Pressure solvers -export DirectPressureSolver, - CGPressureSolver, CGPressureSolverManual, SpectralPressureSolver +export AbstractPressureSolver, + DirectPressureSolver, CGPressureSolver, CGPressureSolverManual, SpectralPressureSolver export pressure_poisson, pressure_poisson!, pressure_additional_solve, pressure_additional_solve! @@ -135,7 +135,7 @@ export plot_force, export plotmat # Closure models -export cnn, fno +export cnn, fno, FourierLayer export train export mean_squared_error, relative_error export create_randloss, create_callback, create_les_data diff --git a/src/boundary_conditions.jl b/src/boundary_conditions.jl index 4336db2c7..e92efb113 100644 --- a/src/boundary_conditions.jl +++ b/src/boundary_conditions.jl @@ -1,11 +1,16 @@ abstract type AbstractBC end +""" + PeriodicBC() + +Periodic boundary conditions. Must be periodic on both sides. +""" struct PeriodicBC <: AbstractBC end """ DirichletBC() -No split boundary conditions, where all velocity components are zero. +No slip boundary conditions, where all velocity components are zero. DirichletBC(u, dudt) diff --git a/src/models/viscosity_models.jl b/src/models/viscosity_models.jl index 10fe03d56..17c0db8f3 100644 --- a/src/models/viscosity_models.jl +++ b/src/models/viscosity_models.jl @@ -8,7 +8,9 @@ abstract type AbstractViscosityModel end """ LaminarModel() -Laminar model. +Laminar model. This model assumes that there are no +sub-grid stresses. It can be used if the grid is sufficiently refined for the +given flow. It has the advantage of having a constant diffusion operator. """ struct LaminarModel <: AbstractViscosityModel end diff --git a/src/operators.jl b/src/operators.jl index ae6a463ae..d3ba23ae1 100644 --- a/src/operators.jl +++ b/src/operators.jl @@ -292,6 +292,11 @@ pressuregradient(p, setup) = pressuregradient!( setup, ) +""" + interpolate_u_p(setup, u) + +Interpolate velocity to pressure points. +""" interpolate_u_p(setup, u) = interpolate_u_p!( setup, ntuple( @@ -301,6 +306,11 @@ interpolate_u_p(setup, u) = interpolate_u_p!( u, ) +""" + interpolate_u_p!(setup, up, u) + +Interpolate velocity to pressure points. +""" function interpolate_u_p!(setup, up, u) (; boundary_conditions, grid, Re, bodyforce) = setup (; dimension, Np, Ip) = grid @@ -319,6 +329,11 @@ function interpolate_u_p!(setup, up, u) up end +""" + interpolate_ω_p(setup, ω) + +Interpolate vorticity to pressure points. +""" interpolate_ω_p(setup, ω) = interpolate_ω_p!( setup, setup.grid.dimension() == 2 ? @@ -330,6 +345,11 @@ interpolate_ω_p(setup, ω) = interpolate_ω_p!( ω, ) +""" + interpolate_ω_p!(setup, ωp, ω) + +Interpolate vorticity to pressure points. +""" interpolate_ω_p!(setup, ωp, ω) = interpolate_ω_p!(setup.grid.dimension, setup, ωp, ω) function interpolate_ω_p!(::Dimension{2}, setup, ωp, ω) @@ -413,6 +433,11 @@ function Dfield!(d, G, p, setup; ϵ = eps(eltype(p))) d end +""" + Dfield(p, setup) + +Compute the ``D``-field. +""" Dfield(p, setup) = Dfield!( KernelAbstractions.zeros(get_backend(p), eltype(p), setup.grid.N), ntuple( @@ -455,6 +480,11 @@ function Qfield!(Q, u, setup; ϵ = eps(eltype(Q))) Q end +""" + Qfield(u, setup) + +Compute the ``Q``-field. +""" Qfield(u, setup) = Qfield!( KernelAbstractions.zeros(get_backend(u[1]), eltype(u[1]), setup.grid.N), u, diff --git a/src/time_steppers/step_implicit_runge_kutta.jl b/src/time_steppers/step_implicit_runge_kutta.jl index c63bfefa0..fe5d0478c 100644 --- a/src/time_steppers/step_implicit_runge_kutta.jl +++ b/src/time_steppers/step_implicit_runge_kutta.jl @@ -408,15 +408,11 @@ function timestep!(method::ImplicitRungeKuttaMethod, stepper, Δt; cache, moment create_stepper(method; setup, pressure_solver, bc_vectors, V, p, t, n) end -""" - momentum_allstage(Vⱼ, ϕⱼ, pⱼ, tⱼ, setup; bc_vectors, nstage, get_jacobian = false) - -Call momentum for multiple `(Vⱼ, pⱼ)` pairs, as required in implicit RK methods. - -Non-mutating/allocating/out-of-place version. - -See also [`momentum_allstage!`](@ref). -""" +# Call momentum for multiple `(Vⱼ, pⱼ)` pairs, as required in implicit RK methods. +# +# Non-mutating/allocating/out-of-place version. +# +# See also [`momentum_allstage!`](@ref). function momentum_allstage(Vⱼ, ϕⱼ, pⱼ, tⱼ, setup; bc_vectors, nstage, get_jacobian = false) (; NV, Np) = setup.grid T = eltype(Vⱼ) @@ -446,28 +442,11 @@ function momentum_allstage(Vⱼ, ϕⱼ, pⱼ, tⱼ, setup; bc_vectors, nstage, g Fⱼ, ∇Fⱼ end -""" - momentum_allstage!( - Fⱼ, - ∇Fⱼ, - Vⱼ, - ϕⱼ, - pⱼ, - tⱼ, - setup, - cache, - momentum_cache; - bc_vectors, - nstage, - get_jacobian = false, - ) - -Call momentum for multiple `(V, p)` pairs, as required in implicit RK methods. - -Mutating/non-allocating/in-place version. - -See also [`momentum_allstage`](@ref). -""" +# Call momentum for multiple `(V, p)` pairs, as required in implicit RK methods. +# +# Mutating/non-allocating/in-place version. +# +# See also [`momentum_allstage`](@ref). function momentum_allstage!( Fⱼ, ∇Fⱼ, From 7ca3b075055f7c439fd6f947601294bd9c808868 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Tue, 24 Oct 2023 08:54:02 +0200 Subject: [PATCH 088/379] Add option to display each step --- src/processors/real_time_plot.jl | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/processors/real_time_plot.jl b/src/processors/real_time_plot.jl index 1de7761c7..bb463bd83 100644 --- a/src/processors/real_time_plot.jl +++ b/src/processors/real_time_plot.jl @@ -44,6 +44,7 @@ function field_plot( sleeptime = 0.001, equal_axis = true, displayfig = true, + displayupdates = false, docolorbar = true, resolution = (800, 600), kwargs..., @@ -134,6 +135,9 @@ function field_plot( docolorbar && Colorbar(fig[1, 2], hm) displayfig && display(fig) + displayupdates && on(state) do _ + display(fig) + end fig end From 58348656d57c56a31050579f7325c8f634947f39 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Tue, 24 Oct 2023 08:54:29 +0200 Subject: [PATCH 089/379] Add pretty printing --- src/closures/fno.jl | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/closures/fno.jl b/src/closures/fno.jl index 4f3cb6425..790bb7ea1 100644 --- a/src/closures/fno.jl +++ b/src/closures/fno.jl @@ -114,6 +114,15 @@ Lux.parameterlength((; dimension, kmax, cin, cout)::FourierLayer) = cout * cin + (kmax + 1)^dimension() * 2 * cout * cin Lux.statelength(::FourierLayer) = 0 +## Pretty printing +function Base.show(io::IO, (; dimension, kmax, cin, cout, σ)::FourierLayer) + print(io, "FourierLayer{", dimension(), "}(") + print(io, kmax) + print(io, ", ", cin, " => ", cout) + print(io, "; σ = ", σ) + print(io, ")") +end + # Pass inputs through Fourier layer function ((; dimension, kmax, cout, cin, σ)::FourierLayer)(x, params, state) # TODO: Check if this is more efficient for From 4fd809c01724f07766bd5f22e9ddb18d8b9286e6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Tue, 24 Oct 2023 09:59:42 +0200 Subject: [PATCH 090/379] fix: typo --- docs/src/equations/spatial.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/src/equations/spatial.md b/docs/src/equations/spatial.md index f2a1fb814..5befe70d3 100644 --- a/docs/src/equations/spatial.md +++ b/docs/src/equations/spatial.md @@ -169,7 +169,7 @@ This yields the discrete mass equation ```math \sum_{\alpha = 1}^d \frac{u^\alpha_{I + \delta(\alpha) / 2} - -u^\alpha_{I - \delta(\alpha) / 2}}{\Delta^\alpha_I} = 0, +u^\alpha_{I - \delta(\alpha) / 2}}{\Delta^\alpha_{I(\alpha)}} = 0, ``` where we have divided by the volume sizes ``\Omega_I``. From 46cafe6413e815b1343324ae456a396d36b88824 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Tue, 24 Oct 2023 10:00:10 +0200 Subject: [PATCH 091/379] Update function --- src/closures/create_les_data.jl | 148 +++++++++++++------------------- 1 file changed, 61 insertions(+), 87 deletions(-) diff --git a/src/closures/create_les_data.jl b/src/closures/create_les_data.jl index d2bf55526..4e5d5d4e2 100644 --- a/src/closures/create_les_data.jl +++ b/src/closures/create_les_data.jl @@ -35,75 +35,52 @@ end _filter_saver( dns, les, - KV, + Ku, Kp; nupdate = 1, - bc_vectors_dns = get_bc_vectors(dns, zero(eltype(KV))), - bc_vectors_les = get_bc_vectors(les, zero(eltype(KV))), ) = processor( function (state) - (; Ω, x) = dns.grid + (; dimension, Ω, x) = dns.grid + D = dimension() Ωbar = les.grid.Ω T = eltype(x) - KVmom = Diagonal(Ωbar) * (KV * Diagonal(1 ./ Ω)) _t = fill(zero(T), 0) - _V = fill(zeros(T, 0), 0) + _u = fill(ntuple(α -> zeros(T, 0), D), 0) _p = fill(zeros(T, 0), 0) - _F = fill(zeros(T, 0), 0) - _FG = fill(zeros(T, 0), 0) - _cF = fill(zeros(T, 0), 0) - _cFG = fill(zeros(T, 0), 0) - on(state) do (; V, p, t) - Vbar = KV * V + _F = fill(ntuple(α -> zeros(T, 0), D), 0) + _FG = fill(ntuple(α -> zeros(T, 0), D), 0) + _cF = fill(ntuple(α -> zeros(T, 0), D), 0) + _cFG = fill(ntuple(α -> zeros(T, 0), D), 0) + on(state) do (; u, p, t) + ubar = Ku .* u pbar = Kp * p - F, = momentum(V, V, p, t, dns; bc_vectors = bc_vectors_dns, nopressure = true) - FG, = momentum( - V, - V, - p, - t, - dns; - bc_vectors = bc_vectors_dns, - nopressure = false, - ) - Fbar = KVmom * F - FGbar = KVmom * FG - FVbar, = momentum( - Vbar, - Vbar, - pbar, - t, - les; - bc_vectors = bc_vectors_les, - nopressure = true, - ) - FGVbar, = momentum( - Vbar, - Vbar, - pbar, - t, - les; - bc_vectors = bc_vectors_les, - nopressure = false, - ) - cF = Fbar - FVbar - cFG = FGbar - FGVbar + F = momentum(u, t, dns) + G = pressuregradient(p, dns) + FG = F .+ G + Fbar = Ku .* F + FGbar = Ku .* FG + FVbar = momentum(ubar, t, les) + GVbar = pressuregradient(pbar, les) + FGVbar = FVbar + GVbar + cF = Fbar .- FVbar + cFG = FGbar .- FGVbar push!(_t, t) - push!(_V, Array(Vbar)) + push!(_u, Array.(ubar)) push!(_p, Array(pbar)) - push!(_F, Array(Fbar)) - push!(_FG, Array(FGbar)) - push!(_cF, Array(cF)) - push!(_cFG, Array(cFG)) + push!(_F, Array.(Fbar)) + push!(_FG, Array.(FGbar)) + push!(_cF, Array.(cF)) + push!(_cFG, Array.(cFG)) end state[] = state[] - (; t = _t, V = _V, p = _p, F = _F, FG = _FG, cF = _cF, cFG = _cFG) + (; t = _t, u = _u, p = _p, F = _F, FG = _FG, cF = _cF, cFG = _cFG) end; nupdate, ) function create_les_data( T; + dimension, Re = T(2_000), lims = (T(0), T(1)), nles = 64, @@ -112,21 +89,19 @@ function create_les_data( tburn = T(0.1), tsim = T(0.1), Δt = T(1e-4), - device = identity, + ArrayType = Array, ) + D = dimension() ndns = compression * nles - xdns = LinRange(lims..., ndns + 1) - ydns = LinRange(lims..., ndns + 1) - xles = xdns[1:compression:end] - yles = ydns[1:compression:end] + xdns = ntuple(α -> LinRange(lims..., ndns + 1), D) + xles = map(x -> x[1:compression:end], xdns) # Build setup and assemble operators - dns = Setup(xdns, ydns; Re) - les = Setup(xles, yles; Re) + dns = Setup(xdns...; Re, ArrayType) + les = Setup(xles...; Re, ArrayType) # Filter - (; KV, Kp) = operator_filter(dns.grid, dns.boundary_conditions, compression) - KVmom = Diagonal(les.grid.Ω) * (KV * Diagonal(1 ./ dns.grid.Ω)) + (; Ku, Kp) = operator_filter(dns.grid, dns.boundary_conditions, compression) # Since the grid is uniform and identical for x and y, we may use a specialized # spectral pressure solver @@ -137,14 +112,15 @@ function create_les_data( Δt = tsim / nt # Filtered quantities to store + (; N) = les.grid filtered = (; - V = zeros(T, nles * nles * 2, nt + 1, nsim), - p = zeros(T, nles * nles, nt + 1, nsim), - F = zeros(T, nles * nles * 2, nt + 1, nsim), - FG = zeros(T, nles * nles * 2, nt + 1, nsim), - cF = zeros(T, nles * nles * 2, nt + 1, nsim), - cFG = zeros(T, nles * nles * 2, nt + 1, nsim), - force = zeros(T, nles * nles * 2, nsim), + u = zeros(T, N..., D, nt + 1, nsim), + p = zeros(T, N..., nt + 1, nsim), + F = zeros(T, N..., D, nt + 1, nsim), + FG = zeros(T, N..., D, nt + 1, nsim), + cF = zeros(T, N..., D, nt + 1, nsim), + cFG = zeros(T, N..., D, nt + 1, nsim), + force = zeros(T, N..., D, nsim), ) @info "Generating $(Base.summarysize(filtered) / 1e6) Mb of LES data" @@ -153,38 +129,36 @@ function create_les_data( @info "Generating data for simulation $isim of $nsim" # Initial conditions - V₀, p₀ = random_field(dns; A = T(10_000_000), σ = T(30), s = 5, pressure_solver) + u₀, p₀ = random_field(dns; pressure_solver) # Random body force - force_dns = gaussian_force(xdns, ydns) + - gaussian_force(xdns, ydns) + - # gaussian_force(xdns, ydns) + - # gaussian_force(xdns, ydns) + - gaussian_force(xdns, ydns) - force_les = KVmom * force_dns + force_dns = gaussian_force(xdns...) + + gaussian_force(xdns...) + + # gaussian_force(xdns...) + + # gaussian_force(xdns...) + + gaussian_force(xdns...) + force_les = Ku .* force_dns _dns = (; dns..., force = force_dns) _les = (; les..., force = force_les) # Solve burn-in DNS @info "Burn-in for simulation $isim of $nsim" - V, p, outputs = solve_unsteady( + u, p, outputs = solve_unsteady( _dns, - V₀, + u₀, p₀, (T(0), tburn); Δt, processors = (step_logger(; nupdate = 10),), pressure_solver, - inplace = true, - device, ) # Solve DNS and store filtered quantities @info "Solving DNS for simulation $isim of $nsim" - V, p, outputs = solve_unsteady( + u, p, outputs = solve_unsteady( _dns, - V, + u, p, (T(0), tsim); Δt, @@ -192,7 +166,7 @@ function create_les_data( _filter_saver( device(_dns), device(_les), - device(KV), + device(Ku), device(Kp); bc_vectors_dns = device(get_bc_vectors(_dns, T(0))), bc_vectors_les = device(get_bc_vectors(_les, T(0))), @@ -206,13 +180,13 @@ function create_les_data( f = outputs[1] # Store result for current IC - filtered.V[:, :, isim] = stack(f.V) - filtered.p[:, :, isim] = stack(f.p) - filtered.F[:, :, isim] = stack(f.F) - filtered.FG[:, :, isim] = stack(f.FG) - filtered.cF[:, :, isim] = stack(f.cF) - filtered.cFG[:, :, isim] = stack(f.cFG) - filtered.force[:, isim] = force_les + filtered.u[ntuple(α -> :, D + 2)..., isim] = stack(stack.(f.u)) + filtered.p[ntuple(α -> :, D + 1)..., isim] = stack(f.p) + filtered.F[ntuple(α -> :, D + 2)..., isim] = stack(stack.(f.F)) + filtered.FG[ntuple(α -> :, D + 2)..., isim] = stack(stack.(f.FG)) + filtered.cF[ntuple(α -> :, D + 2)..., isim] = stack(stack.(f.cF)) + filtered.cFG[ntuple(α -> :, D + 2)..., isim] = stack(stack.(f.cFG)) + filtered.force[ntuple(α -> :, D + 1)..., isim] = stack.(force_les) end filtered From 0ff3a66f8a575a459f3e126247e58dd4b7813fc1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Tue, 24 Oct 2023 10:00:29 +0200 Subject: [PATCH 092/379] Add section --- docs/src/features/operators.md | 40 ++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100644 docs/src/features/operators.md diff --git a/docs/src/features/operators.md b/docs/src/features/operators.md new file mode 100644 index 000000000..d83ba6ac5 --- /dev/null +++ b/docs/src/features/operators.md @@ -0,0 +1,40 @@ +```@meta +CurrentModule = IncompressibleNavierStokes +``` + +# Operators + +All discrete operators are built using +[KernelAbstractions.jl](https://github.com/JuliaGPU/KernelAbstractions.jl/) +and Cartesian indices, similar to +[WaterLily.jl](https://github.com/weymouth/WaterLily.jl/). +This allows for dimension- and backend-agnostic code. See this +[blog post](https://b-fg.github.io/2023/05/07/waterlily-on-gpu.html) +for how to write kernels. IncompressibleNavierStokes previously relied on +assembling sparse operators to perform the same operations. While being very +efficient and also compatible with CUDA (CUSPARSE), storing these matrices in +memory is expensive for large 3D problems. + +```@docs +Offset +divergence! +divergence +vorticity +vorticity! +convection! +diffusion! +bodyforce! +momentum! +momentum +pressuregradient! +pressuregradient +interpolate_u_p +interpolate_u_p! +interpolate_ω_p +interpolate_ω_p! +Dfield! +Dfield +Qfield! +Qfield +kinetic_energy +``` From db02045a2cbdfe248f90ab1d1508a4c825fb4fab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Thu, 26 Oct 2023 14:56:43 +0200 Subject: [PATCH 093/379] Update files --- src/IncompressibleNavierStokes.jl | 3 + src/boundary_conditions.jl | 69 +++++-------------- src/create_initial_conditions.jl | 6 +- src/operators.jl | 18 ++--- src/postprocess/plot_vorticity.jl | 9 ++- src/processors/real_time_plot.jl | 7 +- .../pressure/pressure_additional_solve.jl | 1 + .../step_explicit_runge_kutta.jl | 2 +- 8 files changed, 47 insertions(+), 68 deletions(-) diff --git a/src/IncompressibleNavierStokes.jl b/src/IncompressibleNavierStokes.jl index 449ac3c72..5a84e8a55 100644 --- a/src/IncompressibleNavierStokes.jl +++ b/src/IncompressibleNavierStokes.jl @@ -29,6 +29,9 @@ const WORKGROUP = 64 # Convenience notation const ⊗ = kron +# Easily retrieve value from Val +(::Val{x})() where {x} = x + # Boundary condtions include("boundary_conditions.jl") diff --git a/src/boundary_conditions.jl b/src/boundary_conditions.jl index e92efb113..ee3d3f52a 100644 --- a/src/boundary_conditions.jl +++ b/src/boundary_conditions.jl @@ -167,44 +167,8 @@ function apply_bc_p!(::PeriodicBC, p, β, t, setup; atend, kwargs...) else _bc_a(get_backend(p), WORKGROUP)(p, Val(β); ndrange) end - synchronize(get_backend(p)) end -# function apply_bc_u!(bc::DirichletBC, u, β, t, setup; atend, dudt = false, kwargs...) -# (; dimension, Nu, x, xp, Iu) = setup.grid -# D = dimension() -# δ = Offset{D}() -# isnothing(bc.u) && return -# bcfunc = dudt ? bc.dudt : bc.u -# @kernel function _bc_a(u, α, β, I0) -# I = @index(Global, Cartesian) -# I = I + I0 -# u[α][I] = bcfunc[α](ntuple(γ -> γ == α ? x[γ][I[α]+1] : xp[γ][I[γ]], D)..., t) -# end -# @kernel function _bc_b(u, α, β, I0) -# I = @index(Global, Cartesian) -# I = I + I0 -# u[α][I] = bcfunc[α](ntuple(γ -> γ == α ? x[γ][I[α]+1] : xp[γ][I[γ]], D)..., t) -# end -# for α = 1:D -# Xu = (xp[1:β-1]..., x[β], xp[β+1:end]...) -# ndrange = (Nu[α][1:β-1]..., 1, Nu[α][β+1:end]...) -# if atend -# I0 = first(Iu[α]) -# I0 -= oneunit(I0) -# I0 += Nu[α][β] * δ(β) -# _bc_b(get_backend(u[1]), WORKGROUP)(u, α, β, I0; ndrange) -# synchronize(get_backend(u[1])) -# else -# I0 = first(Iu[α]) -# I0 -= oneunit(I0) -# I0 -= δ(β) -# _bc_a(get_backend(u[1]), WORKGROUP)(u, α, β, I0; ndrange) -# synchronize(get_backend(u[1])) -# end -# end -# end - function apply_bc_u!(bc::DirichletBC, u, β, t, setup; atend, dudt = false, kwargs...) (; dimension, x, xp, N) = setup.grid D = dimension() @@ -214,10 +178,7 @@ function apply_bc_u!(bc::DirichletBC, u, β, t, setup; atend, dudt = false, kwar for α = 1:D if atend I = CartesianIndices( - ntuple( - γ -> γ == β ? isnormal ? (N[γ]-1:N[γ]-1) : (N[γ]:N[γ]) : (1:N[γ]), - D, - ), + ntuple(γ -> γ == β ? α == β ? (N[γ]-1:N[γ]-1) : (N[γ]:N[γ]) : (1:N[γ]), D), ) else I = CartesianIndices(ntuple(γ -> γ == β ? (1:1) : (1:N[γ]), D)) @@ -231,7 +192,7 @@ function apply_bc_u!(bc::DirichletBC, u, β, t, setup; atend, dudt = false, kwar ), D, ) - u[α][I] .= bcfunc[α].(xI..., t) + u[α][I] .= bcfunc.(Val(α), xI..., t) end end @@ -257,37 +218,43 @@ function apply_bc_u!(::SymmetricBC, u, β, t, setup; atend, kwargs...) end function apply_bc_p!(::SymmetricBC, p, β, t, setup; atend, kwargs...) - nothing + (; dimension, N) = setup.grid + D = dimension() + δ = Offset{D}() + if atend + I = CartesianIndices(ntuple(γ -> γ == β ? (N[γ]:N[γ]) : (1:N[γ]), D)) + p[I] .= p[I.-δ(β)] + else + I = CartesianIndices(ntuple(γ -> γ == β ? (1:1) : (1:N[γ]), D)) + p[I] .= p[I.+δ(β)] + end end function apply_bc_u!(bc::PressureBC, u, β, t, setup; atend, kwargs...) (; grid) = setup - (; dimension, Nu, Iu) = grid + (; dimension, N, Nu, Iu) = grid D = dimension() δ = Offset{D}() @kernel function _bc_a!(u, ::Val{α}, ::Val{β}, I0) where {α,β} I = @index(Global, Cartesian) I = I + I0 - u[α][I-δ(β)] = u[α][I] + u[α][I] = u[α][I+δ(β)] end @kernel function _bc_b!(u, ::Val{α}, ::Val{β}, I0) where {α,β} I = @index(Global, Cartesian) I = I + I0 - u[α][I+δ(β)] = u[α][I] + u[α][I] = u[α][I-δ(β)] end + ndrange = (N[1:β-1]..., 1, N[β+1:end]...) for α = 1:D if atend - I0 = first(Iu[α]) + (Nu[α][β] - 1) * δ(β) + I0 = CartesianIndex(ntuple(γ -> γ == β ? N[β] : 1, D)) I0 -= oneunit(I0) - ndrange = (Nu[α][1:β-1]..., 1, Nu[α][β+1:end]...) _bc_b!(get_backend(u[1]), WORKGROUP)(u, Val(α), Val(β), I0; ndrange) - synchronize(get_backend(u[1])) else - I0 = first(Iu[α]) + I0 = CartesianIndex(ntuple(γ -> γ == β && α != β ? 2 : 1, D)) I0 -= oneunit(I0) - ndrange = (Nu[α][1:β-1]..., 1, Nu[α][β+1:end]...) _bc_a!(get_backend(u[1]), WORKGROUP)(u, Val(α), Val(β), I0; ndrange) - synchronize(get_backend(u[1])) end end end diff --git a/src/create_initial_conditions.jl b/src/create_initial_conditions.jl index be7ba187a..7186a22e5 100644 --- a/src/create_initial_conditions.jl +++ b/src/create_initial_conditions.jl @@ -3,10 +3,12 @@ setup, initial_velocity, t = 0; - pressure_solver = DirectPressureSolver(setup), + pressure_solver = CGPressureSolverManual(setup), ) Create initial vectors `(u, p)` at starting time `t`. +The initial conditions of `u[α]` are specified by the function +`initial_velocity(Val(α), x...)`. """ function create_initial_conditions( setup, @@ -27,7 +29,7 @@ function create_initial_conditions( # Initial velocities for α = 1:D xin = ntuple(β -> reshape(α == β ? x[β][2:end] : xp[β], ntuple(Returns(1), β - 1)..., :), D) - u[α][Iu[α]] .= initial_velocity[α].(xin...)[Iu[α]] + u[α][Iu[α]] .= initial_velocity.(Val(α), xin...)[Iu[α]] end apply_bc_u!(u, t, setup) diff --git a/src/operators.jl b/src/operators.jl index d3ba23ae1..088349f35 100644 --- a/src/operators.jl +++ b/src/operators.jl @@ -18,7 +18,7 @@ struct Offset{D} end Compute divergence of velocity field (in-place version). """ function divergence!(M, u, setup) - (; boundary_conditions, grid) = setup + (; grid) = setup (; Δ, N, Ip) = grid D = length(u) δ = Offset{D}() @@ -75,7 +75,7 @@ Compute vorticity field. vorticity!(ω, u, setup) = vorticity!(setup.grid.dimension, ω, u, setup) function vorticity!(::Dimension{2}, ω, u, setup) - (; boundary_conditions, grid) = setup + (; grid) = setup (; dimension, Δu, N) = grid D = dimension() δ = Offset{D}() @@ -92,7 +92,7 @@ function vorticity!(::Dimension{2}, ω, u, setup) end function vorticity!(::Dimension{3}, ω, u, setup) - (; boundary_conditions, grid) = setup + (; grid) = setup (; dimension, Δu, N) = grid D = dimension() δ = Offset{D}() @@ -120,7 +120,7 @@ end Compute convective term. """ function convection!(F, u, setup) - (; boundary_conditions, grid, Re) = setup + (; grid) = setup (; dimension, Δ, Δu, Nu, Iu, A) = grid D = dimension() δ = Offset{D}() @@ -157,7 +157,7 @@ end Compute diffusive term. """ function diffusion!(F, u, setup) - (; boundary_conditions, grid, Re) = setup + (; grid, Re) = setup (; dimension, Δ, Δu, Nu, Iu) = grid D = dimension() δ = Offset{D}() @@ -195,14 +195,14 @@ end Compute body force. """ function bodyforce!(F, u, t, setup) - (; boundary_conditions, grid, Re, bodyforce) = setup + (; grid, bodyforce) = setup (; dimension, Δ, Δu, Nu, Iu, x, xp) = grid D = dimension() δ = Offset{D}() - @kernel function _bodyforce!(F, force, ::Val{α}, t, I0) where {α} + @kernel function _bodyforce!(F, force, valα::Val{α}, t, I0) where {α} I = @index(Global, Cartesian) I = I + I0 - F[α][I] += force[α](ntuple(β -> α == β ? x[β][1+I[β]] : xp[β][I[β]], D)..., t) + F[α][I] += force(valα, ntuple(β -> α == β ? x[β][1+I[β]] : xp[β][I[β]], D)..., t) end for α = 1:D I0 = first(Iu[α]) @@ -260,7 +260,7 @@ momentum(u, t, setup) = momentum!( Compute pressure gradient (in-place). """ function pressuregradient!(G, p, setup) - (; boundary_conditions, grid) = setup + (; grid) = setup (; dimension, Δu, Nu, Iu) = grid D = dimension() δ = Offset{D}() diff --git a/src/postprocess/plot_vorticity.jl b/src/postprocess/plot_vorticity.jl index 813c1d0d3..3a22474d2 100644 --- a/src/postprocess/plot_vorticity.jl +++ b/src/postprocess/plot_vorticity.jl @@ -11,15 +11,18 @@ plot_vorticity(setup, u; kwargs...) = # 2D version function plot_vorticity(::Dimension{2}, setup, u; kwargs...) (; grid, boundary_conditions) = setup - (; xp, xlims) = grid + (; xp, xlims, Ip) = grid T = eltype(xp[1]) + xf = Array.(getindex.(xp, Ip.indices)) + # Get fields ω = vorticity(u, setup) ωp = interpolate_ω_p(setup, ω) + ωp = Array(ωp)[Ip] # Levels - μ, σ = mean(ω), std(ω) + μ, σ = mean(ωp), std(ωp) # ≈(μ + σ, μ; rtol = 1e-8, atol = 1e-8) && (σ = 1e-4) levels = LinRange(μ - T(1.5) * σ, μ + T(1.5) * σ, 10) @@ -34,7 +37,7 @@ function plot_vorticity(::Dimension{2}, setup, u; kwargs...) ) limits!(ax, xlims[1]..., xlims[2]...) # cf = contourf!(ax, xp..., ω; extendlow = :auto, extendhigh = :auto, levels, kwargs...) - cf = heatmap!(ax, Array.(xp)..., Array(ωp); kwargs...) + cf = heatmap!(ax, xf..., ωp; kwargs...) Colorbar(fig[1, 2], cf) # save("output/vorticity.png", fig, pt_per_unit = 2) diff --git a/src/processors/real_time_plot.jl b/src/processors/real_time_plot.jl index bb463bd83..4bc58e5ed 100644 --- a/src/processors/real_time_plot.jl +++ b/src/processors/real_time_plot.jl @@ -56,7 +56,7 @@ function field_plot( xf = Array.(getindex.(setup.grid.xp, Ip.indices)) (; u, p, t) = state[] - if fieldname == :velocity + _f = if fieldname == :velocity up = interpolate_u_p(setup, u) elseif fieldname == :vorticity ω = vorticity(u, setup) @@ -64,7 +64,9 @@ function field_plot( elseif fieldname == :streamfunction ψ = get_streamfunction(setup, u, t) elseif fieldname == :pressure + p end + _f = Array(_f)[Ip] field = @lift begin isnothing(sleeptime) || sleep(sleeptime) (; u, p, t) = $state @@ -80,7 +82,8 @@ function field_plot( elseif fieldname == :pressure p end - Array(f)[Ip] + # Array(f)[Ip] + copyto!(_f, view(f, Ip)) end lims = @lift begin diff --git a/src/solvers/pressure/pressure_additional_solve.jl b/src/solvers/pressure/pressure_additional_solve.jl index f6664f1ed..31449d780 100644 --- a/src/solvers/pressure/pressure_additional_solve.jl +++ b/src/solvers/pressure/pressure_additional_solve.jl @@ -13,6 +13,7 @@ function pressure_additional_solve!(pressure_solver, u, p, t, setup, F, G, M) apply_bc_u!(F, t, setup; dudt = true) + apply_bc_p!(p, t, setup) pressuregradient!(G, p, setup) for α = 1:D F[α] .-= G[α] diff --git a/src/time_steppers/step_explicit_runge_kutta.jl b/src/time_steppers/step_explicit_runge_kutta.jl index 8d2f288af..a35f72988 100644 --- a/src/time_steppers/step_explicit_runge_kutta.jl +++ b/src/time_steppers/step_explicit_runge_kutta.jl @@ -56,9 +56,9 @@ function timestep!(method::ExplicitRungeKuttaMethod, stepper, Δt; cache) # Solve the Poisson equation pressure_poisson!(pressure_solver, p, M) - apply_bc_p!(p, t, setup) # Compute pressure correction term + apply_bc_p!(p, t, setup) pressuregradient!(G, p, setup) # Update velocity current stage, which is now divergence free From 1f6ceae2e8814a0a892d4c0032c517e2eb022739 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Thu, 26 Oct 2023 14:56:56 +0200 Subject: [PATCH 094/379] Fix: Wrong sign --- src/boundary_conditions.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/boundary_conditions.jl b/src/boundary_conditions.jl index ee3d3f52a..f47e9b406 100644 --- a/src/boundary_conditions.jl +++ b/src/boundary_conditions.jl @@ -65,7 +65,7 @@ ghost_a!(::DirichletBC, x) = pushfirst!(x, x[1]) ghost_b!(::DirichletBC, x) = push!(x, x[end]) # Duplicate boundary volume -ghost_a!(::SymmetricBC, x) = pushfirst!(x, x[1] + (x[2] - x[1])) +ghost_a!(::SymmetricBC, x) = pushfirst!(x, x[1] - (x[2] - x[1])) ghost_b!(::SymmetricBC, x) = push!(x, x[end] + (x[end] - x[end-1])) # Add infinitely thin boundary volume From 71521c55f77614cc10b46fb7f11d487495652265 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Thu, 26 Oct 2023 15:02:27 +0200 Subject: [PATCH 095/379] Add Laplacian --- src/operators.jl | 30 +++++++++++++++++++++ src/solvers/pressure/pressure_poisson.jl | 33 ++++++++++++++---------- src/solvers/pressure/pressure_solvers.jl | 13 ++-------- 3 files changed, 52 insertions(+), 24 deletions(-) diff --git a/src/operators.jl b/src/operators.jl index 088349f35..d1b9eb401 100644 --- a/src/operators.jl +++ b/src/operators.jl @@ -292,6 +292,36 @@ pressuregradient(p, setup) = pressuregradient!( setup, ) +""" + laplacian!(L, p, setup) + +Compute Laplacian of pressure field (in-place version). +""" +function laplacian!(L, p, setup) + (; grid) = setup + (; dimension, Δ, Δu, N, Np, Ip, Ω) = grid + D = dimension() + δ = Offset{D}() + @kernel function _laplacian!(L, p, ::Val{α}, I0) where {α} + I = @index(Global, Cartesian) + I = I + I0 + L[I] += + Ω[I] / Δ[α][I[α]] * + ((p[I+δ(α)] - p[I]) / Δu[α][I[α]] - (p[I] - p[I-δ(α)]) / Δu[α][I[α]-1]) + end + L .= 0 + # All volumes have a right velocity + # All volumes have a left velocity except the first one + # Start at second volume + ndrange = Np + I0 = first(Ip) + I0 -= oneunit(I0) + for α = 1:D + _laplacian!(get_backend(L), WORKGROUP)(L, p, Val(α), I0; ndrange) + end + L +end + """ interpolate_u_p(setup, u) diff --git a/src/solvers/pressure/pressure_poisson.jl b/src/solvers/pressure/pressure_poisson.jl index 089ce54eb..d64179845 100644 --- a/src/solvers/pressure/pressure_poisson.jl +++ b/src/solvers/pressure/pressure_poisson.jl @@ -45,19 +45,29 @@ function pressure_poisson!(solver::CGPressureSolver, p, f) cg!(p, A, f; abstol, reltol, maxiter) end +# Solve L p = f +# where Lp = Ω * div(pressurgrad(p)) +# +# L is rank-1 deficient, so we add the constraint sum(p) = 0, i.e. solve +# +# [0 1] [0] [0] +# [1 L] [p] = [f] +# +# instead. This way, the matrix is still positive definite. +# For initial guess, we already know the average is zero. function pressure_poisson!(solver::CGPressureSolverManual, p, f) - (; setup, abstol, reltol, maxiter, r, G, M, q) = solver + (; setup, abstol, reltol, maxiter, r, L, q) = solver (; Ip, Ω) = setup.grid T = typeof(reltol) + p .= 0 + # Initial residual - pressuregradient!(G, p, setup) - divergence!(M, G, setup) - @. M *= Ω + laplacian!(L, p, setup) # Intialize q .= 0 - r .= f .- M + r .= f .- L residual = norm(r[Ip]) prev_residual = one(residual) tolerance = max(reltol * residual, abstol) @@ -67,16 +77,13 @@ function pressure_poisson!(solver::CGPressureSolverManual, p, f) β = residual^2 / prev_residual^2 q .= r .+ β .* q - pressuregradient!(G, q, setup) - divergence!(M, G, setup) - @. M *= Ω - α = residual^2 / sum(q[Ip] .* M[Ip]) + # Periodic paddding (maybe) + apply_bc_p!(q, T(0), setup) + laplacian!(L, q, setup) + α = residual^2 / sum(q[Ip] .* L[Ip]) p .+= α .* q - r .-= α .* M - - # Periodic paddding (maybe) - apply_bc_p!(p, T(0), setup) + r .-= α .* L prev_residual = residual residual = norm(r[Ip]) diff --git a/src/solvers/pressure/pressure_solvers.jl b/src/solvers/pressure/pressure_solvers.jl index 0e55c6194..f77478636 100644 --- a/src/solvers/pressure/pressure_solvers.jl +++ b/src/solvers/pressure/pressure_solvers.jl @@ -64,14 +64,13 @@ Adapt.adapt_structure(to, s::CGPressureSolver) = CGPressureSolver( Conjugate gradients iterative pressure solver. """ -struct CGPressureSolverManual{T,S,A,AT} <: AbstractPressureSolver{T} +struct CGPressureSolverManual{T,S,A} <: AbstractPressureSolver{T} setup::S abstol::T reltol::T maxiter::Int r::A - G::AT - M::A + L::A q::A end @@ -90,14 +89,6 @@ CGPressureSolverManual( eltype(setup.grid.x[1]), setup.grid.N, ), - ntuple( - α -> KernelAbstractions.zeros( - get_backend(setup.grid.x[1]), - eltype(setup.grid.x[1]), - setup.grid.N, - ), - setup.grid.dimension(), - ), KernelAbstractions.zeros( get_backend(setup.grid.x[1]), eltype(setup.grid.x[1]), From a16ae9bdd6bdd65d8d4dff491c71980fa234b56c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Thu, 26 Oct 2023 16:07:33 +0200 Subject: [PATCH 096/379] Update syntax --- README.md | 27 +++++++----- examples/Actuator2D.jl | 35 +++++++++------ examples/Actuator3D.jl | 75 ++++++++++++-------------------- examples/BackwardFacingStep2D.jl | 50 +++++++++------------ examples/BackwardFacingStep3D.jl | 53 +++++++++------------- examples/DecayingTurbulence2D.jl | 30 +++++-------- examples/DecayingTurbulence3D.jl | 29 +++++------- examples/LidDrivenCavity2D.jl | 40 +++++++---------- examples/LidDrivenCavity3D.jl | 59 +++++++++---------------- examples/PlanarMixing2D.jl | 56 ++++++++++-------------- examples/PlaneJets2D.jl | 36 +++++---------- examples/ShearLayer2D.jl | 39 ++++++----------- examples/TaylorGreenVortex2D.jl | 12 +---- examples/TaylorGreenVortex3D.jl | 56 ++++++++++++------------ 14 files changed, 246 insertions(+), 351 deletions(-) diff --git a/README.md b/README.md index c2cc5b3c2..db7960207 100644 --- a/README.md +++ b/README.md @@ -61,33 +61,38 @@ x = LinRange(0.0, 10.0, 5n + 1) y = LinRange(-2.0, 2.0, 2n + 1) # Boundary conditions: Unsteady BC requires time derivatives -U(x, y, t) = cos(π / 6 * sin(π / 6 * t)) -V(x, y, t) = sin(π / 6 * sin(π / 6 * t)) -dUdt(x, y, t) = -(π / 6)^2 * cos(π / 6 * t) * sin(π / 6 * sin(π / 6 * t)) -dVdt(x, y, t) = (π / 6)^2 * cos(π / 6 * t) * cos(π / 6 * sin(π / 6 * t)) boundary_conditions = ( # Inlet, outlet - (DirichletBC((U, V), (dUdt, dVdt)), PressureBC()), + ( + # Unsteady BC requires time derivatives + DirichletBC( + (dim, x, y, t) -> sin(π / 6 * sin(π / 6 * t) + π / 2 * (dim() == 1)), + (dim, x, y, t) -> + (π / 6)^2 * + cos(π / 6 * t) * + cos(π / 6 * sin(π / 6 * t) + π / 2 * (dim() == 1)), + ), + PressureBC(), + ), # Sides - (SymmetricBC(), SymmetricBC()), + (PressureBC(), PressureBC()), ) # Actuator body force: A thrust coefficient `Cₜ` distributed over a thin rectangle xc, yc = 2.0, 0.0 # Disk center D = 1.0 # Disk diameter δ = 0.11 # Disk thickness -Cₜ = 5e-4 # Thrust coefficient +Cₜ = 0.2 # Thrust coefficient cₜ = Cₜ / (D * δ) inside(x, y) = abs(x - xc) ≤ δ / 2 && abs(y - yc) ≤ D / 2 -fu(x, y, t) = -cₜ * inside(x, y) -fv(x, y, t) = 0.0 +bodyforce(dim, x, y, t) = dim() == 1 ? -cₜ * inside(x, y) : 0.0 # Build setup and assemble operators -setup = Setup(x, y; Re = 100.0, boundary_conditions, bodyforce = (fu, fv)); +setup = Setup(x, y; Re = 100.0, boundary_conditions, bodyforce); # Initial conditions (extend inflow) -u₀, p₀ = create_initial_conditions(setup, ((x, y) -> 1.0, (x, y) -> 0.0)); +u₀, p₀ = create_initial_conditions(setup, (dim, x, y) -> dim() == 1 ? 1.0 : 0.0); # Solve unsteady Navier-Stokes equations u, p, outputs = solve_unsteady( diff --git a/examples/Actuator2D.jl b/examples/Actuator2D.jl index 60144bbaa..f2e120c12 100644 --- a/examples/Actuator2D.jl +++ b/examples/Actuator2D.jl @@ -30,40 +30,46 @@ x = LinRange(0.0, 10.0, 5n + 1) y = LinRange(-2.0, 2.0, 2n + 1) plot_grid(x, y) -# Boundary conditions: Unsteady BC requires time derivatives -U(x, y, t) = cos(π / 6 * sin(π / 6 * t)) -V(x, y, t) = sin(π / 6 * sin(π / 6 * t)) -dUdt(x, y, t) = -(π / 6)^2 * cos(π / 6 * t) * sin(π / 6 * sin(π / 6 * t)) -dVdt(x, y, t) = (π / 6)^2 * cos(π / 6 * t) * cos(π / 6 * sin(π / 6 * t)) +# Boundary conditions boundary_conditions = ( ## x left, x right - (DirichletBC((U, V), (dUdt, dVdt)), PressureBC()), + ( + ## Unsteady BC requires time derivatives + DirichletBC( + (dim, x, y, t) -> sin(π / 6 * sin(π / 6 * t) + π / 2 * (dim() == 1)), + (dim, x, y, t) -> + (π / 6)^2 * + cos(π / 6 * t) * + cos(π / 6 * sin(π / 6 * t) + π / 2 * (dim() == 1)), + ), + PressureBC(), + ), ## y rear, y front - (SymmetricBC(), SymmetricBC()), + (PressureBC(), PressureBC()), ) # Actuator body force: A thrust coefficient `Cₜ` distributed over a thin rectangle xc, yc = 2.0, 0.0 # Disk center D = 1.0 # Disk diameter δ = 0.11 # Disk thickness -Cₜ = 5e-4 # Thrust coefficient +Cₜ = 0.2 # Thrust coefficient cₜ = Cₜ / (D * δ) inside(x, y) = abs(x - xc) ≤ δ / 2 && abs(y - yc) ≤ D / 2 -fu(x, y, t) = -cₜ * inside(x, y) -fv(x, y, t) = 0.0 +bodyforce(dim, x, y, t) = dim() == 1 ? -cₜ * inside(x, y) : 0.0 # Build setup and assemble operators -setup = Setup(x, y; Re = 100.0, boundary_conditions, bodyforce = (fu, fv)); +setup = Setup(x, y; Re = 100.0, boundary_conditions, bodyforce); # Initial conditions (extend inflow) -u₀, p₀ = create_initial_conditions(setup, ((x, y) -> 1.0, (x, y) -> 0.0)); +u₀, p₀ = create_initial_conditions(setup, (dim, x, y) -> dim() == 1 ? 1.0 : 0.0); +u, p = u₀, p₀ # Solve unsteady problem u, p, outputs = solve_unsteady( setup, - u₀, - p₀, + ## u₀, p₀, + u, p, (0.0, 12.0); method = RK44P2(), Δt = 0.05, @@ -78,6 +84,7 @@ u, p, outputs = solve_unsteady( ), ); + # ## Post-process # # We may visualize or export the computed fields `(V, p)`. diff --git a/examples/Actuator3D.jl b/examples/Actuator3D.jl index 58b641ecb..b444decc3 100644 --- a/examples/Actuator3D.jl +++ b/examples/Actuator3D.jl @@ -44,79 +44,60 @@ z = LinRange(-2.0, 2.0, 41) plot_grid(x, y, z) # Boundary conditions: Unsteady BC requires time derivatives -U(x, y, z, t) = cos(π / 6 * sin(π / 6 * t)) -V(x, y, z, t) = sin(π / 6 * sin(π / 6 * t)) -W(x, y, z, t) = zero(x) -dUdt(x, y, z, t) = -(π / 6)^2 * cos(π / 6 * t) * sin(π / 6 * sin(π / 6 * t)) -dVdt(x, y, z, t) = (π / 6)^2 * cos(π / 6 * t) * cos(π / 6 * sin(π / 6 * t)) -dWdt(x, y, z, t) = zero(x) boundary_conditions = ( ## x left, x right - (DirichletBC((U, V, W), (dUdt, dVdt, dWdt)), PressureBC()), + ( + DirichletBC( + (dim, x, y, z, t) -> + dim() == 1 ? cos(π / 6 * sin(π / 6 * t)) : + dim() == 2 ? sin(π / 6 * sin(π / 6 * t)) : zero(x), + (dim, x, y, z, t) -> + dim() == 1 ? -(π / 6)^2 * cos(π / 6 * t) * sin(π / 6 * sin(π / 6 * t)) : + dim() == 2 ? (π / 6)^2 * cos(π / 6 * t) * cos(π / 6 * sin(π / 6 * t)) : + zero(x), + ), + PressureBC(), + ), ## y rear, y front - (SymmetricBC(), SymmetricBC()), + (PressureBC(), PressureBC()), ## z rear, z front - (SymmetricBC(), SymmetricBC()), + (PressureBC(), PressureBC()), ) # Actuator body force: A thrust coefficient `Cₜ` distributed over a short cylinder cx, cy, cz = T(2), T(0), T(0) # Disk center D = T(1) # Disk diameter δ = T(0.11) # Disk thickness -Cₜ = T(5e-4) # Thrust coefficient +Cₜ = T(0.2) # Thrust coefficient cₜ = Cₜ / (π * (D / 2)^2 * δ) inside(x, y, z) = abs(x - cx) ≤ δ / 2 && (y - cy)^2 + (z - cz)^2 ≤ (D / 2)^2 -fu(x, y, z) = -cₜ * inside(x, y, z) -fv(x, y, z) = zero(x) -fw(x, y, z) = zero(x) +bodyforce(dim, x, y, z) = dim() == 1 ? -cₜ * inside(x, y, z) : zero(x) # Build setup and assemble operators -setup = Setup( - x, y, z; - Re, - boundary_conditions, - bodyforce = (fu, fv, fw), - ArrayType, -); - -# Time interval -t_start, t_end = tlims = T(0), T(3) +setup = Setup(x, y, z; Re, boundary_conditions, bodyforce, ArrayType); # Initial conditions (extend inflow) -initial_velocity = ( - (x, y, z) -> one(x), - (x, y, z) -> zero(x), - (x, y, z) -> zero(x), -) -u₀, p₀ = create_initial_conditions( - setup, - initial_velocity, - t_start; -); - -# Iteration processors -processors = ( - field_plotter(setup; nupdate = 5), - ## energy_history_plotter(setup; nupdate = 10), - ## energy_spectrum_plotter(setup; nupdate = 10), - ## animator(setup, "vorticity.mkv"; nupdate = 4), - ## vtk_writer(setup; nupdate = 2, dir = "output/$name", filename = "solution"), - ## field_saver(setup; nupdate = 10), - step_logger(; nupdate = 1), -); +u₀, p₀ = create_initial_conditions(setup, (dim, x, y, z) -> dim() == 1 ? one(x) : zero(x)); # Solve unsteady problem u, p, outputs = solve_unsteady( setup, u₀, p₀, - tlims; + (T(0), T(3)); method = RK44P2(), Δt = T(0.05), - processors, - inplace = true, + processors = ( + field_plotter(setup; nupdate = 5), + ## energy_history_plotter(setup; nupdate = 10), + ## energy_spectrum_plotter(setup; nupdate = 10), + ## animator(setup, "vorticity.mkv"; nupdate = 4), + ## vtk_writer(setup; nupdate = 2, dir = "output/$name", filename = "solution"), + ## field_saver(setup; nupdate = 10), + step_logger(; nupdate = 1), + ); ); # ## Post-process diff --git a/examples/BackwardFacingStep2D.jl b/examples/BackwardFacingStep2D.jl index 30aed52e5..e1ab85c7e 100644 --- a/examples/BackwardFacingStep2D.jl +++ b/examples/BackwardFacingStep2D.jl @@ -39,13 +39,11 @@ ArrayType = Array Re = T(3_000) # Boundary conditions: steady inflow on the top half -U(x, y, t::T) where {T} = y ≥ 0 ? 24y * (T(1 / 2) - y) : zero(x) -V(x, y, t) = zero(x) -dUdt(x, y, t) = zero(x) -dVdt(x, y, t) = zero(x) +U(dim, x, y, t) = dim() == 1 && y ≥ 0 ? 24y * (one(x) / 2 - y) : zero(x) +dUdt(dim, x, y, t) = zero(x) boundary_conditions = ( ## x left, x right - (DirichletBC((U, V), (dUdt, dVdt)), PressureBC()), + (DirichletBC(U, dUdt), PressureBC()), ## y rear, y front (DirichletBC(), DirichletBC()), @@ -60,36 +58,30 @@ plot_grid(x, y) # Build setup and assemble operators setup = Setup(x, y; Re, boundary_conditions, ArrayType); -# Time interval -t_start, t_end = tlims = T(0), T(7) - # Initial conditions (extend inflow) -initial_velocity = ( - (x, y) -> U(x, y, zero(x)), - (x, y) -> zero(x), -) -u₀, p₀ = create_initial_conditions( - setup, - initial_velocity, - t_start; -); +u₀, p₀ = create_initial_conditions(setup, (dim, x, y) -> U(dim, x, y, zero(x))); # Solve steady state problem ## u, p = solve_steady_state(setup, u₀, p₀); -# Iteration processors -processors = ( - field_plotter(setup; nupdate = 5), - ## energy_history_plotter(setup; nupdate = 10), - ## energy_spectrum_plotter(setup; nupdate = 10), - ## animator(setup, "vorticity.mkv"; nupdate = 4), - ## vtk_writer(setup; nupdate = 20, dir = "output/$name", filename = "solution"), - ## field_saver(setup; nupdate = 10), - step_logger(; nupdate = 1), -); - # Solve unsteady problem -u, p, outputs = solve_unsteady(setup, u₀, p₀, tlims; Δt = T(0.002), processors, inplace = true); +u, p, outputs = solve_unsteady( + setup, + u₀, + p₀, + (T(0), T(7)); + Δt = T(0.002), + processors = ( + field_plotter(setup; nupdate = 5), + ## energy_history_plotter(setup; nupdate = 10), + ## energy_spectrum_plotter(setup; nupdate = 10), + ## animator(setup, "vorticity.mkv"; nupdate = 4), + ## vtk_writer(setup; nupdate = 20, dir = "output/$name", filename = "solution"), + ## field_saver(setup; nupdate = 10), + step_logger(; nupdate = 1), + ), + inplace = true, +); # ## Post-process # diff --git a/examples/BackwardFacingStep3D.jl b/examples/BackwardFacingStep3D.jl index 2912d3ae5..b0abb1066 100644 --- a/examples/BackwardFacingStep3D.jl +++ b/examples/BackwardFacingStep3D.jl @@ -45,15 +45,11 @@ z = LinRange(-T(0.25), T(0.25), 9) plot_grid(x, y, z) # Boundary conditions: steady inflow on the top half -U(x, y, z, t) = y ≥ 0 ? 24y * (1 - y) / 2 : zero(x) -V(x, y, z, t) = zero(x) -W(x, y, z, t) = zero(x) -dUdt(x, y, z, t) = zero(x) -dVdt(x, y, z, t) = zero(x) -dWdt(x, y, z, t) = zero(x) +U(dim, x, y, z, t) = dim() == 1 && y ≥ 0 ? 24y * (one(x) / 2 - y) : zero(x) +dUdt(dim, x, y, z, t) = zero(x) boundary_conditions = ( ## x left, x right - (DirichletBC((U, V, W), (dUdt, dVdt, dWdt)), PressureBC()), + (DirichletBC(U, dUdt), PressureBC()), ## y rear, y front (DirichletBC(), DirichletBC()), @@ -65,37 +61,30 @@ boundary_conditions = ( # Build setup and assemble operators setup = Setup(x, y, z; Re, boundary_conditions, ArrayType); -# Time interval -t_start, t_end = tlims = T(0), T(7) - # Initial conditions (extend inflow) -initial_velocity = ( - (x, y, z) -> U(x, y, z, zero(x)), - (x, y, z) -> zero(x), - (x, y, z) -> zero(x), -) -u₀, p₀ = create_initial_conditions( - setup, - initial_velocity, - t_start; -); +u₀, p₀ = create_initial_conditions(setup, (dim, x, y, z) -> U(dim, x, y, z, zero(x))); # Solve steady state problem ## u, p = solve_steady_state(setup, u₀, p₀); - -# Iteration processors -processors = ( - field_plotter(setup; nupdate = 50), - ## energy_history_plotter(setup; nupdate = 10), - ## energy_spectrum_plotter(setup; nupdate = 10), - ## animator(setup, "vorticity.mkv"; nupdate = 4), - ## vtk_writer(setup; nupdate = 20, dir = "output/$name", filename = "solution"), - ## field_saver(setup; nupdate = 10), - step_logger(; nupdate = 10), -); +nothing # Solve unsteady problem -u, p, outputs = solve_unsteady(setup, u₀, p₀, tlims; Δt = 0.01, processors, inplace = true) +u, p, outputs = solve_unsteady( + setup, + u₀, + p₀, + (T(0), T(7)); + Δt = T(0.01), + processors = ( + field_plotter(setup; nupdate = 50), + ## energy_history_plotter(setup; nupdate = 10), + ## energy_spectrum_plotter(setup; nupdate = 10), + ## animator(setup, "vorticity.mkv"; nupdate = 4), + ## vtk_writer(setup; nupdate = 20, dir = "output/$name", filename = "solution"), + ## field_saver(setup; nupdate = 10), + step_logger(; nupdate = 10), + ), +) #md current_figure() # ## Post-process diff --git a/examples/DecayingTurbulence2D.jl b/examples/DecayingTurbulence2D.jl index bc199ccf9..55a6a9ea6 100644 --- a/examples/DecayingTurbulence2D.jl +++ b/examples/DecayingTurbulence2D.jl @@ -52,32 +52,26 @@ setup = Setup(x...; Re, ArrayType); # spectral pressure solver pressure_solver = SpectralPressureSolver(setup); -u₀, p₀ = random_field(setup, T(0); A = T(1_000_000), σ = T(30), s = T(5), pressure_solver); - -# Iteration processors -processors = ( - field_plotter(setup; nupdate = 20), - energy_history_plotter(setup; nupdate = 20, displayfig = false), - energy_spectrum_plotter(setup; nupdate = 20, displayfig = false), - ## animator(setup, "vorticity.mp4"; nupdate = 16), - ## vtk_writer(setup; nupdate = 10, dir = "output/$name", filename = "solution"), - ## field_saver(setup; nupdate = 10), - step_logger(; nupdate = 100), -); - -# Time interval -t_start, t_end = tlims = T(0), T(1) +u₀, p₀ = random_field(setup, T(0); pressure_solver); # Solve unsteady problem u, p, outputs = solve_unsteady( setup, u₀, p₀, - tlims; - Δt = T(0.001), - processors, + (T(0), T(1)); + Δt = T(1e-3), pressure_solver, inplace = true, + processors = ( + field_plotter(setup; nupdate = 20), + energy_history_plotter(setup; nupdate = 20, displayfig = false), + energy_spectrum_plotter(setup; nupdate = 20, displayfig = false), + ## animator(setup, "vorticity.mp4"; nupdate = 16), + ## vtk_writer(setup; nupdate = 10, dir = "output/$name", filename = "solution"), + ## field_saver(setup; nupdate = 10), + step_logger(; nupdate = 100), + ), ); # ## Post-process diff --git a/examples/DecayingTurbulence3D.jl b/examples/DecayingTurbulence3D.jl index 121481fe2..b0bb3af10 100644 --- a/examples/DecayingTurbulence3D.jl +++ b/examples/DecayingTurbulence3D.jl @@ -55,32 +55,25 @@ setup = Setup(x, y, z; Re, ArrayType); pressure_solver = SpectralPressureSolver(setup); # Initial conditions -u₀, p₀ = random_field(setup; A = T(1_000_000), σ = T(30), s = 5, pressure_solver) - -# Time interval -t_start, t_end = tlims = T(0), T(1.0) - -# Iteration processors -processors = ( - field_plotter(setup; nupdate = 10), - energy_history_plotter(setup; nupdate = 10), - energy_spectrum_plotter(setup; nupdate = 10), - ## animator(setup, "vorticity.mp4"; nupdate = 4), - ## vtk_writer(setup; nupdate = 10, dir = "output/$name", filename = "solution"), - ## field_saver(setup; nupdate = 10), - step_logger(; nupdate = 1), -); +u₀, p₀ = random_field(setup; pressure_solver) # Solve unsteady problem u, p, outputs = solve_unsteady( setup, u₀, p₀, - tlims; + (T(0), T(1)); Δt = T(0.001), - processors, pressure_solver, - inplace = true, + processors = ( + field_plotter(setup; nupdate = 10), + energy_history_plotter(setup; nupdate = 10), + energy_spectrum_plotter(setup; nupdate = 10), + ## animator(setup, "vorticity.mp4"; nupdate = 4), + ## vtk_writer(setup; nupdate = 10, dir = "output/$name", filename = "solution"), + ## field_saver(setup; nupdate = 10), + step_logger(; nupdate = 1), + ), ); # Field plot diff --git a/examples/LidDrivenCavity2D.jl b/examples/LidDrivenCavity2D.jl index 3ef0dff80..18e6ab107 100644 --- a/examples/LidDrivenCavity2D.jl +++ b/examples/LidDrivenCavity2D.jl @@ -56,15 +56,19 @@ ArrayType = Array Re = T(1_000) # Non-zero Dirichlet boundary conditions are specified as plain Julia functions. -# Other possible BC types are `PeriodicBC()`, `SymmetricBC`, and `PressureBC`. -lidvel = ((x, y, t) -> one(x), (x, y, t) -> zero(x)) -dlidveldt = ((x, y, t) -> zero(x), (x, y, t) -> zero(x)) +# Note that time derivatives are required. boundary_conditions = ( ## x left, x right (DirichletBC(), DirichletBC()), ## y bottom, y top - (DirichletBC(), DirichletBC(lidvel, dlidveldt)), + ( + DirichletBC(), + DirichletBC( + (dim, x, y, t) -> dim() == 1 ? one(x) : zero(x), + (dim, x, y, t) -> zero(x), + ), + ), ) # We create a two-dimensional domain with a box of size `[1, 1]`. The grid is @@ -82,7 +86,7 @@ setup = Setup(x, y; boundary_conditions, Re, ArrayType); # The pressure solver is used to solve the pressure Poisson equation. # Available solvers are -# +# # - [`DirectPressureSolver`](@ref) (only for CPU with `Float64`) # - [`CGPressureSolver`](@ref) # - [`SpectralPressureSolver`](@ref) (only for periodic boundary conditions and @@ -90,15 +94,10 @@ setup = Setup(x, y; boundary_conditions, Re, ArrayType); pressure_solver = CGPressureSolverManual(setup); -# We will solve for a time interval of ten seconds. -t_start, t_end = tlims = T(0), T(10) - -# The initial conditions are defined as plain Julia functions. -initial_velocity = ( - (x, y) -> zero(x), - (x, y) -> zero(x), -) -u₀, p₀ = create_initial_conditions(setup, initial_velocity, t_start; pressure_solver); +# The initial conditions are provided in function. The value `dim()` determines +# the velocity component. +u₀, p₀ = create_initial_conditions(setup, (dim, x, y) -> zero(x); pressure_solver); +u, p = u₀, p₀ # ## Solve problems # @@ -120,22 +119,15 @@ processors = ( ## energy_history_plotter(setup; nupdate = 1), ## energy_spectrum_plotter(setup; nupdate = 100), ## animator(setup, "vorticity.mkv"; nupdate = 4), - vtk_writer(setup; nupdate = 100, dir = "output/$name", filename = "solution"), + ## vtk_writer(setup; nupdate = 100, dir = "output/$name", filename = "solution"), ## field_saver(setup; nupdate = 10), step_logger(; nupdate = 1000), ); # By default, a standard fourth order Runge-Kutta method is used. If we don't # provide the time step explicitly, an adaptive time step is used. -u, p, outputs = solve_unsteady( - setup, - u₀, - p₀, - tlims; - Δt = T(0.001), - processors, - pressure_solver, -); +u, p, outputs = + solve_unsteady(setup, u, p, (T(0), T(0.1)); Δt = T(0.001), pressure_solver, processors); # ## Post-process # diff --git a/examples/LidDrivenCavity3D.jl b/examples/LidDrivenCavity3D.jl index 1622e141f..d4c8a6f74 100644 --- a/examples/LidDrivenCavity3D.jl +++ b/examples/LidDrivenCavity3D.jl @@ -45,22 +45,14 @@ z = LinRange(-T(0.2), T(0.2), 11) plot_grid(x, y, z) # Boundary conditions: horizontal movement of the top lid -lidvel = ( - (x, y, z, t) -> one(x), - (x, y, z, t) -> zero(x), - (x, y, z, t) -> one(x) / 5, -) -dlidveldt = ( - (x, y, z, t) -> zero(x), - (x, y, z, t) -> zero(x), - (x, y, z, t) -> zero(x), -) +U(dim, x, y, z, t) = dim() == 1 ? one(x) : dim() == 2 ? zero(x) : one(x) / 5 +dUdt(dim, x, y, z, t) = zero(x) boundary_conditions = ( ## x left, x right (DirichletBC(), DirichletBC()), ## y rear, y front - (DirichletBC(), DirichletBC(lidvel, dlidveldt)), + (DirichletBC(), DirichletBC(U, dUdt)), ## z bottom, z top (PeriodicBC(), PeriodicBC()), @@ -69,39 +61,30 @@ boundary_conditions = ( # Build setup and assemble operators setup = Setup(x, y, z; Re, boundary_conditions, ArrayType); -# Time interval -t_start, t_end = tlims = T(0), T(0.2) - # Initial conditions -initial_velocity = ( - (x, y, z) -> zero(x), - (x, y, z) -> zero(x), - (x, y, z) -> zero(x), -) -u₀, p₀ = create_initial_conditions( - setup, - initial_velocity, - t_start; - pressure_solver, -) +u₀, p₀ = create_initial_conditions(setup, (dim, x, y, z) -> zero(x)) # Solve steady state problem ## u, p = solve_steady_state(setup, u₀, p₀; npicard = 5, maxiter = 15); - -# Iteration processors -processors = ( - field_plotter(setup; nupdate = 1), - ## energy_history_plotter(setup; nupdate = 1), - ## energy_spectrum_plotter(setup; nupdate = 100), - ## animator(setup, "vorticity.mkv"; nupdate = 4), - ## vtk_writer(setup; nupdate = 5, dir = "output/$name", filename = "solution"), - ## field_saver(setup; nupdate = 10), - step_logger(; nupdate = 1), -); +nothing # Solve unsteady problem -u, p, outputs = - solve_unsteady(setup, u₀, p₀, tlims; Δt = T(0.001), processors, device); +u, p, outputs = solve_unsteady( + setup, + u₀, + p₀, + (T(0), T(0.2)); + Δt = T(0.001), + processors = ( + field_plotter(setup; nupdate = 1), + ## energy_history_plotter(setup; nupdate = 1), + ## energy_spectrum_plotter(setup; nupdate = 100), + ## animator(setup, "vorticity.mkv"; nupdate = 4), + ## vtk_writer(setup; nupdate = 5, dir = "output/$name", filename = "solution"), + ## field_saver(setup; nupdate = 10), + step_logger(; nupdate = 1), + ), +); # ## Post-process # diff --git a/examples/PlanarMixing2D.jl b/examples/PlanarMixing2D.jl index 544ad19ef..9f25b5302 100644 --- a/examples/PlanarMixing2D.jl +++ b/examples/PlanarMixing2D.jl @@ -31,13 +31,18 @@ Ubar = 1.0 ϵ = (0.082Ubar, 0.012Ubar) n = (0.4π, 0.3π) ω = (0.22, 0.11) -U(x, y, t) = 1.0 + ΔU / 2 * tanh(2y) + sum(@. ϵ * (1 - tanh(y / 2)^2) * cos(n * y) * sin(ω * t)) -V(x, y, t) = 0.0 -dUdt(x, y, t) = sum(@. ϵ * (1 - tanh(y / 2)^2) * cos(n * y) * ω * cos(ω * t)) -dVdt(x, y, t) = 0.0 +U(dim, x, y, t) = + dim() == 1 ? + 1.0 + ΔU / 2 * tanh(2y) + sum(@. ϵ * (1 - tanh(y / 2)^2) * cos(n * y) * sin(ω * t)) : + 0.0 +dUdt(dim, x, y, t) = + dim() == 1 ? sum(@. ϵ * (1 - tanh(y / 2)^2) * cos(n * y) * ω * cos(ω * t)) : 0.0 boundary_conditions = ( - (DirichletBC((U, V), (dUdt, dVdt)), PressureBC()), - (SymmetricBC(), SymmetricBC()) + ## x left, x right + (DirichletBC(U, dUdt), PressureBC()), + + ## y rear, y front + (SymmetricBC(), SymmetricBC()), ) # A 2D grid is a Cartesian product of two vectors @@ -50,41 +55,26 @@ plot_grid(x, y) # Build setup and assemble operators setup = Setup(x, y; Re, boundary_conditions); -# Time interval -t_start, t_end = tlims = 0.0, 100.0 - -# Initial conditions (exten inflow) -initial_velocity = ( - (x, y) -> U(x, y, 0.0), - (x, y) -> 0.0, -) -u₀, p₀ = create_initial_conditions( - setup, - initial_velocity, - t_start; -); - -# Iteration processors -processors = ( - field_plotter(setup; nupdate = 1), - ## energy_history_plotter(setup; nupdate = 1), - ## energy_spectrum_plotter(setup; nupdate = 100), - ## animator(setup, "vorticity.mkv"; nupdate = 4), - ## vtk_writer(setup; nupdate = 10, dir = "output/$name", filename = "solution"), - ## field_saver(setup; nupdate = 10), - step_logger(; nupdate = 1), -); +# Initial conditions (extend inflow) +u₀, p₀ = create_initial_conditions(setup, (dim, x, y) -> U(dim, x, y, 0.0)); # Solve unsteady problem u, p, outputs = solve_unsteady( setup, u₀, p₀, - tlims; + (0.0, 100.0); method = RK44P2(), Δt = 0.1, - processors, - inplace = true, + processors = ( + field_plotter(setup; nupdate = 1), + ## energy_history_plotter(setup; nupdate = 1), + ## energy_spectrum_plotter(setup; nupdate = 100), + ## animator(setup, "vorticity.mkv"; nupdate = 4), + ## vtk_writer(setup; nupdate = 10, dir = "output/$name", filename = "solution"), + ## field_saver(setup; nupdate = 10), + step_logger(; nupdate = 1), + ), ); # ## Post-process diff --git a/examples/PlaneJets2D.jl b/examples/PlaneJets2D.jl index cf79989ff..dd56846a8 100644 --- a/examples/PlaneJets2D.jl +++ b/examples/PlaneJets2D.jl @@ -97,18 +97,10 @@ setup = Setup(x, y; Re, ArrayType); # spectral pressure solver pressure_solver = SpectralPressureSolver(setup) -# Time interval -t_start, t_end = tlims = T(0), T(1) - # Initial conditions -initial_velocity = ( - (x, y) -> U(x, y), - (x, y) -> zero(x), -) u₀, p₀ = create_initial_conditions( setup, - initial_velocity, - t_start; + (dim, x, y) -> dim() == 1 ? U(x, y) : zero(x); pressure_solver, ); @@ -175,29 +167,25 @@ mean_plotter(setup; nupdate = 1) = processor( nupdate, ) -# Iteration processors -processors = ( - field_plotter(setup; nupdate = 1), - ## energy_history_plotter(setup; nupdate = 1), - ## energy_spectrum_plotter(setup; nupdate = 100), - ## animator(setup, "vorticity.mkv"; nupdate = 4), - ## vtk_writer(setup; nupdate = 10, dir = "output/$name", filename = "solution"), - ## field_saver(setup; nupdate = 10), - mean_plotter(setup), - step_logger(; nupdate = 1), -); - # Solve unsteady problem toto, p, outputs = solve_unsteady( setup, u₀, p₀, - tlims; + (T(0), T(1)); method = RK44P2(), Δt = 0.001, - processors, pressure_solver, - inplace = true, + processors = ( + field_plotter(setup; nupdate = 1), + ## energy_history_plotter(setup; nupdate = 1), + ## energy_spectrum_plotter(setup; nupdate = 100), + ## animator(setup, "vorticity.mkv"; nupdate = 4), + ## vtk_writer(setup; nupdate = 10, dir = "output/$name", filename = "solution"), + ## field_saver(setup; nupdate = 10), + mean_plotter(setup), + step_logger(; nupdate = 1), + ), ); # ## Post-process diff --git a/examples/ShearLayer2D.jl b/examples/ShearLayer2D.jl index d1ee28d13..41b869b40 100644 --- a/examples/ShearLayer2D.jl +++ b/examples/ShearLayer2D.jl @@ -47,50 +47,37 @@ setup = Setup(x, y; Re, ArrayType); pressure_solver = SpectralPressureSolver(setup) -# Time interval -t_start, t_end = tlims = T(0), T(8) - # Initial conditions: We add 1 to u in order to make global momentum # conservation less trivial d = T(π / 15) e = T(0.05) -initial_velocity = ( - (x, y) -> y ≤ π ? tanh((y - T(π / 2)) / d) : tanh((T(3π / 2) - y) / d), - (x, y) -> e * sin(x), -) -## initial_velocity = ( -## (x, y) -> T(1) + (y ≤ π ? tanh((y - T(π / 2)) / d) : tanh((T(3π / 2) - y) / d)), -## (x, y) -> e * sin(x), -## ) +U1(y) = y ≤ π ? tanh((y - T(π / 2)) / d) : tanh((T(3π / 2) - y) / d) +## U1(y) = T(1) + (y ≤ π ? tanh((y - T(π / 2)) / d) : tanh((T(3π / 2) - y) / d)) u₀, p₀ = create_initial_conditions( setup, - initial_velocity, - t_start; + (dim, x, y) -> dim() == 1 ? U1(y) : e * sin(x); pressure_solver, ); # Iteration processors -processors = ( - field_plotter(setup; nupdate = 1), - ## energy_history_plotter(setup; nupdate = 1), - ## energy_spectrum_plotter(setup; nupdate = 100), - ## animator(setup, "vorticity.mkv"; nupdate = 4), - ## vtk_writer(setup; nupdate = 10, dir = "output/$name", filename = "solution"), - ## field_saver(setup; nupdate = 10), - step_logger(; nupdate = 1), -); # Solve unsteady problem u, p, outputs = solve_unsteady( setup, u₀, p₀, - tlims; - method = RK44(), + (T(0), T(8)); Δt = T(0.01), - processors, - inplace = true, pressure_solver, + processors = ( + field_plotter(setup; nupdate = 1), + ## energy_history_plotter(setup; nupdate = 1), + ## energy_spectrum_plotter(setup; nupdate = 100), + ## animator(setup, "vorticity.mkv"; nupdate = 4), + ## vtk_writer(setup; nupdate = 10, dir = "output/$name", filename = "solution"), + ## field_saver(setup; nupdate = 10), + step_logger(; nupdate = 1), + ), ); # ## Post-process diff --git a/examples/TaylorGreenVortex2D.jl b/examples/TaylorGreenVortex2D.jl index 6d01c640f..362e9c94c 100644 --- a/examples/TaylorGreenVortex2D.jl +++ b/examples/TaylorGreenVortex2D.jl @@ -48,18 +48,10 @@ setup = Setup(x...; Re, ArrayType); # spectral pressure solver pressure_solver = SpectralPressureSolver(setup) -# Time interval -t_start, t_end = tlims = T(0), T(5) - # Initial conditions -initial_velocity = ( - (x, y) -> -sin(x) * cos(y), - (x, y) -> cos(x) * sin(y), -) u₀, p₀ = create_initial_conditions( setup, - initial_velocity, - t_start; + (dim, x, y) -> dim() == 1 ? -sin(x) * cos(y) : cos(x) * sin(y); pressure_solver, ); @@ -82,7 +74,7 @@ u, p, outputs = solve_unsteady( setup, u₀, p₀, - tlims; + (T(0), T(5)); Δt = T(0.01), processors, pressure_solver, diff --git a/examples/TaylorGreenVortex3D.jl b/examples/TaylorGreenVortex3D.jl index 916c5f737..465fbe756 100644 --- a/examples/TaylorGreenVortex3D.jl +++ b/examples/TaylorGreenVortex3D.jl @@ -33,11 +33,11 @@ ArrayType = Array ## using Metal; ArrayType = MtlArray # Reynolds number -Re = T(6_000) +Re = T(10_000) # A 3D grid is a Cartesian product of three vectors n = 32 -lims = T(0), T(2π) +lims = T(0), T(1) x = LinRange(lims..., n + 1) y = LinRange(lims..., n + 1) z = LinRange(lims..., n + 1) @@ -51,17 +51,14 @@ setup = Setup(x, y, z; Re, ArrayType); pressure_solver = SpectralPressureSolver(setup); # Initial conditions -initial_velocity = ( - (x, y, z) -> sin(x)cos(y)cos(z), - (x, y, z) -> -cos(x)sin(y)cos(z), - (x, y, z) -> zero(x), -) u₀, p₀ = create_initial_conditions( setup, - initial_velocity, - T(0); + (dim, x, y, z) -> + dim() == 1 ? sinpi(2x) * cospi(2y) * sinpi(2z) : + dim() == 2 ? -cospi(2x) * sinpi(2y) * sinpi(2z) : zero(x); pressure_solver, ); +u, p = u₀, p₀ GC.gc() CUDA.reclaim() @@ -70,28 +67,33 @@ CUDA.reclaim() ## u, p = solve_steady_state(setup, u₀, p₀; npicard = 6) nothing -# Iteration processors -processors = ( - # field_plotter(setup; fieldname = :velocity, nupdate = 1), - # energy_history_plotter(setup; nupdate = 1), - energy_spectrum_plotter(setup; nupdate = 100), - ## animator(setup, "vorticity.mp4"; nupdate = 4), - ## vtk_writer(setup; nupdate = 10, dir = "output/$name", filename = "solution"), - ## field_saver(setup; nupdate = 10), - step_logger(; nupdate = 1), -); - -# Time interval -t_start, t_end = tlims = T(0), T(5) - # Solve unsteady problem u, p, outputs = solve_unsteady( setup, - u₀, p₀, - # u, p, - tlims; + u₀, + p₀, + (T(0), T(5)); Δt = T(0.01), - processors, + processors = ( + ## field_plotter(setup; nupdate = 1), + energy_history_plotter(setup; nupdate = 1), + ## energy_spectrum_plotter(setup; nupdate = 100), + ## animator( + ## setup, + ## "vorticity3D.mp4"; + ## plotter = field_plotter( + ## setup; + ## fieldname = :Dfield, + ## levels = LinRange(-T(3), T(1), 5), + ## docolorbar = false, + ## resolution = (1024, 1024), + ## ), + ## nupdate = 20, + ## ), + ## vtk_writer(setup; nupdate = 10, dir = "output/$name", filename = "solution"), + ## field_saver(setup; nupdate = 10), + step_logger(; nupdate = 1), + ), pressure_solver, inplace = true, ); From 81e4f73d5855240c32591e88701385b1e599a0d6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Tue, 31 Oct 2023 19:34:39 +0100 Subject: [PATCH 097/379] Add plan_fft --- src/solvers/pressure/pressure_poisson.jl | 12 ++++++------ src/solvers/pressure/pressure_solvers.jl | 8 +++++--- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/src/solvers/pressure/pressure_poisson.jl b/src/solvers/pressure/pressure_poisson.jl index d64179845..d80e83fbe 100644 --- a/src/solvers/pressure/pressure_poisson.jl +++ b/src/solvers/pressure/pressure_poisson.jl @@ -95,21 +95,21 @@ function pressure_poisson!(solver::CGPressureSolverManual, p, f) end function pressure_poisson!(solver::SpectralPressureSolver, p, f) - (; setup, Ahat, fhat, phat) = solver + (; setup, plan, Ahat, fhat, phat) = solver (; Ip) = setup.grid - f = @view f[Ip] + f = view(f, Ip) - phat .= complex.(f) + fhat .= complex.(f) # Fourier transform of right hand side - fft!(phat) + mul!(phat, plan, fhat) # Solve for coefficients in Fourier space - @. phat = -phat / Ahat + @. fhat = -phat / Ahat # Transform back - ifft!(phat) + ldiv!(phat, plan, fhat) @. p[Ip] = real(phat) p diff --git a/src/solvers/pressure/pressure_solvers.jl b/src/solvers/pressure/pressure_solvers.jl index f77478636..b3c973d44 100644 --- a/src/solvers/pressure/pressure_solvers.jl +++ b/src/solvers/pressure/pressure_solvers.jl @@ -101,17 +101,18 @@ CGPressureSolverManual( ), ) -struct SpectralPressureSolver{T,A<:AbstractArray{Complex{T}},S} <: AbstractPressureSolver{T} +struct SpectralPressureSolver{T,A<:AbstractArray{Complex{T}},S,P} <: AbstractPressureSolver{T} setup::S Ahat::A phat::A fhat::A + plan::P end # This moves all the inner arrays to the GPU when calling # `cu(::SpectralPressureSolver)` from CUDA.jl Adapt.adapt_structure(to, s::SpectralPressureSolver) = - SpectralPressureSolver(adapt(to, s.Ahat), adapt(to, s.phat), adapt(to, s.fhat)) + SpectralPressureSolver(adapt(to, s.Ahat), adapt(to, s.phat), adapt(to, s.fhat), adapt(to, s.plan)) """ SpectralPressureSolver(setup) @@ -166,6 +167,7 @@ function SpectralPressureSolver(setup) # Placeholders for intermediate results phat = zero(Ahat) fhat = zero(Ahat) + plan = plan_fft(fhat) - SpectralPressureSolver{T,typeof(Ahat),typeof(setup)}(setup, Ahat, phat, fhat) + SpectralPressureSolver{T,typeof(Ahat),typeof(setup),typeof(plan)}(setup, Ahat, phat, fhat, plan) end From d42cd79a7fc75f5eff6cc332147f3f4581cce88e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Tue, 31 Oct 2023 19:36:46 +0100 Subject: [PATCH 098/379] Add preconditioner --- docs/src/features/operators.md | 1 + src/boundary_conditions.jl | 11 ++++- src/solvers/pressure/pressure_poisson.jl | 52 +++++++++++++++++++----- src/solvers/pressure/pressure_solvers.jl | 29 ++++++++++++- 4 files changed, 80 insertions(+), 13 deletions(-) diff --git a/docs/src/features/operators.md b/docs/src/features/operators.md index d83ba6ac5..efc841fdd 100644 --- a/docs/src/features/operators.md +++ b/docs/src/features/operators.md @@ -26,6 +26,7 @@ diffusion! bodyforce! momentum! momentum +laplacian! pressuregradient! pressuregradient interpolate_u_p diff --git a/src/boundary_conditions.jl b/src/boundary_conditions.jl index f47e9b406..9a11262ac 100644 --- a/src/boundary_conditions.jl +++ b/src/boundary_conditions.jl @@ -197,7 +197,16 @@ function apply_bc_u!(bc::DirichletBC, u, β, t, setup; atend, dudt = false, kwar end function apply_bc_p!(::DirichletBC, p, β, t, setup; atend, kwargs...) - nothing + (; dimension, N) = setup.grid + D = dimension() + δ = Offset{D}() + if atend + I = CartesianIndices(ntuple(γ -> γ == β ? (N[γ]:N[γ]) : (1:N[γ]), D)) + p[I] .= p[I.-δ(β)] + else + I = CartesianIndices(ntuple(γ -> γ == β ? (1:1) : (1:N[γ]), D)) + p[I] .= p[I.+δ(β)] + end end function apply_bc_u!(::SymmetricBC, u, β, t, setup; atend, kwargs...) diff --git a/src/solvers/pressure/pressure_poisson.jl b/src/solvers/pressure/pressure_poisson.jl index d80e83fbe..e21626af8 100644 --- a/src/solvers/pressure/pressure_poisson.jl +++ b/src/solvers/pressure/pressure_poisson.jl @@ -45,7 +45,7 @@ function pressure_poisson!(solver::CGPressureSolver, p, f) cg!(p, A, f; abstol, reltol, maxiter) end -# Solve L p = f +# Solve Lp = f # where Lp = Ω * div(pressurgrad(p)) # # L is rank-1 deficient, so we add the constraint sum(p) = 0, i.e. solve @@ -56,10 +56,25 @@ end # instead. This way, the matrix is still positive definite. # For initial guess, we already know the average is zero. function pressure_poisson!(solver::CGPressureSolverManual, p, f) - (; setup, abstol, reltol, maxiter, r, L, q) = solver - (; Ip, Ω) = setup.grid + (; setup, abstol, reltol, maxiter, r, L, q, preconditioner) = solver + (; Np, Ip, Ω) = setup.grid T = typeof(reltol) + function innerdot(a, b) + @kernel function innerdot!(d, a, b, I0) + I = @index(Global, Cartesian) + I = I + I0 + d[I-I+I0] += a[I] * b[I] + # a[I] = b[I] + end + # d = zero(eltype(a)) + I0 = first(Ip) + I0 -= oneunit(I0) + d = KernelAbstractions.zeros(get_backend(a), eltype(a), ntuple(Returns(1), length(I0))) + innerdot!(get_backend(a), WORKGROUP)(d, a, b, I0; ndrange = Np) + d[] + end + p .= 0 # Initial residual @@ -68,25 +83,40 @@ function pressure_poisson!(solver::CGPressureSolverManual, p, f) # Intialize q .= 0 r .= f .- L - residual = norm(r[Ip]) - prev_residual = one(residual) + ρ_prev = one(T) + # residual = norm(r[Ip]) + residual = sqrt(sum(abs2, view(r, Ip))) + # residual = norm(r) tolerance = max(reltol * residual, abstol) iteration = 0 while iteration < maxiter && residual > tolerance - β = residual^2 / prev_residual^2 - q .= r .+ β .* q + preconditioner(L, r) + + # ρ = sum(L[Ip] .* r[Ip]) + ρ = dot(view(L, Ip), view(r, Ip)) + # ρ = innerdot(L, r) + # ρ = dot(L, r) + + β = ρ / ρ_prev + q .= L .+ β .* q - # Periodic paddding (maybe) + # Periodic/symmetric padding (maybe) apply_bc_p!(q, T(0), setup) laplacian!(L, q, setup) - α = residual^2 / sum(q[Ip] .* L[Ip]) + # α = ρ / sum(q[Ip] .* L[Ip]) + # α = ρ / dot(view(q, Ip), view(L, Ip)) + # α = ρ / innerdot(q, L) + α = ρ / dot(q, L) p .+= α .* q r .-= α .* L - prev_residual = residual - residual = norm(r[Ip]) + ρ_prev = ρ + # residual = norm(r[Ip]) + residual = sqrt(sum(abs2, view(r, Ip))) + # residual = sqrt(sum(abs2, r)) + # residual = sqrt(innerdot(r, r)) iteration += 1 end diff --git a/src/solvers/pressure/pressure_solvers.jl b/src/solvers/pressure/pressure_solvers.jl index b3c973d44..9ffba09a5 100644 --- a/src/solvers/pressure/pressure_solvers.jl +++ b/src/solvers/pressure/pressure_solvers.jl @@ -64,7 +64,7 @@ Adapt.adapt_structure(to, s::CGPressureSolver) = CGPressureSolver( Conjugate gradients iterative pressure solver. """ -struct CGPressureSolverManual{T,S,A} <: AbstractPressureSolver{T} +struct CGPressureSolverManual{T,S,A,F} <: AbstractPressureSolver{T} setup::S abstol::T reltol::T @@ -72,6 +72,30 @@ struct CGPressureSolverManual{T,S,A} <: AbstractPressureSolver{T} r::A L::A q::A + preconditioner::F +end + +function create_laplace_diag(setup) + (; grid) = setup + (; dimension, Δ, Δu, N, Np, Ip, Ω) = grid + D = dimension() + δ = Offset{D}() + @kernel function _laplace_diag!(z, p, I0) + I = @index(Global, Cartesian) + I = I + I0 + d = zero(eltype(z)) + for α = 1:length(I) + d -= Ω[I] / Δ[α][I[α]] * (1 / Δu[α][I[α]] + 1 / Δu[α][I[α]-1]) + end + z[I] = -p[I] / d + end + ndrange = Np + I0 = first(Ip) + I0 -= oneunit(I0) + function laplace_diag(z, p) + _laplace_diag!(get_backend(z), WORKGROUP)(z, p, I0; ndrange) + # synchronize(get_backend(z)) + end end CGPressureSolverManual( @@ -79,6 +103,8 @@ CGPressureSolverManual( abstol = zero(eltype(setup.grid.x[1])), reltol = sqrt(eps(eltype(setup.grid.x[1]))), maxiter = prod(setup.grid.Np), + # preconditioner = copy!, + preconditioner = create_laplace_diag(setup), ) = CGPressureSolverManual( setup, abstol, @@ -99,6 +125,7 @@ CGPressureSolverManual( eltype(setup.grid.x[1]), setup.grid.N, ), + preconditioner, ) struct SpectralPressureSolver{T,A<:AbstractArray{Complex{T}},S,P} <: AbstractPressureSolver{T} From 9a6707ac24ea8186048f1c4163f448d1f101e74d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Tue, 31 Oct 2023 19:37:17 +0100 Subject: [PATCH 099/379] docs: Add fourth order --- docs/src/equations/spatial.md | 38 ++++++++++++++++++++++++++++++++--- 1 file changed, 35 insertions(+), 3 deletions(-) diff --git a/docs/src/equations/spatial.md b/docs/src/equations/spatial.md index 5befe70d3..0e730126b 100644 --- a/docs/src/equations/spatial.md +++ b/docs/src/equations/spatial.md @@ -285,7 +285,7 @@ Finally, the discrete ``\alpha``-momentum equations are given by - \frac{ u^\alpha_{I + \delta(\alpha) / 2} - u^\alpha_{I + \delta(\alpha) / 2 - \delta(\beta)} - }{\Delta^\beta_{I(\beta) - 1 / 2}} + }{\Delta^\beta_{I(\beta) + \delta_{\alpha \beta} / 2 - 1 / 2}} \right) \\ + & f^\alpha(x_{I + \delta(\alpha) / 2}), \end{split} @@ -379,13 +379,45 @@ column-major convention. Note that the ``d`` discrete velocity fields ``u^1_h, u^1_{(2, 1, 1)}, \dots u^3_{(N_{u^3}(1), N_{u^3}(2), N_{u^3}(3))})`` in 3D. +## Fourth order accurate discretization + +The above discretization is second order accurate. +A fourth order accurate discretization can be obtained by judiciously combining +the second order discretization with itself on a grid with three times larger +cells in each dimension [Verstappen2003](@cite) [Sanderse2014](@cite). The +coarse discretization is identical, but the mass equation is derived for the +control volume $\bigcup_{\alpha = 1}^d \Omega_{I - \delta(\alpha)} \cup +\Omega_I \cup \Omega_{I + \delta(\alpha)}$, while the momentum equation is +derived for the control volume $\bigcup_{\alpha = 1}^d \Omega_{I - +\delta(\alpha) / 2} \cup \Omega_{I + \delta(\alpha) / 2} \cup \Omega_{I + 3 +\delta(\alpha) / 2}$. The resulting fourth order accurate equations are given +by + +```math +\sum_{\alpha = 1}^d +\frac{u^\alpha_{I + \delta(\alpha) / 2} - +u^\alpha_{I - \delta(\alpha) / 2}}{\Delta^\alpha_{I(\alpha)}} +- +\frac{\sum_{e \in \{-1, 0, 1\}^d} | \Omega_{I + e} |}{3^{2 + d} | \Omega_I |} +\sum_{\alpha = 1}^d +\frac{u^\alpha_{I + 3 \delta(\alpha) / 2} - +u^\alpha_{I - 3 \delta(\alpha) / 2}}{\Delta^\alpha_{I(\alpha) - 1} ++ \Delta^\alpha_{I(\alpha)} + \Delta^\alpha_{I(\alpha) + 1}} += 0 +``` + +and + ## Matrix representation -We can write the mass and momentum equations in matrix form. The discrete mass -equation then becomes +We can write the mass and momentum equations in matrix form. We will use the +same matrix notation for the second- and fourth order accurate discretizations. +The discrete mass equation becomes + ```math M u_h + y_M = 0, ``` + where ``M`` is the discrete divergence operator and ``y_M`` contains the boundary value contributions of the velocity to the divergence field. From 34b3f56e567fe648c6ca30a3e9bf3e84810ed8d6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Wed, 1 Nov 2023 13:23:45 +0100 Subject: [PATCH 100/379] Update docs --- docs/src/equations/ns.md | 14 ++- docs/src/equations/spatial.md | 172 +++++++++++++++++++--------------- docs/src/features/closure.md | 14 +-- 3 files changed, 116 insertions(+), 84 deletions(-) diff --git a/docs/src/equations/ns.md b/docs/src/equations/ns.md index 769f96646..0a714c6e3 100644 --- a/docs/src/equations/ns.md +++ b/docs/src/equations/ns.md @@ -39,6 +39,7 @@ be denoted ``\partial \mathcal{O}``, with normal ``n`` and surface element The mass equation in integral form is given by ```math +\frac{1}{| \mathcal{O} |} \int_{\partial \mathcal{O}} u \cdot n \, \mathrm{d} \Gamma = 0, ``` @@ -46,9 +47,16 @@ where we have used the divergence theorem to convert the volume integral to a surface integral. Similarly, the momentum equations take the form ```math -\frac{\partial }{\partial t} \int_\mathcal{O} u \, \mathrm{d} \Omega -= \int_{\partial \mathcal{O}} \left( - u u^\mathsf{T} - P + \nu S \right) \cdot n \, -\mathrm{d} \Gamma + \int_\mathcal{O} f \mathrm{d} \Omega +\frac{\mathrm{d}}{\mathrm{d} t} +\frac{1}{| \mathcal{O} |} +\int_\mathcal{O} u \, \mathrm{d} \Omega += +\frac{1}{| \mathcal{O} |} +\int_{\partial \mathcal{O}} +\left( - u u^\mathsf{T} - P + \nu S \right) \cdot n +\, \mathrm{d} \Gamma + +\frac{1}{| \mathcal{O} |} +\int_\mathcal{O} f \mathrm{d} \Omega ``` where ``P = p \mathrm{I}`` is the hydrostatic stress tensor diff --git a/docs/src/equations/spatial.md b/docs/src/equations/spatial.md index 0e730126b..fd54b7808 100644 --- a/docs/src/equations/spatial.md +++ b/docs/src/equations/spatial.md @@ -29,7 +29,7 @@ shape of a box with side lengths ``L^\alpha > 0``. This allows for partitioning x^\alpha_{I(\alpha) + \frac{1}{2}} \right], \quad I \in \mathcal{I}. ``` -Just like ``\Omega`` itself, they represent rectangles in 2D and prisms in 3D. +Just like ``\Omega`` itself, they represent rectangles in 2D and prisms n 3D. They are fully defined by the vectors of volume face coordinates ``x^\alpha = \left( x^\alpha_{i} \right)_{i = 0}^{N(\alpha)} \in \mathbb{R}^{N(\alpha) + 1}``, where ``N = (N(1), \dots, N(d)) \in \mathbb{N}^d`` are the numbers of @@ -54,20 +54,10 @@ We also define the volume widths/depths/heights ``\Delta x^\alpha_i = x^\alpha_{i + \frac{1}{2}} - x^\alpha_{i - \frac{1}{2}}``, where ``i`` can take half values. The volume sizes are thus ``| \Omega_{I} | = \prod_{\alpha = 1}^d \Delta x^\alpha_{I(\alpha)}``. - In addition to the finite volumes and their shifted variants, we -define the surface - -```math -\Gamma^\alpha_I = \prod_{\beta = 1}^d \begin{cases} - \left\{ x^\beta_{I(\beta)} \right\}, & \quad \alpha = \beta \\ - \left[ x^\beta_{I(\beta) - 1 / 2}, x^\beta_{I(\beta) + 1 / 2} \right], & \quad - \text{otherwise}, -\end{cases} -``` -where ``I`` can take integer or half-values. It is the interface between -``\Omega_{I - \delta(\alpha) / 2}`` and ``\Omega_{I + \delta(\alpha) / 2}``, -and has surface normal ``\delta(\alpha)``. +define the interface +``\Gamma^\alpha_I = \Omega_{I - \delta(\alpha) / 2} \cup \Omega_{I + +\delta(\alpha) / 2}``. In each reference finite volume ``\Omega_{I}`` (``I \in \mathcal{I}``), there are three different types of positions in which quantities of interest can be @@ -91,7 +81,6 @@ In 2D, this finite volume configuration is illustrated as follows: ![Grid](../assets/grid.png) - ## Interpolation When a quantity is required *outside* of its native point, we will use interpolation. Examples: @@ -136,20 +125,34 @@ We will consider the integral form of the Navier-Stokes equations. This has the advantage that some of the spatial derivatives dissapear, reducing the amount of finite difference approximations we need to perform. +We define the finite difference operator ``\partial_\alpha`` equivalent to +the continuous operator ``\frac{\partial}{\partial x^\alpha}``. For all fields +discrete fields ``\varphi``, it is given by + +```math +(\partial_\alpha \varphi)_I = \frac{\varphi_{I + \delta(\alpha) / 2} - \varphi_{I - +\delta(\alpha) / 2}}{\Delta^\alpha_{I(\alpha)}}, +``` + +where ``\varphi`` is interpolated first if necessary. + ### Mass equation The mass equation takes the form ```math -\int_{\partial \mathcal{O}} u \cdot n \, \mathrm{d} \Gamma = 0, \quad \forall -\mathcal{O} \subset \Omega. +\frac{1}{| \mathcal{O} |} +\int_{\partial \mathcal{O}} u \cdot n \, \mathrm{d} \Gamma = 0, +\quad \forall \mathcal{O} \subset \Omega. ``` Using the pressure volume ``\mathcal{O} = \Omega_{I}``, we get ```math -\sum_{\alpha = 1}^d \left( \int_{\Gamma^\alpha_{I + \delta(\alpha) / 2}} +\sum_{\alpha = 1}^d +\frac{1}{| \Omega_I |} +\left( \int_{\Gamma^\alpha_{I + \delta(\alpha) / 2}} u^\alpha \, \mathrm{d} \Gamma - \int_{\Gamma_{I - \delta(\alpha) / 2}^\alpha} u^\alpha \, \mathrm{d} \Gamma \right) = 0. @@ -167,13 +170,9 @@ local approximation (quadrature) This yields the discrete mass equation ```math -\sum_{\alpha = 1}^d -\frac{u^\alpha_{I + \delta(\alpha) / 2} - -u^\alpha_{I - \delta(\alpha) / 2}}{\Delta^\alpha_{I(\alpha)}} = 0, +\sum_{\alpha = 1}^d (\partial_\alpha u^\alpha)_{I} = 0. ``` -where we have divided by the volume sizes ``\Omega_I``. - !!! note "Approximation error" For the mass equation, the only approximation we have performed is quadrature. No interpolation or finite difference error is present. @@ -185,14 +184,26 @@ Grouping the convection, pressure gradient, diffusion, and body force terms in each of their own integrals, we get, for all ``\mathcal{O} \subset \Omega``: ```math -\frac{\partial }{\partial t} \int_\mathcal{O} u^\alpha \, \mathrm{d} \Omega +\begin{split} +\frac{\mathrm{d}}{\mathrm{d} t} +\frac{1}{| \mathcal{O} |} +\int_\mathcal{O} u^\alpha \, \mathrm{d} \Omega = -- \sum_{\beta = 1}^d \int_{\partial \mathcal{O}} u^\alpha u^\beta n^\beta \, -\mathrm{d} \Gamma -- \int_{\partial \mathcal{O}} p n^\alpha \, \mathrm{d} \Gamma -+ \nu \sum_{\beta = 1}^d \int_{\partial \mathcal{O}} \frac{\partial -u^\alpha}{\partial x^\beta} n^\beta \, \mathrm{d} \Gamma -+ \int_\mathcal{O} f^\alpha \mathrm{d} \Omega, +& - \sum_{\beta = 1}^d +\frac{1}{| \mathcal{O} |} +\int_{\partial \mathcal{O}} +u^\alpha u^\beta n^\beta +\, \mathrm{d} \Gamma \\ +& + \nu \sum_{\beta = 1}^d +\frac{1}{| \mathcal{O} |} +\int_{\partial \mathcal{O}} +\frac{\partial u^\alpha}{\partial x^\beta} n^\beta +\, \mathrm{d} \Gamma \\ +& + \frac{1}{| \mathcal{O} |} +\int_\mathcal{O} f^\alpha \mathrm{d} \Omega \\ +& - \frac{1}{| \mathcal{O} |} +\int_{\partial \mathcal{O}} p n^\alpha \, \mathrm{d} \Gamma, +\end{split} ``` where ``n = (n^1, \dots, n^d)`` is the surface normal vector to ``\partial @@ -206,7 +217,8 @@ gives ```math \begin{split} - \frac{\partial }{\partial t} + \frac{\mathrm{d}}{\mathrm{d} t} + \frac{1}{| \Omega_{I + \delta(\alpha) / 2} |} \int_{\Omega_{I + \delta(\alpha) / 2}} \! \! \! \! \! \! @@ -215,7 +227,9 @@ gives u^\alpha \, \mathrm{d} \Omega = & - - \sum_{\beta = 1}^d \left( + \sum_{\beta = 1}^d + \frac{1}{| \Omega_{I + \delta(\alpha) / 2} |} + \left( \int_{\Gamma^{\beta}_{I + \delta(\alpha) / 2 + \delta(\beta) / 2}} \! \! \! \! \! \! @@ -231,12 +245,8 @@ gives \! \! \! u^\alpha u^\beta \, \mathrm{d} \Gamma \right) \\ - & - - \left( - \int_{\Gamma^{\alpha}_{I + \delta(\alpha)}} p \, \mathrm{d} \Gamma - - \int_{\Gamma^{\alpha}_{I}} p \, \mathrm{d} \Gamma - \right) \\ & + \nu \sum_{\beta = 1}^d + \frac{1}{| \Omega_{I + \delta(\alpha) / 2} |} \left( \int_{\Gamma^{\beta}_{I + \delta(\alpha) / 2 + \delta(\beta) / 2}} \frac{\partial u^\alpha}{\partial x^\beta} \, \mathrm{d} \Gamma @@ -244,8 +254,15 @@ gives \frac{\partial u^\alpha}{\partial x^\beta} \, \mathrm{d} \Gamma \right) \\ & + + \frac{1}{| \Omega_{I + \delta(\alpha) / 2} |} \int_{\Omega_{I + \delta(\alpha) / 2}} - f^\alpha \, \mathrm{d} \Omega + f^\alpha \, \mathrm{d} \Omega \\ + & - + \frac{1}{| \Omega_{I + \delta(\alpha) / 2} |} + \left( + \int_{\Gamma^{\alpha}_{I + \delta(\alpha)}} p \, \mathrm{d} \Gamma - + \int_{\Gamma^{\alpha}_{I}} p \, \mathrm{d} \Gamma + \right). \end{split} ``` @@ -257,10 +274,7 @@ continuous quantities. 1. The mid-point values of derivatives are approximated using a central-like finite difference: ```math - \frac{\partial u^\alpha}{\partial x^\beta}(x_I) \approx - \frac{u^\alpha_{I + \delta(\beta) / 2} - - u^\alpha_{I - \delta(\beta) / 2}}{x^\beta_{I(\beta) + 1 / 2} - - x^\beta_{I(\beta) - 1 / 2}}. + \frac{\partial u^\alpha}{\partial x^\beta}(x_I) \approx (\partial_\beta u^\alpha)_I ``` 1. Quantities outside their canonical positions are obtained through interpolation. @@ -269,31 +283,16 @@ Finally, the discrete ``\alpha``-momentum equations are given by ```math \begin{split} - & \frac{\mathrm{d} }{\mathrm{d} t} u^\alpha_{I + \delta(\alpha) / 2} = \\ + \frac{\mathrm{d} }{\mathrm{d} t} u^\alpha_{I + \delta(\alpha) / 2} = - & \sum_{\beta = 1}^d - \frac{ - (u^\alpha u^\beta)_{I + \delta(\alpha) / 2 + \delta(\beta) / 2} - - - (u^\alpha u^\beta )_{I + \delta(\alpha) / 2 - \delta(\beta) / 2} - }{\Delta^\beta_{I(\beta) + \delta_{\alpha \beta} / 2}} \\ - - & \frac{p_{I + \delta(\alpha)} - p_{I}}{\Delta^\alpha_{I(\alpha) + 1 / 2}} \\ + (\partial_\beta (u^\alpha u^\beta))_{I + \delta(\alpha) / 2} \\ + & \nu \sum_{\beta = 1}^d - \frac{1}{\Delta^\beta_{I(\beta)+ \delta_{\alpha \beta} / 2}} - \left( - \frac{u^\alpha_{I + \delta(\alpha) / 2 + \delta(\beta)} - u^\alpha_{I + - \delta(\alpha) / 2}}{\Delta^\beta_{I(\beta) + 1 / 2}} - - \frac{ - u^\alpha_{I + \delta(\alpha) / 2} - - u^\alpha_{I + \delta(\alpha) / 2 - \delta(\beta)} - }{\Delta^\beta_{I(\beta) + \delta_{\alpha \beta} / 2 - 1 / 2}} - \right) \\ - + & f^\alpha(x_{I + \delta(\alpha) / 2}), + (\partial_\beta \partial_\beta u^\alpha)_{I + \delta(\alpha) / 2} \\ + + & f^\alpha(x_{I + \delta(\alpha) / 2}, t) + - (\partial_\alpha p)_{I + \delta(\alpha) / 2}. \end{split} ``` -where we have divided each equation by the volume size -``| \Omega_{I + \delta(\alpha) / 2} |``. - ## Boundary conditions Depending on the type of boundary conditions, certain indices used in the left- @@ -386,28 +385,51 @@ A fourth order accurate discretization can be obtained by judiciously combining the second order discretization with itself on a grid with three times larger cells in each dimension [Verstappen2003](@cite) [Sanderse2014](@cite). The coarse discretization is identical, but the mass equation is derived for the -control volume $\bigcup_{\alpha = 1}^d \Omega_{I - \delta(\alpha)} \cup -\Omega_I \cup \Omega_{I + \delta(\alpha)}$, while the momentum equation is -derived for the control volume $\bigcup_{\alpha = 1}^d \Omega_{I - -\delta(\alpha) / 2} \cup \Omega_{I + \delta(\alpha) / 2} \cup \Omega_{I + 3 -\delta(\alpha) / 2}$. The resulting fourth order accurate equations are given -by +three times coarser control volume + +```math +\Omega^3_I = +\bigcup_{\alpha = 1}^d \Omega_{I - \delta(\alpha)} \cup +\Omega_I \cup \Omega_{I + \delta(\alpha)}, +``` +while the momentum equation is +derived for its shifted variant ``\Omega^3_{I + \delta(\alpha) / 2}``. +The resulting fourth order accurate equations are given by ```math \sum_{\alpha = 1}^d -\frac{u^\alpha_{I + \delta(\alpha) / 2} - -u^\alpha_{I - \delta(\alpha) / 2}}{\Delta^\alpha_{I(\alpha)}} +(\partial_\alpha u^\alpha)_I - -\frac{\sum_{e \in \{-1, 0, 1\}^d} | \Omega_{I + e} |}{3^{2 + d} | \Omega_I |} +\frac{| \Omega^3_I |}{3^{2 + d} | \Omega_I |} \sum_{\alpha = 1}^d -\frac{u^\alpha_{I + 3 \delta(\alpha) / 2} - -u^\alpha_{I - 3 \delta(\alpha) / 2}}{\Delta^\alpha_{I(\alpha) - 1} -+ \Delta^\alpha_{I(\alpha)} + \Delta^\alpha_{I(\alpha) + 1}} +(\partial^3_\alpha u^\alpha)_I = 0 ``` and +```math +\begin{split} + \frac{\mathrm{d} }{\mathrm{d} t} u^\alpha_{I + \delta(\alpha) / 2} = + - & \sum_{\beta = 1}^d + (\partial_\beta (u^\alpha u^\beta))_{I + \delta(\alpha) / 2} \\ + + & \nu \sum_{\beta = 1}^d + (\partial_\beta \partial_\beta u^\alpha)_{I + \delta(\alpha) / 2} \\ + + & f^\alpha(x_{I + \delta(\alpha) / 2}, t) + - (\partial_\alpha p)_{I + \delta(\alpha) / 2}, \\ + + & \text{fourth order} +\end{split} +``` + +where + +```math +(\partial^3_\alpha \varphi)_I = +\frac{\varphi_{I + 3 \delta(\alpha) / 2} - +\varphi_{I - 3 \delta(\alpha) / 2}}{\Delta^\alpha_{I(\alpha) - 1} + +\Delta^\alpha_{I(\alpha)} + \Delta^\alpha_{I(\alpha) + 1}}. +``` + ## Matrix representation We can write the mass and momentum equations in matrix form. We will use the @@ -431,7 +453,7 @@ The discrete momentum equations become \end{split} ``` -where ``C`` is the convection operator (including boundary contributions), +where ``C`` is she convection operator (including boundary contributions), ``D`` is the diffusion operator, ``y_D`` is boundary contribution to the diffusion term, ``G = W_u^{-1} M^\mathsf{T} W`` is the pressure gradient diff --git a/docs/src/features/closure.md b/docs/src/features/closure.md index 474de01bb..868d1492e 100644 --- a/docs/src/features/closure.md +++ b/docs/src/features/closure.md @@ -1,8 +1,8 @@ # Neural closure models For [large eddy simulation (LES)](../features/les.md), a closure model is -required. With IncompressibleNavierStokes, a neural closure model can be traine -d on filtered DNS data. The discrete DNS equations are given by +required. With IncompressibleNavierStokes, a neural closure model can be +trained on filtered DNS data. The discrete DNS equations are given by ```math \begin{split} @@ -19,11 +19,13 @@ M \bar{u} & = 0, \\ \frac{\mathrm{d} \bar{u}}{\mathrm{d} t} & = F(\bar{u}) + c - G \bar{p}, \end{split} ``` + where the discretizations ``M``, ``F``, and ``G`` are adapted to the size of -their inputs and ``c = \overline{F(u)} - F(\bar{u})`` is a commutator error. -Replacing ``c`` with a parameterized closure model ``m(\bar{u}, \theta) \approx -c`` gives the LES equations for the approximate large scale velocity ``\bar{v} -\approx \bar{u}`` +their inputs and ``c = \overline{F(u)} - F(\bar{u})`` is a commutator error. We +here assumed that ``M`` and ``\Phi`` commute, which is the case for face +averaging filters. Replacing ``c`` with a parameterized closure model +``m(\bar{u}, \theta) \approx c`` gives the LES equations for the approximate +large scale velocity ``\bar{v} \approx \bar{u}`` ```math \begin{split} From 8a826d6b82db220c0d4aa02d6741afd09a7894fd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Wed, 1 Nov 2023 18:38:24 +0100 Subject: [PATCH 101/379] Update files --- .gitignore | 3 + examples/PlaneJets2D.jl | 29 +++-- examples/TaylorGreenVortex3D.jl | 5 +- scratch/train_model.jl | 170 ++++++++++++++---------------- src/IncompressibleNavierStokes.jl | 1 + src/boundary_conditions.jl | 1 - src/closures/cnn.jl | 34 ++---- src/closures/create_les_data.jl | 158 ++++++++++++++------------- src/closures/fno.jl | 37 ++----- src/create_initial_conditions.jl | 3 +- src/filter.jl | 39 +++++++ src/operators.jl | 16 +-- src/postprocess/plot_vorticity.jl | 4 +- src/processors/real_time_plot.jl | 3 +- 14 files changed, 239 insertions(+), 264 deletions(-) create mode 100644 src/filter.jl diff --git a/.gitignore b/.gitignore index 41f11d726..f30000905 100644 --- a/.gitignore +++ b/.gitignore @@ -6,6 +6,9 @@ /docs/src/generated/ *output/ # scratch/* +test.jl +toto.jl +tata.jl # Vim swap *.swp diff --git a/examples/PlaneJets2D.jl b/examples/PlaneJets2D.jl index dd56846a8..0ec9a24a3 100644 --- a/examples/PlaneJets2D.jl +++ b/examples/PlaneJets2D.jl @@ -42,28 +42,28 @@ Re = T(6_000) ## V() = sqrt(T(467.4)) V() = T(21.619435700313733) -U_A(y) = u() / 2 * (tanh((y + T(0.5)) / T(0.1)) - tanh((y - T(0.5)) / T(0.1))) +U_A(y) = V() / 2 * (tanh((y + T(0.5)) / T(0.1)) - tanh((y - T(0.5)) / T(0.1))) U_B(y) = - u() / 2 * (tanh((y + 1 + T(0.5)) / T(0.1)) - tanh((y + 1 - T(0.5)) / T(0.1))) + - u() / 2 * (tanh((y - 1 + T(0.5)) / T(0.1)) - tanh((y - 1 - T(0.5)) / T(0.1))) + V() / 2 * (tanh((y + 1 + T(0.5)) / T(0.1)) - tanh((y + 1 - T(0.5)) / T(0.1))) + + V() / 2 * (tanh((y - 1 + T(0.5)) / T(0.1)) - tanh((y - 1 - T(0.5)) / T(0.1))) U_C(y) = - u() / 2 * ( + V() / 2 * ( tanh(((y + T(1.0)) / 1 + T(0.5)) / T(0.1)) - tanh(((y + T(1.0)) / 1 - T(0.5)) / T(0.1)) ) + - u() / 4 * ( + V() / 4 * ( tanh(((y - T(1.5)) / 2 + T(0.5)) / T(0.2)) - tanh(((y - T(1.5)) / 2 - T(0.5)) / T(0.2)) ) U_D(y) = - u() / 2 * ( + V() / 2 * ( tanh(((y + T(1.0)) / 1 + T(0.5)) / T(0.1)) - tanh(((y + T(1.0)) / 1 - T(0.5)) / T(0.1)) ) - - u() / 4 * ( + V() / 4 * ( tanh(((y - T(1.5)) / 2 + T(0.5)) / T(0.2)) - tanh(((y - T(1.5)) / 2 - T(0.5)) / T(0.2)) ) @@ -107,24 +107,23 @@ u₀, p₀ = create_initial_conditions( # Real time plot: Streamwise average and spectrum mean_plotter(setup; nupdate = 1) = processor( function (state) - (; indu, yu, yin, Nux_in, Nuy_in) = setup.grid + (; Ip) = setup.grid umean = @lift begin (; u, p, t) = $state + up = IncompressibleNavierStokes.interpolate_u_p(setup, u) u1 = u[1] - sleep(0.001) - reshape(sum(reshape(u1, size(yu)); dims = 1), :) ./ (Nux_in * V()) + reshape(sum(u1[Ip]; dims = 1), :) ./ size(u1, 1) ./ V() end - K = Nux_in ÷ 2 + K = size(Ip, 1) ÷ 2 k = 1:(K-1) # Find energy spectrum where y = 0 - n₀ = Nuy_in ÷ 2 + n₀ = size(Ip, 2) ÷ 2 E₀ = @lift begin - (; V, p, t) = $state - u = V[indu] - u_y = reshape(u, size(yu))[:, n₀] + (; u, p, t) = $state + u_y = u[1][:, n₀] abs.(fft(u_y .^ 2))[k.+1] end diff --git a/examples/TaylorGreenVortex3D.jl b/examples/TaylorGreenVortex3D.jl index 465fbe756..eb6fec39c 100644 --- a/examples/TaylorGreenVortex3D.jl +++ b/examples/TaylorGreenVortex3D.jl @@ -54,8 +54,8 @@ pressure_solver = SpectralPressureSolver(setup); u₀, p₀ = create_initial_conditions( setup, (dim, x, y, z) -> - dim() == 1 ? sinpi(2x) * cospi(2y) * sinpi(2z) : - dim() == 2 ? -cospi(2x) * sinpi(2y) * sinpi(2z) : zero(x); + dim() == 1 ? sinpi(2x) * cospi(2y) * sinpi(2z) / 2 : + dim() == 2 ? -cospi(2x) * sinpi(2y) * sinpi(2z) / 2 : zero(x); pressure_solver, ); u, p = u₀, p₀ @@ -95,7 +95,6 @@ u, p, outputs = solve_unsteady( step_logger(; nupdate = 1), ), pressure_solver, - inplace = true, ); # ## Post-process diff --git a/scratch/train_model.jl b/scratch/train_model.jl index 1cf6731a5..9c9a4d56e 100644 --- a/scratch/train_model.jl +++ b/scratch/train_model.jl @@ -21,16 +21,17 @@ using Random using Zygote # Floating point precision -T = Float32 +T = Float64 -# # To use CPU: Do not move any arrays -# device = identity +# Array type +ArrayType = Array +## using CUDA; ArrayType = CuArray +## using AMDGPU; ArrayType = ROCArray +## using oneAPI; ArrayType = oneArray +## using Metal; ArrayType = MtlArray -# To use GPU, use `cu` to move arrays to the GPU. -# Note: `cu` converts to Float32 -using CUDA using LuxCUDA -device = cu +using CUDA; T = Float32; ArrayType = CuArray; CUDA.allowscalar(false) # Setup n = 128 @@ -51,6 +52,7 @@ ntest = 5 # Create LES data from DNS params = (; + D = 2, Re, lims, nles = n, @@ -58,95 +60,72 @@ params = (; tburn, tsim, Δt = T(1e-4), - device, + ArrayType, ) -data_train = create_les_data(T; params..., nsim = ntrain) -data_valid = create_les_data(T; params..., nsim = nvalid) -data_test = create_les_data(T; params..., nsim = ntest) - -# jldsave("output/filtered/data.jld2"; data_train, data_valid, data_test) - -# Load previous LES data -data_train, data_valid, data_test = load("output/filtered/data.jld2", "data_train", "data_valid", "data_test") +data_train = create_les_data(T; params..., nsim = ntrain); +data_valid = create_les_data(T; params..., nsim = nvalid); +data_test = create_les_data(T; params..., nsim = ntest); + +length(data_train.u) +length(data_train.u[1]) +length(data_train.u[1][1]) +size(data_train.u[1][1][1]) + +o = Observable(data_train.u[1][end][1]) +heatmap(o) +for i = 1:501 + # o[] = data_train.u[1][i][1] + o[] = data_train.cF[1][i][1] + sleep(0.001) +end -nt = size(data_train.V, 2) - 1 +# # Save filtered DNS data +# jldsave("output/forced/data.jld2"; data_train, data_valid, data_test) -size(data_train.V) -size(data_valid.V) -size(data_test.V) +# # Load previous LES data +# data_train, data_valid, data_test = load("output/forced/data.jld2", "data_train", "data_valid", "data_test") -# Inspect data -plot_vorticity(setup, data_valid.V[:, 1, 1], T(0)) -plot_vorticity(setup, data_valid.V[:, end, 1], T(0)) -norm(data_valid.cF[:, 1, 1]) / norm(data_valid.F[:, 1, 1]) -norm(data_valid.cF[:, end, 1]) / norm(data_valid.F[:, end, 1]) +nt = length(data_train.u[1]) - 1 # Uniform periodic grid pressure_solver = SpectralPressureSolver(setup); -q = data_valid.V -# q = data_train.FG -q = q[:, :, 1] -q = selectdim(reshape(q, n, n, 2, :), 3, 1) - -qc = reshape(selectdim(data_valid.cF, 3, 2), n, n, :) +closure, θ₀ = cnn( + setup, -obs = Observable(randn(T, 1, 1)) -# obs = Observable(selectdim(q, 3, 1)) -# obs = Observable([selectdim(q, 3, 1) selectdim(qc, 3, 1)]) -fig = heatmap(obs) -fig + # Radius + [2, 2, 2, 2], -# for snap in eachslice(q; dims = 3) -for (s1, s2) in zip(eachslice(q; dims = 3), eachslice(qc; dims = 3)) - obs[] = s1 - # obs[] = [s1 s2] - autolimits!(fig.axis) - sleep(0.005) -end + # Channels + [5, 5, 5, 2], -heatmap(selectdim(reshape(data_valid.force[:, 1], n, n, 2), 3, 1)) + # Activations + [leakyrelu, leakyrelu, leakyrelu, identity], -fx, fy = eachslice(reshape(data_valid.force[:, 1], n, n, 2); dims = 3) -heatmap(fx) -arrows(x, y, fx, fy; lengthscale = 1.0f0) + # Bias + [true, true, true, false]; +) -# closure, θ₀ = cnn( +# closure, θ₀ = fno( # setup, # -# # Radius -# [2, 2, 2, 2], +# # Cut-off wavenumbers +# [32, 32, 32, 32], # -# # Channels -# [64, 64, 64, 2], +# # Channel sizes +# [24, 12, 8, 8], # -# # Activations -# [leakyrelu, leakyrelu, leakyrelu, identity], +# # Fourier layer activations +# [gelu, gelu, gelu, identity], # -# # Bias -# [true, true, true, false]; -# ) - -closure, θ₀ = fno( - setup, - - # Cut-off wavenumbers - [32, 32, 32, 32], - - # Channel sizes - [24, 12, 8, 8], - - # Fourier layer activations - [gelu, gelu, gelu, identity], - - # Dense activation - gelu, -); +# # Dense activation +# gelu, +# ); @info "Closure model has $(length(θ₀)) parameters" # Test data -V_test = device(reshape(data_test.V[:, 1:20, 1:2], :, 40)) +u_test = device(reshape(data_test.u[:, 1:20, 1:2], :, 40)) c_test = device(reshape(data_test.cF[:, 1:20, 1:2], :, 40)) # Prepare training @@ -177,10 +156,10 @@ first(gradient(randloss, θ)); randloss, opt, θ; - niter = 1000, + niter = 2000, ncallback = 10, callbackstate, - callback = create_callback(closure, V_test, c_test; state = callbackstate), + callback = create_callback(closure, u_test, c_test; state = callbackstate), ) GC.gc() CUDA.reclaim() @@ -188,17 +167,19 @@ CUDA.reclaim() Array(θ) # # Save trained parameters -# jldsave("output/theta.jld2"; θ = Array(θ)) +# jldsave("output/forced/theta_cnn.jld2"; θ = Array(θ)) +# jldsave("output/forced/theta_fno.jld2"; θ = Array(θ)) # # Load trained parameters -# θθ = load("output/theta.jld2") +# θθ = load("output/theta_cnn.jld2") +# θθ = load("output/theta_fno.jld2") # θθ = θθ["θ"] # θθ = cu(θθ) # θ .= θθ relative_error(closure(device(data_train.V[:, 1, :]), θ), device(data_train.cF[:, 1, :])) relative_error(closure(device(data_train.V[:, end, :]), θ), device(data_train.cF[:, end, :])) -relative_error(closure(V_test, θ), c_test) +relative_error(closure(u_test, θ), c_test) function energy_history(setup, state) (; Ωp) = setup.grid @@ -216,16 +197,19 @@ end energy_history_writer(setup; nupdate = 1, kwargs...) = processor(state -> energy_history(setup, state; kwargs...); nupdate) -devsetup = device(setup); +isample = 1 +forcedsetup = (; setup..., force = data_train.force[:, isample]); + +devsetup = device(forcedsetup); V_nm, p_nm, outputs_nm = solve_unsteady( - setup, - data_train.V[:, 1, 1], - data_train.p[:, 1, 1], + forcedsetup, + data_test.V[:, 1, isample], + data_test.p[:, 1, isample], (T(0), tsim); Δt = T(2e-4), processors = ( field_plotter(devsetup; type = heatmap, nupdate = 1), - energy_history_writer(setup), + energy_history_writer(forcedsetup), step_logger(; nupdate = 10), ), pressure_solver, @@ -235,17 +219,17 @@ V_nm, p_nm, outputs_nm = solve_unsteady( ) ehist_nm = outputs_nm[2] -setup_fno = (; setup..., closure_model = V -> closure(V, θ)) +setup_fno = (; forcedsetup..., closure_model = V -> closure(V, θ)) devsetup = device(setup_fno); V_fno, p_fno, outputs_fno = solve_unsteady( setup_fno, - data_train.V[:, 1, 1], - data_train.p[:, 1, 1], + data_test.V[:, 1, isample], + data_test.p[:, 1, isample], (T(0), tsim); Δt = T(2e-4), processors = ( field_plotter(devsetup; type = heatmap, nupdate = 1), - energy_history_writer(setup), + energy_history_writer(forcedsetup), step_logger(; nupdate = 10), ), pressure_solver, @@ -256,11 +240,11 @@ V_fno, p_fno, outputs_fno = solve_unsteady( ehist_fno = outputs_fno[2] state = Observable((; V = data_train.V[:, 1, 1], p = data_train.p[:, 1, 1], t = T(0))) -ehist = energy_history(setup, state) +ehist = energy_history(forcedsetup, state) for i = 2:nt+1 t = (i - 1) / T(nt - 1) * tsim - V = data_train.V[:, i, 1] - p = data_train.p[:, i, 1] + V = data_test.V[:, i, isample] + p = data_test.p[:, i, isample] state[] = (; V, p, t) end ehist @@ -275,8 +259,8 @@ fig save("output/train/energy.png", fig) -V = data_train.V[:, end, 1] -p = data_train.p[:, end, 1] +V = data_train.V[:, end, isample] +p = data_train.p[:, end, isample] relative_error(V_nm, V) relative_error(V_fno, V) diff --git a/src/IncompressibleNavierStokes.jl b/src/IncompressibleNavierStokes.jl index 5a84e8a55..600bc6c95 100644 --- a/src/IncompressibleNavierStokes.jl +++ b/src/IncompressibleNavierStokes.jl @@ -73,6 +73,7 @@ include("processors/animator.jl") # Discrete operators include("operators.jl") +include("filter.jl") # Solvers include("solvers/get_timestep.jl") diff --git a/src/boundary_conditions.jl b/src/boundary_conditions.jl index 9a11262ac..f45054adf 100644 --- a/src/boundary_conditions.jl +++ b/src/boundary_conditions.jl @@ -144,7 +144,6 @@ function apply_bc_u!(::PeriodicBC, u, β, t, setup; atend, kwargs...) else _bc_a!(get_backend(u[1]), WORKGROUP)(u, Val(α), Val(β); ndrange) end - synchronize(get_backend(u[1])) end end diff --git a/src/closures/cnn.jl b/src/closures/cnn.jl index a7148a8b9..842d7dadf 100644 --- a/src/closures/cnn.jl +++ b/src/closures/cnn.jl @@ -2,12 +2,9 @@ cnn(setup, r, c, σ, b; kwargs...) Create CNN closure model. Return a tuple `(closure, θ)` where `θ` are the initial -parameters and `closure(V, θ)` predicts the commutator error. +parameters and `closure(u, θ)` predicts the commutator error. """ -cnn(setup, r, c, σ, b; kwargs...) = cnn(setup.grid.dimension, setup, r, c, σ, b; kwargs...) - function cnn( - ::Dimension{2}, setup, r, c, @@ -17,52 +14,37 @@ function cnn( rng = Random.default_rng(), ) (; grid) = setup - (; Nx, Ny, x) = grid + (; dimension, x) = grid + D = dimension() # For now - T = eltype(x) + T = eltype(x[1]) @assert T == Float32 # Make sure there are two force fields in output - @assert c[end] == 2 + @assert c[end] == D # Add input channel size c = [2; c] # Create convolutional closure model NN = Chain( - # Unflatten and separate u and v velocities - V -> reshape(V, Nx, Ny, 2, :), - - # # uu, uv, vu, vv - # V -> reshape(V, Nx, Ny, 2, 1, :) .* reshape(V, Nx, Ny, 1, 2, :), - # V -> reshape(V, Nx, Ny, 4, :), - # Add padding so that output has same shape as commutator error u -> pad_circular(u, sum(r)), # Some convolutional layers ( - Conv((2r[i] + 1, 2r[i] + 1), c[i] => c[i+1], σ[i]; use_bias = b[i]) for + Conv(ntuple(α -> 2r[i] + 1, D), c[i] => c[i+1], σ[i]; use_bias = b[i]) for i ∈ eachindex(r) )..., - - # Flatten to vector - u -> reshape(u, :, size(u, 4)), ) # Create parameter vector (empty state) params, state = Lux.setup(rng, NN) θ = ComponentArray(params) - """ - closure(V, θ) - - Compute closure term for given parameters `θ`. - """ - function closure end - closure(V, θ) = first(NN(V, θ, state)) - closure(V::AbstractVector, θ) = reshape(closure(reshape(V, :, 1), θ), :) + # Compute closure term for given parameters + closure(u, θ) = first(NN(u, θ, state)) closure, θ end diff --git a/src/closures/create_les_data.jl b/src/closures/create_les_data.jl index 4e5d5d4e2..c97b0d586 100644 --- a/src/closures/create_les_data.jl +++ b/src/closures/create_les_data.jl @@ -32,55 +32,61 @@ function gaussian_force( force end -_filter_saver( - dns, - les, - Ku, - Kp; - nupdate = 1, -) = processor( - function (state) - (; dimension, Ω, x) = dns.grid - D = dimension() - Ωbar = les.grid.Ω - T = eltype(x) - _t = fill(zero(T), 0) - _u = fill(ntuple(α -> zeros(T, 0), D), 0) - _p = fill(zeros(T, 0), 0) - _F = fill(ntuple(α -> zeros(T, 0), D), 0) - _FG = fill(ntuple(α -> zeros(T, 0), D), 0) - _cF = fill(ntuple(α -> zeros(T, 0), D), 0) - _cFG = fill(ntuple(α -> zeros(T, 0), D), 0) - on(state) do (; u, p, t) - ubar = Ku .* u - pbar = Kp * p - F = momentum(u, t, dns) - G = pressuregradient(p, dns) - FG = F .+ G - Fbar = Ku .* F - FGbar = Ku .* FG - FVbar = momentum(ubar, t, les) - GVbar = pressuregradient(pbar, les) - FGVbar = FVbar + GVbar - cF = Fbar .- FVbar - cFG = FGbar .- FGVbar - push!(_t, t) - push!(_u, Array.(ubar)) - push!(_p, Array(pbar)) - push!(_F, Array.(Fbar)) - push!(_FG, Array.(FGbar)) - push!(_cF, Array.(cF)) - push!(_cFG, Array.(cFG)) +_filter_saver(dns, les, comp; nupdate = 1) = processor(function (state) + (; dimension, x) = dns.grid + D = dimension() + F = zero.(state[].u) + # pbar = zero(volume_average(state[].p, les, comp)) + ubar = zero.(face_average(state[].u, les, comp)) + Fbar = zero.(ubar) + Fubar = zero.(ubar) + cF = zero.(ubar) + _t = fill(zero(eltype(x[1])), 0) + _u = fill(Array.(ubar), 0) + # _p = fill(Array.(ubar), 0) + _F = fill(Array.(ubar), 0) + # _FG = fill(Array.(ubar), 0) + _cF = fill(Array.(ubar), 0) + # _cFG = fill(Array.(ubar), 0) + on(state) do (; u, p, t) + face_average!(ubar, u, les, comp) + apply_bc_u!(ubar, t, les) + # pbar = Kp * p + momentum!(F, u, t, dns) + # G = pressuregradient(p, dns) + # FG = F .+ G + face_average!(Fbar, F, les, comp) + # FGbar = Ku .* FG + momentum!(Fubar, ubar, t, les) + # Gubar = pressuregradient(pbar, les) + # FGubar = Fubar + Gubar + for α = 1:D + cF[α] .= Fbar[α] .- Fubar[α] end - state[] = state[] - (; t = _t, u = _u, p = _p, F = _F, FG = _FG, cF = _cF, cFG = _cFG) - end; - nupdate, -) + # cFG = FGbar .- FGVbar + push!(_t, t) + push!(_u, Array.(ubar)) + # push!(_p, Array(pbar)) + push!(_F, Array.(Fubar)) + # push!(_FG, Array.(FGbar)) + push!(_cF, Array.(cF)) + # push!(_cFG, Array.(cFG)) + end + state[] = state[] # Save initial conditions + (; + t = _t, + u = _u, + # p = _p, + F = _F, + # FG = _FG, + cF = _cF, + # cFG = _cFG, + ) +end; nupdate) function create_les_data( T; - dimension, + D = 2, Re = T(2_000), lims = (T(0), T(1)), nles = 64, @@ -91,18 +97,14 @@ function create_les_data( Δt = T(1e-4), ArrayType = Array, ) - D = dimension() ndns = compression * nles xdns = ntuple(α -> LinRange(lims..., ndns + 1), D) - xles = map(x -> x[1:compression:end], xdns) + xles = ntuple(α -> LinRange(lims..., nles + 1), D) # Build setup and assemble operators dns = Setup(xdns...; Re, ArrayType) les = Setup(xles...; Re, ArrayType) - # Filter - (; Ku, Kp) = operator_filter(dns.grid, dns.boundary_conditions, compression) - # Since the grid is uniform and identical for x and y, we may use a specialized # spectral pressure solver pressure_solver = SpectralPressureSolver(dns) @@ -114,13 +116,14 @@ function create_les_data( # Filtered quantities to store (; N) = les.grid filtered = (; - u = zeros(T, N..., D, nt + 1, nsim), - p = zeros(T, N..., nt + 1, nsim), - F = zeros(T, N..., D, nt + 1, nsim), - FG = zeros(T, N..., D, nt + 1, nsim), - cF = zeros(T, N..., D, nt + 1, nsim), - cFG = zeros(T, N..., D, nt + 1, nsim), - force = zeros(T, N..., D, nsim), + Δt, + u = fill(fill(ntuple(α -> zeros(T, N...), D), 0), 0), + # p = fill(fill(zeros(T, N...), 0), 0), + F = fill(fill(ntuple(α -> zeros(T, N...), D), 0), 0), + # FG = fill(fill(ntuple(α -> zeros(T, N...), D), 0), 0), + cF = fill(fill(ntuple(α -> zeros(T, N...), D), 0), 0), + # cFG = fill(fill(ntuple(α -> zeros(T, N...), D), 0), 0), + # force = fill(fill(ntuple(α -> zeros(T, N...), D), 0), 0), ) @info "Generating $(Base.summarysize(filtered) / 1e6) Mb of LES data" @@ -129,15 +132,17 @@ function create_les_data( @info "Generating data for simulation $isim of $nsim" # Initial conditions - u₀, p₀ = random_field(dns; pressure_solver) + u₀, p₀ = random_field(dns, T(0); pressure_solver) # Random body force - force_dns = gaussian_force(xdns...) + - gaussian_force(xdns...) + - # gaussian_force(xdns...) + - # gaussian_force(xdns...) + - gaussian_force(xdns...) - force_les = Ku .* force_dns + # force_dns = + # gaussian_force(xdns...) + + # gaussian_force(xdns...) + + # # gaussian_force(xdns...) + + # # gaussian_force(xdns...) + + # gaussian_force(xdns...) + force_dns = zero.(u₀) + force_les = face_average(force_dns, les, compression) _dns = (; dns..., force = force_dns) _les = (; les..., force = force_les) @@ -163,30 +168,21 @@ function create_les_data( (T(0), tsim); Δt, processors = ( - _filter_saver( - device(_dns), - device(_les), - device(Ku), - device(Kp); - bc_vectors_dns = device(get_bc_vectors(_dns, T(0))), - bc_vectors_les = device(get_bc_vectors(_les, T(0))), - ), + _filter_saver(_dns, _les, compression), step_logger(; nupdate = 10), ), pressure_solver, - inplace = true, - device, ) f = outputs[1] # Store result for current IC - filtered.u[ntuple(α -> :, D + 2)..., isim] = stack(stack.(f.u)) - filtered.p[ntuple(α -> :, D + 1)..., isim] = stack(f.p) - filtered.F[ntuple(α -> :, D + 2)..., isim] = stack(stack.(f.F)) - filtered.FG[ntuple(α -> :, D + 2)..., isim] = stack(stack.(f.FG)) - filtered.cF[ntuple(α -> :, D + 2)..., isim] = stack(stack.(f.cF)) - filtered.cFG[ntuple(α -> :, D + 2)..., isim] = stack(stack.(f.cFG)) - filtered.force[ntuple(α -> :, D + 1)..., isim] = stack.(force_les) + push!(filtered.u, f.u) + # push!(filtered.p, f.p) + push!(filtered.F, f.F) + # push!(filtered.FG, f.FG) + push!(filtered.cF, f.cF) + # push!(filtered.cFG, f.cFG) + # push!(filtered.force, f.force) end filtered diff --git a/src/closures/fno.jl b/src/closures/fno.jl index 790bb7ea1..c9472cde6 100644 --- a/src/closures/fno.jl +++ b/src/closures/fno.jl @@ -6,18 +6,11 @@ initial parameters and `closure(V, θ)` predicts the commutator error. """ function fno(setup, kmax, c, σ, ψ; rng = Random.default_rng(), kwargs...) (; grid) = setup - (; dimension) = grid + (; dimension, N) = grid - N = dimension() + D = dimension() - if N == 2 - (; Nx, Ny) = grid - _nx = (Nx, Ny) - elseif N == 3 - (; Nx, Ny, Nz) = grid - _nx = (Nx, Ny, Nz) - end - @assert all(==(first(_nx)), _nx) + @assert all(==(first(N)), N) # Fourier layers @assert length(kmax) == length(c) == length(σ) @@ -27,44 +20,28 @@ function fno(setup, kmax, c, σ, ψ; rng = Random.default_rng(), kwargs...) # Create FNO closure model NN = Chain( - # Unflatten and separate u and v velocities - V -> reshape(V, _nx..., 2, :), - - # # uu, uv, vu, vv - # V -> reshape(V, Nx, Ny, 2, 1, :) .* reshape(V, Nx, Ny, 1, 2, :), - # V -> reshape(V, Nx, Ny, 4, :), - # Some Fourier layers ( FourierLayer(dimension, kmax[i], c[i] => c[i+1]; σ = σ[i]) for i ∈ eachindex(σ) )..., # Put channels in first dimension - V -> permutedims(V, (N + 1, (1:N)..., N + 2)), + u -> permutedims(u, (D + 1, (1:D)..., D + 2)), # Compress with a final dense layer Dense(c[end] => 2 * c[end], ψ), Dense(2 * c[end] => 2; use_bias = false), # Put channels back after spatial dimensions - u -> permutedims(u, ((2:N+1)..., 1, N + 2)), - - # Flatten to vector - u -> reshape(u, 2 * prod(_nx), :), + u -> permutedims(u, ((2:D+1)..., 1, D + 2)), ) # Create parameter vector (empty state) params, state = Lux.setup(rng, NN) θ = ComponentArray(params) - """ - closure(V, θ) - - Compute closure term for given parameters `θ`. - """ - function closure end - closure(V, θ) = first(NN(V, θ, state)) - closure(V::AbstractVector, θ) = reshape(closure(reshape(V, :, 1), θ), :) + # Compute closure term for given parameters + closure(u, θ) = first(NN(u, θ, state)) closure, θ end diff --git a/src/create_initial_conditions.jl b/src/create_initial_conditions.jl index 7186a22e5..853492726 100644 --- a/src/create_initial_conditions.jl +++ b/src/create_initial_conditions.jl @@ -75,8 +75,9 @@ function create_spectrum(N; A, σ, s, backend) a .*= A / sqrt(τ^2 * 2σ^2) for α = 1:D kα = k[α] - @. a *= exp(-(kα - s)^2 / 2σ^2 - im * τ * rand(T)) + @. a *= randn(T) * exp(-(kα - s)^2 / 2σ^2) end + @. a *= exp(im * τ * rand(T)) for α = 1:D a = cat(a, reverse(a; dims = α); dims = α) end diff --git a/src/filter.jl b/src/filter.jl new file mode 100644 index 000000000..7b81cd2b7 --- /dev/null +++ b/src/filter.jl @@ -0,0 +1,39 @@ +""" + face_average!(v, u, setup, comp) + +Average `u` over volume faces. Put result in `v`. +""" +function face_average!(v, u, setup_les, comp) + (; grid) = setup_les + (; Nu, Iu) = grid + D = length(u) + δ = Offset{D}() + @kernel function _face_average!(v, u, ::Val{α}, face, I0) where {α} + I = @index(Global, Cartesian) + J = I0 + comp * (I - oneunit(I)) + s = zero(eltype(v[α])) + for i in face + s += u[α][J+i] + end + v[α][I0+I] = s / comp^(D - 1) + end + for α = 1:D + ndrange = Nu[α] + I0 = first(Iu[α]) + I0 -= oneunit(I0) + face = CartesianIndices(ntuple(β -> β == α ? (comp:comp) : (1:comp), D)) + _face_average!(get_backend(v[1]), WORKGROUP)(v, u, Val(α), face, I0; ndrange) + end + # synchronize(get_backend(u[1])) + v +end + +face_average(u, setup_les, comp) = face_average!( + ntuple( + α -> KernelAbstractions.zeros(get_backend(u[1]), eltype(u[1]), setup_les.grid.N), + length(u), + ), + u, + setup_les, + comp, +) diff --git a/src/operators.jl b/src/operators.jl index d1b9eb401..f8ed914fa 100644 --- a/src/operators.jl +++ b/src/operators.jl @@ -27,13 +27,15 @@ function divergence!(M, u, setup) I = I + I0 M[I] += (u[α][I] - u[α][I-δ(α)]) / Δ[α][I[α]] end - M .= 0 # All volumes have a right velocity # All volumes have a left velocity except the first one # Start at second volume ndrange = N .- 1 I0 = 2 * oneunit(first(Ip)) + # ndrange = Np + # I0 = first(Ip) I0 -= oneunit(I0) + M .= 0 for α = 1:D _divergence!(get_backend(M), WORKGROUP)(M, u, Val(α), I0; ndrange) end @@ -244,15 +246,7 @@ end Right hand side of momentum equations, excluding pressure gradient. """ -momentum(u, t, setup) = momentum!( - ntuple( - α -> KernelAbstractions.zeros(get_backend(u[1]), typeof(t), setup.grid.N), - length(u), - ), - u, - t, - setup, -) +momentum(u, t, setup) = momentum!(zero.(u), u, t, setup) """ pressuregradient!(G, p, setup) @@ -469,7 +463,7 @@ end Compute the ``D``-field. """ Dfield(p, setup) = Dfield!( - KernelAbstractions.zeros(get_backend(p), eltype(p), setup.grid.N), + zero(p), ntuple( α -> KernelAbstractions.zeros(get_backend(p), eltype(p), setup.grid.N), setup.grid.dimension(), diff --git a/src/postprocess/plot_vorticity.jl b/src/postprocess/plot_vorticity.jl index 3a22474d2..dd5155687 100644 --- a/src/postprocess/plot_vorticity.jl +++ b/src/postprocess/plot_vorticity.jl @@ -10,7 +10,7 @@ plot_vorticity(setup, u; kwargs...) = # 2D version function plot_vorticity(::Dimension{2}, setup, u; kwargs...) - (; grid, boundary_conditions) = setup + (; grid) = setup (; xp, xlims, Ip) = grid T = eltype(xp[1]) @@ -47,7 +47,7 @@ end # 3D version function plot_vorticity(::Dimension{3}, setup, u; kwargs...) - (; grid, boundary_conditions) = setup + (; grid) = setup (; xp) = grid ωp = interpolate_ω_p(setup, vorticity(u, setup)) diff --git a/src/processors/real_time_plot.jl b/src/processors/real_time_plot.jl index 4bc58e5ed..2069634a4 100644 --- a/src/processors/real_time_plot.jl +++ b/src/processors/real_time_plot.jl @@ -41,7 +41,8 @@ function field_plot( state; fieldname = :vorticity, type = heatmap, - sleeptime = 0.001, + # sleeptime = 0.001, + sleeptime = nothing, equal_axis = true, displayfig = true, displayupdates = false, From b5a72c396dca4c02c22285298723684b918fceb6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Wed, 1 Nov 2023 18:38:36 +0100 Subject: [PATCH 102/379] Add spell check --- .github/workflows/SpellCheck.yml | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 .github/workflows/SpellCheck.yml diff --git a/.github/workflows/SpellCheck.yml b/.github/workflows/SpellCheck.yml new file mode 100644 index 000000000..c344f6dda --- /dev/null +++ b/.github/workflows/SpellCheck.yml @@ -0,0 +1,13 @@ +name: Spell Check + +on: [pull_request] + +jobs: + typos-check: + name: Spell Check with Typos + runs-on: ubuntu-latest + steps: + - name: Checkout Actions Repository + uses: actions/checkout@v4 + - name: Check spelling + uses: crate-ci/typos@master From a5577a1fd6e87efd8d4994c0c29de7521a85694b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Wed, 1 Nov 2023 21:13:03 +0100 Subject: [PATCH 103/379] fix(docs): missing docstring --- docs/src/features/closure.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/src/features/closure.md b/docs/src/features/closure.md index 868d1492e..9f99eedd2 100644 --- a/docs/src/features/closure.md +++ b/docs/src/features/closure.md @@ -32,6 +32,10 @@ large scale velocity ``\bar{v} \approx \bar{u}`` M \bar{v} & = 0, \\ \frac{\mathrm{d} \bar{v}}{\mathrm{d} t} & = F(\bar{v}) + m(\bar{v}, \theta) - G \bar{q}. \end{split} + +```@docs +face_average! +``` ``` ## Training From 29f67fad7b9380f4bdb31e8c7176d70f483db96d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Wed, 1 Nov 2023 21:15:11 +0100 Subject: [PATCH 104/379] fix: typos --- docs/src/equations/spatial.md | 2 +- examples/LidDrivenCavity2D.jl | 2 +- src/IncompressibleNavierStokes.jl | 2 +- src/grid/stretched_grid.jl | 2 +- src/processors/animator.jl | 4 ++-- src/processors/processors.jl | 4 ++-- src/solvers/pressure/pressure_poisson.jl | 2 +- src/time_steppers/methods.jl | 2 +- 8 files changed, 10 insertions(+), 10 deletions(-) diff --git a/docs/src/equations/spatial.md b/docs/src/equations/spatial.md index fd54b7808..c420156b4 100644 --- a/docs/src/equations/spatial.md +++ b/docs/src/equations/spatial.md @@ -122,7 +122,7 @@ When a quantity is required *outside* of its native point, we will use interpola ## Finite volume discretization of the Navier-Stokes equations We will consider the integral form of the Navier-Stokes equations. This has the -advantage that some of the spatial derivatives dissapear, reducing the amount +advantage that some of the spatial derivatives disappear, reducing the amount of finite difference approximations we need to perform. We define the finite difference operator ``\partial_\alpha`` equivalent to diff --git a/examples/LidDrivenCavity2D.jl b/examples/LidDrivenCavity2D.jl index 18e6ab107..c38137420 100644 --- a/examples/LidDrivenCavity2D.jl +++ b/examples/LidDrivenCavity2D.jl @@ -134,7 +134,7 @@ u, p, outputs = # We may visualize or export the computed fields `(V, p)` # Export fields to VTK. The file `output/solution.vti` may be opened for -# visulization in [ParaView](https://www.paraview.org/). This is particularly +# visualization in [ParaView](https://www.paraview.org/). This is particularly # useful for inspecting results from 3D simulations. save_vtk(setup, u, p, "output/solution") diff --git a/src/IncompressibleNavierStokes.jl b/src/IncompressibleNavierStokes.jl index 600bc6c95..371509ad9 100644 --- a/src/IncompressibleNavierStokes.jl +++ b/src/IncompressibleNavierStokes.jl @@ -32,7 +32,7 @@ const ⊗ = kron # Easily retrieve value from Val (::Val{x})() where {x} = x -# Boundary condtions +# Boundary conditions include("boundary_conditions.jl") # Grid diff --git a/src/grid/stretched_grid.jl b/src/grid/stretched_grid.jl index 91ef76790..c122d46ed 100644 --- a/src/grid/stretched_grid.jl +++ b/src/grid/stretched_grid.jl @@ -16,7 +16,7 @@ Note that `stretched_grid(a, b, N, s)[n]` corresponds to ``x_{n - 1}``. See also [`cosine_grid`](@ref). """ function stretched_grid(a, b, N, s = 1) - s > 0 || error("The strecth factor must be positive") + s > 0 || error("The stretch factor must be positive") if s ≈ 1 LinRange(a, b, N + 1) else diff --git a/src/processors/animator.jl b/src/processors/animator.jl index de83a3af1..6bd41cbed 100644 --- a/src/processors/animator.jl +++ b/src/processors/animator.jl @@ -10,8 +10,8 @@ of the following extensions: - ".webm" - ".gif" -The plot is determined by a `plotter` processsor. -Addtional `kwargs` are passed to Makie's `VideoStream`. +The plot is determined by a `plotter` processor. +Additional `kwargs` are passed to Makie's `VideoStream`. """ animator(setup, path; nupdate = 1, plotter = field_plotter(setup), kwargs...) = processor( function (state) diff --git a/src/processors/processors.jl b/src/processors/processors.jl index 86585d6d1..378b869dc 100644 --- a/src/processors/processors.jl +++ b/src/processors/processors.jl @@ -24,7 +24,7 @@ function initialize(step_observer) end finalize(s, stepper) = println("The final sum (at time t=$(stepper.t)) is $s") -p = Processor(intialize; finalize, nupdate = 5) +p = Processor(initialize; finalize, nupdate = 5) ``` When solved for 20 time steps from t=0 to t=2 the displayed output is @@ -90,7 +90,7 @@ vtk_writer( coords = (xp, yp, zp) end - # Move arryas to CPU before writing + # Move arrays to CPU before writing V isa Array || (V = Array(V)) p isa Array || (p = Array(p)) diff --git a/src/solvers/pressure/pressure_poisson.jl b/src/solvers/pressure/pressure_poisson.jl index e21626af8..d1215c806 100644 --- a/src/solvers/pressure/pressure_poisson.jl +++ b/src/solvers/pressure/pressure_poisson.jl @@ -80,7 +80,7 @@ function pressure_poisson!(solver::CGPressureSolverManual, p, f) # Initial residual laplacian!(L, p, setup) - # Intialize + # Initialize q .= 0 r .= f .- L ρ_prev = one(T) diff --git a/src/time_steppers/methods.jl b/src/time_steppers/methods.jl index e8020bf54..45cca744d 100644 --- a/src/time_steppers/methods.jl +++ b/src/time_steppers/methods.jl @@ -22,7 +22,7 @@ second order for `θ = 1/2`. The LU decomposition of the LHS matrix is computed every time the time step changes. -Note that, in constrast to explicit methods, the pressure from previous time +Note that, in contrast to explicit methods, the pressure from previous time steps has an influence on the accuracy of the velocity. """ struct AdamsBashforthCrankNicolsonMethod{T,M} <: AbstractODEMethod{T} From a15f75d12c24a78b6d19ce4ac0348985e335e7d6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Wed, 1 Nov 2023 21:24:29 +0100 Subject: [PATCH 105/379] Add compat bounds --- Project.toml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/Project.toml b/Project.toml index 6a281121d..9bce042b5 100644 --- a/Project.toml +++ b/Project.toml @@ -24,10 +24,22 @@ Zygote = "e88e6eb3-aa80-5325-afca-941959d7151f" [compat] Adapt = "3" +ComponentArrays = "0.15" FFTW = "1" IterativeSolvers = "0.9" +KernelAbstractions = "0.9" +Lux = "0.5" Makie = "0.19" +NNlib = "0.9" +Observables = "0.5" +Optimisers = "0.3" WriteVTK = "1" +Zygote = "0.6" +LinearAlgebra = "1" +Printf = "1" +Random = "1" +SparseArrays = "1" +Statistics = "1" julia = "1.7" [extras] From 713f5b1b1056b13a5178894382e60967284a5852 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Wed, 1 Nov 2023 21:33:27 +0100 Subject: [PATCH 106/379] Format files --- examples/Actuator2D.jl | 4 ++-- examples/Actuator3D.jl | 2 +- scratch/train_model.jl | 23 ++++++++----------- src/closures/cnn.jl | 10 +------- src/create_initial_conditions.jl | 13 ++++++++--- src/filter.jl | 3 ++- src/grid/grid.jl | 4 ++-- src/models/viscosity_models.jl | 6 ++--- src/postprocess/plot_pressure.jl | 4 +--- src/postprocess/plot_vorticity.jl | 1 - src/postprocess/save_vtk.jl | 2 +- src/processors/real_time_plot.jl | 10 ++------ .../pressure/pressure_additional_solve.jl | 10 ++++++-- src/solvers/pressure/pressure_poisson.jl | 6 ++++- src/solvers/pressure/pressure_solvers.jl | 21 +++++++++++++---- src/solvers/solve_unsteady.jl | 9 +------- 16 files changed, 64 insertions(+), 64 deletions(-) diff --git a/examples/Actuator2D.jl b/examples/Actuator2D.jl index f2e120c12..bccc53977 100644 --- a/examples/Actuator2D.jl +++ b/examples/Actuator2D.jl @@ -69,7 +69,8 @@ u, p = u₀, p₀ u, p, outputs = solve_unsteady( setup, ## u₀, p₀, - u, p, + u, + p, (0.0, 12.0); method = RK44P2(), Δt = 0.05, @@ -84,7 +85,6 @@ u, p, outputs = solve_unsteady( ), ); - # ## Post-process # # We may visualize or export the computed fields `(V, p)`. diff --git a/examples/Actuator3D.jl b/examples/Actuator3D.jl index b444decc3..3da5aa49a 100644 --- a/examples/Actuator3D.jl +++ b/examples/Actuator3D.jl @@ -97,7 +97,7 @@ u, p, outputs = solve_unsteady( ## vtk_writer(setup; nupdate = 2, dir = "output/$name", filename = "solution"), ## field_saver(setup; nupdate = 10), step_logger(; nupdate = 1), - ); + ), ); # ## Post-process diff --git a/scratch/train_model.jl b/scratch/train_model.jl index 9c9a4d56e..a3d2edbdc 100644 --- a/scratch/train_model.jl +++ b/scratch/train_model.jl @@ -31,7 +31,10 @@ ArrayType = Array ## using Metal; ArrayType = MtlArray using LuxCUDA -using CUDA; T = Float32; ArrayType = CuArray; CUDA.allowscalar(false) +using CUDA; +T = Float32; +ArrayType = CuArray; +CUDA.allowscalar(false); # Setup n = 128 @@ -51,17 +54,8 @@ nvalid = 2 ntest = 5 # Create LES data from DNS -params = (; - D = 2, - Re, - lims, - nles = n, - compression = 4, - tburn, - tsim, - Δt = T(1e-4), - ArrayType, -) +params = + (; D = 2, Re, lims, nles = n, compression = 4, tburn, tsim, Δt = T(1e-4), ArrayType) data_train = create_les_data(T; params..., nsim = ntrain); data_valid = create_les_data(T; params..., nsim = nvalid); data_test = create_les_data(T; params..., nsim = ntest); @@ -178,7 +172,10 @@ Array(θ) # θ .= θθ relative_error(closure(device(data_train.V[:, 1, :]), θ), device(data_train.cF[:, 1, :])) -relative_error(closure(device(data_train.V[:, end, :]), θ), device(data_train.cF[:, end, :])) +relative_error( + closure(device(data_train.V[:, end, :]), θ), + device(data_train.cF[:, end, :]), +) relative_error(closure(u_test, θ), c_test) function energy_history(setup, state) diff --git a/src/closures/cnn.jl b/src/closures/cnn.jl index 842d7dadf..f03c78490 100644 --- a/src/closures/cnn.jl +++ b/src/closures/cnn.jl @@ -4,15 +4,7 @@ Create CNN closure model. Return a tuple `(closure, θ)` where `θ` are the initial parameters and `closure(u, θ)` predicts the commutator error. """ -function cnn( - setup, - r, - c, - σ, - b; - channel_augmenter = identity, - rng = Random.default_rng(), -) +function cnn(setup, r, c, σ, b; channel_augmenter = identity, rng = Random.default_rng()) (; grid) = setup (; dimension, x) = grid D = dimension() diff --git a/src/create_initial_conditions.jl b/src/create_initial_conditions.jl index 853492726..a4b3f99ef 100644 --- a/src/create_initial_conditions.jl +++ b/src/create_initial_conditions.jl @@ -28,7 +28,10 @@ function create_initial_conditions( # Initial velocities for α = 1:D - xin = ntuple(β -> reshape(α == β ? x[β][2:end] : xp[β], ntuple(Returns(1), β - 1)..., :), D) + xin = ntuple( + β -> reshape(α == β ? x[β][2:end] : xp[β], ntuple(Returns(1), β - 1)..., :), + D, + ) u[α][Iu[α]] .= initial_velocity.(Val(α), xin...)[Iu[α]] end @@ -66,7 +69,10 @@ function create_spectrum(N; A, σ, s, backend) T = typeof(A) D = length(N) K = N .÷ 2 - k = ntuple(α -> reshape(1:K[α], ntuple(Returns(1), α - 1)..., :, ntuple(Returns(1), D-α)...), D) + k = ntuple( + α -> reshape(1:K[α], ntuple(Returns(1), α - 1)..., :, ntuple(Returns(1), D - α)...), + D, + ) a = KernelAbstractions.ones(backend, Complex{T}, K) AT = typeof(a) # k = AT.(Array{Complex{T}}.(k)) @@ -101,7 +107,8 @@ Create random field. - `s` Wavenumber offset before energy starts decaying """ function random_field( - setup, t; + setup, + t; A = convert(eltype(setup.grid.x[1]), setup.grid.N[1] * 7_500), σ = convert(eltype(setup.grid.x[1]), 30), s = convert(eltype(setup.grid.x[1]), 5), diff --git a/src/filter.jl b/src/filter.jl index 7b81cd2b7..f93b0e4a2 100644 --- a/src/filter.jl +++ b/src/filter.jl @@ -30,7 +30,8 @@ end face_average(u, setup_les, comp) = face_average!( ntuple( - α -> KernelAbstractions.zeros(get_backend(u[1]), eltype(u[1]), setup_les.grid.N), + α -> + KernelAbstractions.zeros(get_backend(u[1]), eltype(u[1]), setup_les.grid.N), length(u), ), u, diff --git a/src/grid/grid.jl b/src/grid/grid.jl index acb983a9a..9382e93ab 100644 --- a/src/grid/grid.jl +++ b/src/grid/grid.jl @@ -66,12 +66,12 @@ function Grid(x, boundary_conditions; ArrayType = Array) # Infinitely thin widths are set to `eps(T)` to avoid division by zero Δ = ntuple(D) do d Δ = diff(x[d]) - Δ[Δ .== 0] .= eps(eltype(Δ)) + Δ[Δ.==0] .= eps(eltype(Δ)) Δ end Δu = ntuple(D) do d Δu = push!(diff(xp[d]), Δ[d][end] / 2) - Δu[Δu .== 0] .= eps(eltype(Δu)) + Δu[Δu.==0] .= eps(eltype(Δu)) Δu end diff --git a/src/models/viscosity_models.jl b/src/models/viscosity_models.jl index 17c0db8f3..fbe67fa18 100644 --- a/src/models/viscosity_models.jl +++ b/src/models/viscosity_models.jl @@ -12,8 +12,7 @@ Laminar model. This model assumes that there are no sub-grid stresses. It can be used if the grid is sufficiently refined for the given flow. It has the advantage of having a constant diffusion operator. """ -struct LaminarModel <: AbstractViscosityModel -end +struct LaminarModel <: AbstractViscosityModel end """ MixingLengthModel() @@ -38,5 +37,4 @@ end QR-model. """ -struct QRModel{T} <: AbstractViscosityModel -end +struct QRModel{T} <: AbstractViscosityModel end diff --git a/src/postprocess/plot_pressure.jl b/src/postprocess/plot_pressure.jl index d734f1984..f098a6618 100644 --- a/src/postprocess/plot_pressure.jl +++ b/src/postprocess/plot_pressure.jl @@ -32,9 +32,7 @@ function plot_pressure(::Dimension{2}, setup, p; kwargs...) ylabel = "y", ) limits!(ax, xlims[1]..., xlims[2]...) - cf = contourf!(ax, xp..., p; extendlow = :auto, extendhigh = :auto, - levels, - kwargs...) + cf = contourf!(ax, xp..., p; extendlow = :auto, extendhigh = :auto, levels, kwargs...) Colorbar(fig[1, 2], cf) # save("output/pressure.png", fig, pt_per_unit = 2) diff --git a/src/postprocess/plot_vorticity.jl b/src/postprocess/plot_vorticity.jl index dd5155687..2e8588bde 100644 --- a/src/postprocess/plot_vorticity.jl +++ b/src/postprocess/plot_vorticity.jl @@ -57,7 +57,6 @@ function plot_vorticity(::Dimension{3}, setup, u; kwargs...) μ, σ = mean(qp), std(qp) levels = LinRange(μ - 3σ, μ + 3σ, 10) - xp = Array.(xp) qp = Array(qp) contour(xp..., qp; levels, kwargs...) diff --git a/src/postprocess/save_vtk.jl b/src/postprocess/save_vtk.jl index 91bfe149a..325f4d601 100644 --- a/src/postprocess/save_vtk.jl +++ b/src/postprocess/save_vtk.jl @@ -21,7 +21,7 @@ function save_vtk(setup, u, p, filename = "output/solution") # ParaView prefers 3D vectors. Add zero z-component. up3 = zero(up[1]) up = (up..., up3) - ωp = Array(ωp) + ωp = Array(ωp) else ωp = Array.(ωp) end diff --git a/src/processors/real_time_plot.jl b/src/processors/real_time_plot.jl index 2069634a4..14dc43ef9 100644 --- a/src/processors/real_time_plot.jl +++ b/src/processors/real_time_plot.jl @@ -109,13 +109,7 @@ function field_plot( fig = Figure(; resolution) if type ∈ (heatmap, image) - ax, hm = type( - fig[1, 1], - xf..., - field; - colorrange = lims, - kwargs..., - ) + ax, hm = type(fig[1, 1], xf..., field; colorrange = lims, kwargs...) elseif type ∈ (contour, contourf) ax, hm = type( fig[1, 1], @@ -215,7 +209,7 @@ function field_plot( fig = Figure(; resolution) # ax = Axis3(fig[1, 1]; title = titlecase(string(fieldname)), aspect...) hm = contour( - fig[1,1], + fig[1, 1], # ax, xf..., field; diff --git a/src/solvers/pressure/pressure_additional_solve.jl b/src/solvers/pressure/pressure_additional_solve.jl index 31449d780..046de7e29 100644 --- a/src/solvers/pressure/pressure_additional_solve.jl +++ b/src/solvers/pressure/pressure_additional_solve.jl @@ -38,8 +38,14 @@ field, resulting in same order pressure as velocity. function pressure_additional_solve(pressure_solver, u, t, setup) D = setup.grid.dimension() p = KernelAbstractions.zeros(get_backend(u[1]), eltype(u[1]), setup.grid.N) - F = ntuple(α -> KernelAbstractions.zeros(get_backend(u[1]), eltype(u[1]), setup.grid.N), D) - G = ntuple(α -> KernelAbstractions.zeros(get_backend(u[1]), eltype(u[1]), setup.grid.N), D) + F = ntuple( + α -> KernelAbstractions.zeros(get_backend(u[1]), eltype(u[1]), setup.grid.N), + D, + ) + G = ntuple( + α -> KernelAbstractions.zeros(get_backend(u[1]), eltype(u[1]), setup.grid.N), + D, + ) M = KernelAbstractions.zeros(get_backend(u[1]), eltype(u[1]), setup.grid.N) pressure_additional_solve!(pressure_solver, u, p, t, setup, F, G, M) end diff --git a/src/solvers/pressure/pressure_poisson.jl b/src/solvers/pressure/pressure_poisson.jl index d1215c806..d4f5f23b8 100644 --- a/src/solvers/pressure/pressure_poisson.jl +++ b/src/solvers/pressure/pressure_poisson.jl @@ -70,7 +70,11 @@ function pressure_poisson!(solver::CGPressureSolverManual, p, f) # d = zero(eltype(a)) I0 = first(Ip) I0 -= oneunit(I0) - d = KernelAbstractions.zeros(get_backend(a), eltype(a), ntuple(Returns(1), length(I0))) + d = KernelAbstractions.zeros( + get_backend(a), + eltype(a), + ntuple(Returns(1), length(I0)), + ) innerdot!(get_backend(a), WORKGROUP)(d, a, b, I0; ndrange = Np) d[] end diff --git a/src/solvers/pressure/pressure_solvers.jl b/src/solvers/pressure/pressure_solvers.jl index 9ffba09a5..2d40f740f 100644 --- a/src/solvers/pressure/pressure_solvers.jl +++ b/src/solvers/pressure/pressure_solvers.jl @@ -92,7 +92,7 @@ function create_laplace_diag(setup) ndrange = Np I0 = first(Ip) I0 -= oneunit(I0) - function laplace_diag(z, p) + function laplace_diag(z, p) _laplace_diag!(get_backend(z), WORKGROUP)(z, p, I0; ndrange) # synchronize(get_backend(z)) end @@ -128,7 +128,8 @@ CGPressureSolverManual( preconditioner, ) -struct SpectralPressureSolver{T,A<:AbstractArray{Complex{T}},S,P} <: AbstractPressureSolver{T} +struct SpectralPressureSolver{T,A<:AbstractArray{Complex{T}},S,P} <: + AbstractPressureSolver{T} setup::S Ahat::A phat::A @@ -138,8 +139,12 @@ end # This moves all the inner arrays to the GPU when calling # `cu(::SpectralPressureSolver)` from CUDA.jl -Adapt.adapt_structure(to, s::SpectralPressureSolver) = - SpectralPressureSolver(adapt(to, s.Ahat), adapt(to, s.phat), adapt(to, s.fhat), adapt(to, s.plan)) +Adapt.adapt_structure(to, s::SpectralPressureSolver) = SpectralPressureSolver( + adapt(to, s.Ahat), + adapt(to, s.phat), + adapt(to, s.fhat), + adapt(to, s.plan), +) """ SpectralPressureSolver(setup) @@ -196,5 +201,11 @@ function SpectralPressureSolver(setup) fhat = zero(Ahat) plan = plan_fft(fhat) - SpectralPressureSolver{T,typeof(Ahat),typeof(setup),typeof(plan)}(setup, Ahat, phat, fhat, plan) + SpectralPressureSolver{T,typeof(Ahat),typeof(setup),typeof(plan)}( + setup, + Ahat, + phat, + fhat, + plan, + ) end diff --git a/src/solvers/solve_unsteady.jl b/src/solvers/solve_unsteady.jl index 2f3cd2ae1..a3d32dddf 100644 --- a/src/solvers/solve_unsteady.jl +++ b/src/solvers/solve_unsteady.jl @@ -67,14 +67,7 @@ function solve_unsteady( end # Time stepper - stepper = create_stepper( - method; - setup, - pressure_solver, - u = u₀, - p = p₀, - t = t_start, - ) + stepper = create_stepper(method; setup, pressure_solver, u = u₀, p = p₀, t = t_start) # Get initial time step isadaptive && (Δt = get_timestep(stepper, cfl)) From 3a877c28d705754081c9f9ee4b669fff3ab65322 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Wed, 1 Nov 2023 21:37:41 +0100 Subject: [PATCH 107/379] Remove non-existing symbols --- src/IncompressibleNavierStokes.jl | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/IncompressibleNavierStokes.jl b/src/IncompressibleNavierStokes.jl index 371509ad9..7b79af094 100644 --- a/src/IncompressibleNavierStokes.jl +++ b/src/IncompressibleNavierStokes.jl @@ -102,9 +102,6 @@ include("closures/create_les_data.jl") # Boundary conditions export PeriodicBC, DirichletBC, SymmetricBC, PressureBC -# Force -export SteadyBodyForce - # Models export AbstractViscosityModel, LaminarModel, MixingLengthModel, SmagorinskyModel, QRModel export NoRegConvectionModel, C2ConvectionModel, C4ConvectionModel, LerayConvectionModel @@ -132,7 +129,7 @@ export momentum, divergence, pressuregradient, Dfield!, Qfield! # Problems export solve_unsteady, solve_steady_state -export create_initial_conditions, random_field, get_velocity +export create_initial_conditions, random_field export plot_force, plot_grid, plot_pressure, plot_streamfunction, plot_velocity, plot_vorticity, save_vtk From 91ee2643fb5bf69776d2361923a8100ea2a637c3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Wed, 1 Nov 2023 21:46:28 +0100 Subject: [PATCH 108/379] Format Project.toml --- Project.toml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Project.toml b/Project.toml index 9bce042b5..8061c4ca0 100644 --- a/Project.toml +++ b/Project.toml @@ -28,18 +28,18 @@ ComponentArrays = "0.15" FFTW = "1" IterativeSolvers = "0.9" KernelAbstractions = "0.9" +LinearAlgebra = "1" Lux = "0.5" Makie = "0.19" NNlib = "0.9" Observables = "0.5" Optimisers = "0.3" -WriteVTK = "1" -Zygote = "0.6" -LinearAlgebra = "1" Printf = "1" Random = "1" SparseArrays = "1" Statistics = "1" +WriteVTK = "1" +Zygote = "0.6" julia = "1.7" [extras] From 4cb3bc0d190a19e88a600e9d096d2d4e6d8d85fb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Thu, 2 Nov 2023 09:45:38 +0100 Subject: [PATCH 109/379] Remove type piracy --- src/IncompressibleNavierStokes.jl | 4 ++-- src/boundary_conditions.jl | 2 +- src/create_initial_conditions.jl | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/IncompressibleNavierStokes.jl b/src/IncompressibleNavierStokes.jl index 7b79af094..738a08a91 100644 --- a/src/IncompressibleNavierStokes.jl +++ b/src/IncompressibleNavierStokes.jl @@ -29,8 +29,8 @@ const WORKGROUP = 64 # Convenience notation const ⊗ = kron -# Easily retrieve value from Val -(::Val{x})() where {x} = x +# # Easily retrieve value from Val +# (::Val{x})() where {x} = x # Boundary conditions include("boundary_conditions.jl") diff --git a/src/boundary_conditions.jl b/src/boundary_conditions.jl index f45054adf..2da7151f1 100644 --- a/src/boundary_conditions.jl +++ b/src/boundary_conditions.jl @@ -191,7 +191,7 @@ function apply_bc_u!(bc::DirichletBC, u, β, t, setup; atend, dudt = false, kwar ), D, ) - u[α][I] .= bcfunc.(Val(α), xI..., t) + u[α][I] .= bcfunc.((Dimension(α),), xI..., t) end end diff --git a/src/create_initial_conditions.jl b/src/create_initial_conditions.jl index a4b3f99ef..e90a8869a 100644 --- a/src/create_initial_conditions.jl +++ b/src/create_initial_conditions.jl @@ -8,7 +8,7 @@ Create initial vectors `(u, p)` at starting time `t`. The initial conditions of `u[α]` are specified by the function -`initial_velocity(Val(α), x...)`. +`initial_velocity(Dimension(α), x...)`. """ function create_initial_conditions( setup, @@ -32,7 +32,7 @@ function create_initial_conditions( β -> reshape(α == β ? x[β][2:end] : xp[β], ntuple(Returns(1), β - 1)..., :), D, ) - u[α][Iu[α]] .= initial_velocity.(Val(α), xin...)[Iu[α]] + u[α][Iu[α]] .= initial_velocity.((Dimension(α),), xin...)[Iu[α]] end apply_bc_u!(u, t, setup) From 95eb8202f16337b5cc077a934103121b6483175a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Thu, 2 Nov 2023 14:38:18 +0100 Subject: [PATCH 110/379] fix: docs --- docs/src/features/closure.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/src/features/closure.md b/docs/src/features/closure.md index 9f99eedd2..03ad41823 100644 --- a/docs/src/features/closure.md +++ b/docs/src/features/closure.md @@ -32,11 +32,11 @@ large scale velocity ``\bar{v} \approx \bar{u}`` M \bar{v} & = 0, \\ \frac{\mathrm{d} \bar{v}}{\mathrm{d} t} & = F(\bar{v}) + m(\bar{v}, \theta) - G \bar{q}. \end{split} +``` ```@docs face_average! ``` -``` ## Training From f58dc632cbbb2b119e57215883685c98a70256b8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Fri, 3 Nov 2023 20:26:54 +0100 Subject: [PATCH 111/379] Update files --- scratch/train_model.jl | 58 ++++++++++++++++++------------- src/closures/create_les_data.jl | 17 ++++----- src/closures/training.jl | 6 ++-- src/operators.jl | 28 +++++++-------- src/postprocess/plot_velocity.jl | 4 +-- src/postprocess/plot_vorticity.jl | 4 +-- src/postprocess/save_vtk.jl | 4 +-- src/processors/real_time_plot.jl | 14 ++++---- 8 files changed, 71 insertions(+), 64 deletions(-) diff --git a/scratch/train_model.jl b/scratch/train_model.jl index a3d2edbdc..84e87fd80 100644 --- a/scratch/train_model.jl +++ b/scratch/train_model.jl @@ -68,8 +68,8 @@ size(data_train.u[1][1][1]) o = Observable(data_train.u[1][end][1]) heatmap(o) for i = 1:501 - # o[] = data_train.u[1][i][1] - o[] = data_train.cF[1][i][1] + o[] = data_train.u[1][i][1] + # o[] = data_train.cF[1][i][1] sleep(0.001) end @@ -98,7 +98,7 @@ closure, θ₀ = cnn( # Bias [true, true, true, false]; -) +); # closure, θ₀ = fno( # setup, @@ -116,28 +116,38 @@ closure, θ₀ = cnn( # gelu, # ); -@info "Closure model has $(length(θ₀)) parameters" +closure.NN + +# Create input/output arrays +function create_io_arrays(data, setup) + nsample = length(data.u) + nt = length(data.u[1]) - 1 + D = setup.grid.dimension() + T = eltype(data.u[1][1][1]) + (; N) = setup.grid + u = zeros(T, (N .- 2)..., D, nt + 1, nsample) + c = zeros(T, (N .- 2)..., D, nt + 1, nsample) + ifield = ntuple(Returns(:), D) + for i = 1:nsample, j = 1:nt+1, α = 1:D + copyto!(view(u, ifield..., α, j, i), view(data.u[i][j][α], setup.grid.Iu[α])) + copyto!(view(c, ifield..., α, j, i), view(data.cF[i][j][α], setup.grid.Iu[α])) + end + reshape(u, (N .- 2)..., D, :), reshape(c, (N .- 2)..., D, :) +end -# Test data -u_test = device(reshape(data_test.u[:, 1:20, 1:2], :, 40)) -c_test = device(reshape(data_test.cF[:, 1:20, 1:2], :, 40)) +io_train = create_io_arrays(data_train, setup) +io_valid = create_io_arrays(data_valid, setup) +io_test = create_io_arrays(data_test, setup) # Prepare training -θ = 1.0f-1 * device(θ₀) -# θ = device(θ₀) +θ = 1.0f-1 * cu(θ₀) +# θ = cu(θ₀) opt = Optimisers.setup(Adam(1.0f-3), θ) callbackstate = Point2f[] -randloss = create_randloss( - mean_squared_error, - closure, - data_train.V, - data_train.cF; - nuse = 50, - device, -) +randloss = create_randloss(mean_squared_error, closure, io_train...; nuse = 50, device = cu) # Warm-up -randloss(θ); +randloss(θ) @time randloss(θ); first(gradient(randloss, θ)); @time first(gradient(randloss, θ)); @@ -153,23 +163,21 @@ first(gradient(randloss, θ)); niter = 2000, ncallback = 10, callbackstate, - callback = create_callback(closure, u_test, c_test; state = callbackstate), -) + callback = create_callback(closure, cu(io_valid)...; state = callbackstate), +); GC.gc() CUDA.reclaim() Array(θ) # # Save trained parameters -# jldsave("output/forced/theta_cnn.jld2"; θ = Array(θ)) -# jldsave("output/forced/theta_fno.jld2"; θ = Array(θ)) +# jldsave("output/forced/theta_cnn.jld2"; theta = Array(θ)) +# jldsave("output/forced/theta_fno.jld2"; theta = Array(θ)) # # Load trained parameters # θθ = load("output/theta_cnn.jld2") # θθ = load("output/theta_fno.jld2") -# θθ = θθ["θ"] -# θθ = cu(θθ) -# θ .= θθ +# copyto!(θ, θθ["theta"]) relative_error(closure(device(data_train.V[:, 1, :]), θ), device(data_train.cF[:, 1, :])) relative_error( diff --git a/src/closures/create_les_data.jl b/src/closures/create_les_data.jl index c97b0d586..6e12f9261 100644 --- a/src/closures/create_les_data.jl +++ b/src/closures/create_les_data.jl @@ -67,7 +67,7 @@ _filter_saver(dns, les, comp; nupdate = 1) = processor(function (state) push!(_t, t) push!(_u, Array.(ubar)) # push!(_p, Array(pbar)) - push!(_F, Array.(Fubar)) + # push!(_F, Array.(Fubar)) # push!(_FG, Array.(FGbar)) push!(_cF, Array.(cF)) # push!(_cFG, Array.(cFG)) @@ -77,7 +77,7 @@ _filter_saver(dns, les, comp; nupdate = 1) = processor(function (state) t = _t, u = _u, # p = _p, - F = _F, + # F = _F, # FG = _FG, cF = _cF, # cFG = _cFG, @@ -119,17 +119,18 @@ function create_les_data( Δt, u = fill(fill(ntuple(α -> zeros(T, N...), D), 0), 0), # p = fill(fill(zeros(T, N...), 0), 0), - F = fill(fill(ntuple(α -> zeros(T, N...), D), 0), 0), + # F = fill(fill(ntuple(α -> zeros(T, N...), D), 0), 0), # FG = fill(fill(ntuple(α -> zeros(T, N...), D), 0), 0), cF = fill(fill(ntuple(α -> zeros(T, N...), D), 0), 0), # cFG = fill(fill(ntuple(α -> zeros(T, N...), D), 0), 0), # force = fill(fill(ntuple(α -> zeros(T, N...), D), 0), 0), ) - @info "Generating $(Base.summarysize(filtered) / 1e6) Mb of LES data" + # @info "Generating $(Base.summarysize(filtered) / 1e6) Mb of LES data" + @info "Generating $(nsim * (nt + 1) * nles * 3 * 2 * length(bitstring(zero(T))) / 8 / 1e6) Mb of LES data" for isim = 1:nsim - @info "Generating data for simulation $isim of $nsim" + # @info "Generating data for simulation $isim of $nsim" # Initial conditions u₀, p₀ = random_field(dns, T(0); pressure_solver) @@ -155,7 +156,7 @@ function create_les_data( p₀, (T(0), tburn); Δt, - processors = (step_logger(; nupdate = 10),), + # processors = (step_logger(; nupdate = 10),), pressure_solver, ) @@ -169,7 +170,7 @@ function create_les_data( Δt, processors = ( _filter_saver(_dns, _les, compression), - step_logger(; nupdate = 10), + # step_logger(; nupdate = 10), ), pressure_solver, ) @@ -178,7 +179,7 @@ function create_les_data( # Store result for current IC push!(filtered.u, f.u) # push!(filtered.p, f.p) - push!(filtered.F, f.F) + # push!(filtered.F, f.F) # push!(filtered.FG, f.FG) push!(filtered.cF, f.cF) # push!(filtered.cFG, f.cFG) diff --git a/src/closures/training.jl b/src/closures/training.jl index d7cda16a7..b0d3f0495 100644 --- a/src/closures/training.jl +++ b/src/closures/training.jl @@ -43,10 +43,8 @@ The function `loss` should take inputs like `loss(f, x, y, θ)`. The batch is moved to `device` before the loss is evaluated. """ -function create_randloss(loss, f, x, y; nuse = size(x, 2), device = identity) - x = reshape(x, size(x, 1), :) - y = reshape(y, size(y, 1), :) - nsample = size(x, 2) +function create_randloss(loss, f, x, y; nuse = 50, device = identity) + nsample = size(x)[end] d = ndims(x) function randloss(θ) i = Zygote.@ignore sort(shuffle(1:nsample)[1:nuse]) diff --git a/src/operators.jl b/src/operators.jl index f8ed914fa..edc781225 100644 --- a/src/operators.jl +++ b/src/operators.jl @@ -317,25 +317,25 @@ function laplacian!(L, p, setup) end """ - interpolate_u_p(setup, u) + interpolate_u_p(u, setup) Interpolate velocity to pressure points. """ -interpolate_u_p(setup, u) = interpolate_u_p!( - setup, +interpolate_u_p(u, setup) = interpolate_u_p!( ntuple( α -> KernelAbstractions.zeros(get_backend(u[1]), eltype(u[1]), setup.grid.N), setup.grid.dimension(), ), u, + setup, ) """ - interpolate_u_p!(setup, up, u) + interpolate_u_p!(up, u, setup) Interpolate velocity to pressure points. """ -function interpolate_u_p!(setup, up, u) +function interpolate_u_p!(up, u, setup) (; boundary_conditions, grid, Re, bodyforce) = setup (; dimension, Np, Ip) = grid D = dimension() @@ -354,12 +354,11 @@ function interpolate_u_p!(setup, up, u) end """ - interpolate_ω_p(setup, ω) + interpolate_ω_p(ω, setup) Interpolate vorticity to pressure points. """ -interpolate_ω_p(setup, ω) = interpolate_ω_p!( - setup, +interpolate_ω_p(ω, setup) = interpolate_ω_p!( setup.grid.dimension() == 2 ? KernelAbstractions.zeros(get_backend(ω), eltype(ω), setup.grid.N) : ntuple( @@ -367,16 +366,17 @@ interpolate_ω_p(setup, ω) = interpolate_ω_p!( setup.grid.dimension(), ), ω, + setup, ) """ - interpolate_ω_p!(setup, ωp, ω) + interpolate_ω_p!(ωp, ω, setup) Interpolate vorticity to pressure points. """ -interpolate_ω_p!(setup, ωp, ω) = interpolate_ω_p!(setup.grid.dimension, setup, ωp, ω) +interpolate_ω_p!(ωp, ω, setup) = interpolate_ω_p!(setup.grid.dimension, ωp, ω, setup) -function interpolate_ω_p!(::Dimension{2}, setup, ωp, ω) +function interpolate_ω_p!(::Dimension{2}, ωp, ω, setup) (; boundary_conditions, grid, Re, bodyforce) = setup (; dimension, Np, Ip) = grid D = dimension() @@ -392,7 +392,7 @@ function interpolate_ω_p!(::Dimension{2}, setup, ωp, ω) ωp end -function interpolate_ω_p!(::Dimension{3}, setup, ωp, ω) +function interpolate_ω_p!(::Dimension{3}, ωp, ω, setup) (; boundary_conditions, grid, Re) = setup (; dimension, Np, Ip) = grid D = dimension() @@ -521,10 +521,10 @@ Qfield(u, setup) = Qfield!( Compute total kinetic energy. The velocity components are interpolated to the volume centers and squared. """ -function kinetic_energy(setup, u) +function kinetic_energy(u, setup) (; dimension, Ω, Ip) = setup.grid D = dimension() - up = interpolate_u_p(setup, u) + up = interpolate_u_p(u, setup) E = zero(eltype(up[1])) for α = 1:D # E += sum(I -> Ω[I] * up[α][I]^2, Ip) diff --git a/src/postprocess/plot_velocity.jl b/src/postprocess/plot_velocity.jl index b757e393c..260d72959 100644 --- a/src/postprocess/plot_velocity.jl +++ b/src/postprocess/plot_velocity.jl @@ -14,7 +14,7 @@ function plot_velocity(::Dimension{2}, setup, u; kwargs...) T = eltype(xp[1]) # Get velocity at pressure points - up = interpolate_u_p(setup, u) + up = interpolate_u_p(u, setup) # qp = map((u, v) -> √(u^2 + v^2), up, vp) qp = sqrt.(up[1] .^ 2 .+ up[2] .^ 2) @@ -43,7 +43,7 @@ function plot_velocity(::Dimension{3}, setup, u; kwargs...) (; xp) = setup.grid # Get velocity at pressure points - up = interpolate_u_p(setup, u) + up = interpolate_u_p(u, setup) qp = map((u, v, w) -> √sum(u^2 + v^2 + w^2), up...) # Levels diff --git a/src/postprocess/plot_vorticity.jl b/src/postprocess/plot_vorticity.jl index 2e8588bde..1f7cda9b8 100644 --- a/src/postprocess/plot_vorticity.jl +++ b/src/postprocess/plot_vorticity.jl @@ -18,7 +18,7 @@ function plot_vorticity(::Dimension{2}, setup, u; kwargs...) # Get fields ω = vorticity(u, setup) - ωp = interpolate_ω_p(setup, ω) + ωp = interpolate_ω_p(ω, setup) ωp = Array(ωp)[Ip] # Levels @@ -50,7 +50,7 @@ function plot_vorticity(::Dimension{3}, setup, u; kwargs...) (; grid) = setup (; xp) = grid - ωp = interpolate_ω_p(setup, vorticity(u, setup)) + ωp = interpolate_ω_p(vorticity(u, setup), setup) qp = map((u, v, w) -> √sum(u^2 + v^2 + w^2), ωp...) # Levels diff --git a/src/postprocess/save_vtk.jl b/src/postprocess/save_vtk.jl index 325f4d601..351153fdc 100644 --- a/src/postprocess/save_vtk.jl +++ b/src/postprocess/save_vtk.jl @@ -15,8 +15,8 @@ function save_vtk(setup, u, p, filename = "output/solution") D = dimension() xp = Array.(xp) vtk_grid(filename, xp...) do vtk - up = interpolate_u_p(setup, u) - ωp = interpolate_ω_p(setup, vorticity(u, setup)) + up = interpolate_u_p(u, setup) + ωp = interpolate_ω_p(vorticity(u, setup), setup) if D == 2 # ParaView prefers 3D vectors. Add zero z-component. up3 = zero(up[1]) diff --git a/src/processors/real_time_plot.jl b/src/processors/real_time_plot.jl index 14dc43ef9..1cded6595 100644 --- a/src/processors/real_time_plot.jl +++ b/src/processors/real_time_plot.jl @@ -58,10 +58,10 @@ function field_plot( (; u, p, t) = state[] _f = if fieldname == :velocity - up = interpolate_u_p(setup, u) + up = interpolate_u_p(u, setup) elseif fieldname == :vorticity ω = vorticity(u, setup) - ωp = interpolate_ω_p(setup, ω) + ωp = interpolate_ω_p(ω, setup) elseif fieldname == :streamfunction ψ = get_streamfunction(setup, u, t) elseif fieldname == :pressure @@ -72,12 +72,12 @@ function field_plot( isnothing(sleeptime) || sleep(sleeptime) (; u, p, t) = $state f = if fieldname == :velocity - interpolate_u_p!(setup, up, u) + interpolate_u_p!(up, u, setup) map((u, v) -> √sum(u^2 + v^2), up...) elseif fieldname == :vorticity apply_bc_u!(u, t, setup) vorticity!(ω, u, setup) - interpolate_ω_p!(setup, ωp, ω) + interpolate_ω_p!(ωp, ω, setup) elseif fieldname == :streamfunction get_streamfunction!(setup, ψ, u, t) elseif fieldname == :pressure @@ -181,10 +181,10 @@ function field_plot( isnothing(sleeptime) || sleep(sleeptime) (; u, p, t) = $state f = if fieldname == :velocity - up = interpolate_u_p(setup, u) + up = interpolate_u_p(u, setup) map((u, v, w) -> √sum(u^2 + v^2 + w^2), up...) elseif fieldname == :vorticity - ωp = interpolate_ω_p(setup, vorticity(u, setup)) + ωp = interpolate_ω_p(vorticity(u, setup), setup) map((u, v, w) -> √sum(u^2 + v^2 + w^2), ωp...) elseif fieldname == :streamfunction get_streamfunction(setup, u, t) @@ -275,7 +275,7 @@ function energy_spectrum_plot(setup, state; displayfig = true) k = Array(reshape(k, :)) ehat = @lift begin (; u, p, t) = $state - up = interpolate_u_p(setup, u) + up = interpolate_u_p(u, setup) e = sum(up -> up[Ip] .^ 2, up) Array(reshape(abs.(fft(e)[ntuple(α -> kx[α] .+ 1, D)...]), :)) end From 2a3e3f4c0c069f7e2797c796968b2bd47ee17569 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Sat, 4 Nov 2023 18:23:35 +0100 Subject: [PATCH 112/379] Update files --- docs/src/features/closure.md | 6 +++++ scratch/train_model.jl | 18 ++------------- src/IncompressibleNavierStokes.jl | 4 +++- src/closures/closure.jl | 13 +++++++++++ src/closures/create_les_data.jl | 38 +++++++++++++++++++++++++++++++ 5 files changed, 62 insertions(+), 17 deletions(-) create mode 100644 src/closures/closure.jl diff --git a/docs/src/features/closure.md b/docs/src/features/closure.md index 03ad41823..8103b1572 100644 --- a/docs/src/features/closure.md +++ b/docs/src/features/closure.md @@ -1,3 +1,7 @@ +```@meta +CurrentModule = IncompressibleNavierStokes +``` + # Neural closure models For [large eddy simulation (LES)](../features/les.md), a closure model is @@ -36,6 +40,8 @@ M \bar{v} & = 0, \\ ```@docs face_average! +create_les_data +create_io_arrays ``` ## Training diff --git a/scratch/train_model.jl b/scratch/train_model.jl index 84e87fd80..ed4f68a9e 100644 --- a/scratch/train_model.jl +++ b/scratch/train_model.jl @@ -119,22 +119,6 @@ closure, θ₀ = cnn( closure.NN # Create input/output arrays -function create_io_arrays(data, setup) - nsample = length(data.u) - nt = length(data.u[1]) - 1 - D = setup.grid.dimension() - T = eltype(data.u[1][1][1]) - (; N) = setup.grid - u = zeros(T, (N .- 2)..., D, nt + 1, nsample) - c = zeros(T, (N .- 2)..., D, nt + 1, nsample) - ifield = ntuple(Returns(:), D) - for i = 1:nsample, j = 1:nt+1, α = 1:D - copyto!(view(u, ifield..., α, j, i), view(data.u[i][j][α], setup.grid.Iu[α])) - copyto!(view(c, ifield..., α, j, i), view(data.cF[i][j][α], setup.grid.Iu[α])) - end - reshape(u, (N .- 2)..., D, :), reshape(c, (N .- 2)..., D, :) -end - io_train = create_io_arrays(data_train, setup) io_valid = create_io_arrays(data_valid, setup) io_test = create_io_arrays(data_test, setup) @@ -186,6 +170,8 @@ relative_error( ) relative_error(closure(u_test, θ), c_test) + + function energy_history(setup, state) (; Ωp) = setup.grid points = Point2f[] diff --git a/src/IncompressibleNavierStokes.jl b/src/IncompressibleNavierStokes.jl index 738a08a91..bb537b739 100644 --- a/src/IncompressibleNavierStokes.jl +++ b/src/IncompressibleNavierStokes.jl @@ -94,6 +94,7 @@ include("postprocess/plot_streamfunction.jl") include("postprocess/save_vtk.jl") # Closure models +include("closures/closure.md") include("closures/cnn.jl") include("closures/fno.jl") include("closures/training.jl") @@ -139,7 +140,8 @@ export plotmat export cnn, fno, FourierLayer export train export mean_squared_error, relative_error -export create_randloss, create_callback, create_les_data +export create_randloss, create_callback, create_les_data, create_io_arrays +export create_neural_closure # ODE methods diff --git a/src/closures/closure.jl b/src/closures/closure.jl new file mode 100644 index 000000000..b3a3c386e --- /dev/null +++ b/src/closures/closure.jl @@ -0,0 +1,13 @@ +function create_neural_closure(m, θ, setup) + (; dimension, Iu) = setup.grid + D = dimension() + function neural_closure(u) + u = stack(ntuple(α -> u[α][Iu[α]], D)) + u = reshape(u, size(u)..., 1) # One sample + mu = m(u, θ) + mu = pad_circular(u) + sz..., _ = size(mu) + i = ntuple(α -> Returns(:), D) + mu = ntuple(α -> mu[i..., α, 1], D) + end +end diff --git a/src/closures/create_les_data.jl b/src/closures/create_les_data.jl index 6e12f9261..95abe830d 100644 --- a/src/closures/create_les_data.jl +++ b/src/closures/create_les_data.jl @@ -84,6 +84,23 @@ _filter_saver(dns, les, comp; nupdate = 1) = processor(function (state) ) end; nupdate) +""" + create_les_data( + T; + D = 2, + Re = T(2_000), + lims = (T(0), T(1)), + nles = 64, + compression = 4, + nsim = 10, + tburn = T(0.1), + tsim = T(0.1), + Δt = T(1e-4), + ArrayType = Array, +) + +Create filtered DNS data. +""" function create_les_data( T; D = 2, @@ -188,3 +205,24 @@ function create_les_data( filtered end + +""" + create_io_arrays(data, setup) + +Create ``(\\bar{u}, c)`` pairs for training. +""" +function create_io_arrays(data, setup) + nsample = length(data.u) + nt = length(data.u[1]) - 1 + D = setup.grid.dimension() + T = eltype(data.u[1][1][1]) + (; N) = setup.grid + u = zeros(T, (N .- 2)..., D, nt + 1, nsample) + c = zeros(T, (N .- 2)..., D, nt + 1, nsample) + ifield = ntuple(Returns(:), D) + for i = 1:nsample, j = 1:nt+1, α = 1:D + copyto!(view(u, ifield..., α, j, i), view(data.u[i][j][α], setup.grid.Iu[α])) + copyto!(view(c, ifield..., α, j, i), view(data.cF[i][j][α], setup.grid.Iu[α])) + end + reshape(u, (N .- 2)..., D, :), reshape(c, (N .- 2)..., D, :) +end From 3ebcedfc45f6955e21c0b7867c66768764b54c30 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Sun, 5 Nov 2023 09:14:27 +0100 Subject: [PATCH 113/379] Minor fixes --- scratch/train_model.jl | 12 +++++------- src/IncompressibleNavierStokes.jl | 2 +- src/closures/cnn.jl | 13 +++++++++---- 3 files changed, 15 insertions(+), 12 deletions(-) diff --git a/scratch/train_model.jl b/scratch/train_model.jl index ed4f68a9e..2e7b72623 100644 --- a/scratch/train_model.jl +++ b/scratch/train_model.jl @@ -124,11 +124,11 @@ io_valid = create_io_arrays(data_valid, setup) io_test = create_io_arrays(data_test, setup) # Prepare training -θ = 1.0f-1 * cu(θ₀) +θ = T(1.0e-1) * device(θ₀) # θ = cu(θ₀) -opt = Optimisers.setup(Adam(1.0f-3), θ) -callbackstate = Point2f[] -randloss = create_randloss(mean_squared_error, closure, io_train...; nuse = 50, device = cu) +opt = Optimisers.setup(Adam(T(1.0e-3)), θ); +callbackstate = Point2f[]; +randloss = create_randloss(mean_squared_error, closure, io_train...; nuse = 50, device); # Warm-up randloss(θ) @@ -147,7 +147,7 @@ first(gradient(randloss, θ)); niter = 2000, ncallback = 10, callbackstate, - callback = create_callback(closure, cu(io_valid)...; state = callbackstate), + callback = create_callback(closure, device(io_valid)...; state = callbackstate), ); GC.gc() CUDA.reclaim() @@ -170,8 +170,6 @@ relative_error( ) relative_error(closure(u_test, θ), c_test) - - function energy_history(setup, state) (; Ωp) = setup.grid points = Point2f[] diff --git a/src/IncompressibleNavierStokes.jl b/src/IncompressibleNavierStokes.jl index bb537b739..a442b0069 100644 --- a/src/IncompressibleNavierStokes.jl +++ b/src/IncompressibleNavierStokes.jl @@ -94,7 +94,7 @@ include("postprocess/plot_streamfunction.jl") include("postprocess/save_vtk.jl") # Closure models -include("closures/closure.md") +include("closures/closure.jl") include("closures/cnn.jl") include("closures/fno.jl") include("closures/training.jl") diff --git a/src/closures/cnn.jl b/src/closures/cnn.jl index f03c78490..a14ec0419 100644 --- a/src/closures/cnn.jl +++ b/src/closures/cnn.jl @@ -9,9 +9,9 @@ function cnn(setup, r, c, σ, b; channel_augmenter = identity, rng = Random.defa (; dimension, x) = grid D = dimension() - # For now + # Weight initializer T = eltype(x[1]) - @assert T == Float32 + glorot_uniform_T(rng::AbstractRNG, dims...) = glorot_uniform(rng, T, dims...) # Make sure there are two force fields in output @assert c[end] == D @@ -26,8 +26,13 @@ function cnn(setup, r, c, σ, b; channel_augmenter = identity, rng = Random.defa # Some convolutional layers ( - Conv(ntuple(α -> 2r[i] + 1, D), c[i] => c[i+1], σ[i]; use_bias = b[i]) for - i ∈ eachindex(r) + Conv( + ntuple(α -> 2r[i] + 1, D), + c[i] => c[i+1], + σ[i]; + use_bias = b[i], + init_weight = glorot_uniform_T, + ) for i ∈ eachindex(r) )..., ) From 28617c1deeadee7336d81dfdeafe4632be4efea0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Sun, 5 Nov 2023 20:27:28 +0100 Subject: [PATCH 114/379] Minor changes --- scratch/train_model.jl | 33 +++++++++++++++++++++++++++++---- src/closures/closure.jl | 4 ++-- src/operators.jl | 7 ++++++- 3 files changed, 37 insertions(+), 7 deletions(-) diff --git a/scratch/train_model.jl b/scratch/train_model.jl index 2e7b72623..204de9114 100644 --- a/scratch/train_model.jl +++ b/scratch/train_model.jl @@ -25,6 +25,7 @@ T = Float64 # Array type ArrayType = Array +device = identity ## using CUDA; ArrayType = CuArray ## using AMDGPU; ArrayType = ROCArray ## using oneAPI; ArrayType = oneArray @@ -35,6 +36,7 @@ using CUDA; T = Float32; ArrayType = CuArray; CUDA.allowscalar(false); +device = cu # Setup n = 128 @@ -46,7 +48,7 @@ tsim = T(0.05) # Build LES setup and assemble operators x = LinRange(lims..., n + 1) y = LinRange(lims..., n + 1) -setup = Setup(x, y; Re); +setup = Setup(x, y; Re, ArrayType); # Number of simulations ntrain = 10 @@ -119,9 +121,9 @@ closure, θ₀ = cnn( closure.NN # Create input/output arrays -io_train = create_io_arrays(data_train, setup) -io_valid = create_io_arrays(data_valid, setup) -io_test = create_io_arrays(data_test, setup) +io_train = create_io_arrays(data_train, setup); +io_valid = create_io_arrays(data_valid, setup); +io_test = create_io_arrays(data_test, setup); # Prepare training θ = T(1.0e-1) * device(θ₀) @@ -163,6 +165,29 @@ Array(θ) # θθ = load("output/theta_fno.jld2") # copyto!(θ, θθ["theta"]) +u₀ = device(data_test.u[1][1]) +p₀ = pressure_additional_solve(pressure_solver, u₀, T(0), setup) + +u, p, outputs = solve_unsteady( + setup, + copy.(u₀), + copy(p₀), + (T(0), T(0.1)); + Δt = T(1e-4), + pressure_solver, + processors = (field_plotter(setup; nupdate = 10), step_logger(; nupdate = 1)), +) + +u, p, outputs = solve_unsteady( + (; setup..., closure_model = create_neural_closure(closure, θ, setup)), + copy.(u₀), + copy(p₀), + (T(0), T(0.1)); + Δt = T(1e-4), + pressure_solver, + processors = (field_plotter(setup; nupdate = 10), step_logger(; nupdate = 1)), +) + relative_error(closure(device(data_train.V[:, 1, :]), θ), device(data_train.cF[:, 1, :])) relative_error( closure(device(data_train.V[:, end, :]), θ), diff --git a/src/closures/closure.jl b/src/closures/closure.jl index b3a3c386e..ed50b0cd4 100644 --- a/src/closures/closure.jl +++ b/src/closures/closure.jl @@ -5,9 +5,9 @@ function create_neural_closure(m, θ, setup) u = stack(ntuple(α -> u[α][Iu[α]], D)) u = reshape(u, size(u)..., 1) # One sample mu = m(u, θ) - mu = pad_circular(u) + mu = pad_circular(u, 1) sz..., _ = size(mu) - i = ntuple(α -> Returns(:), D) + i = ntuple(Returns(:), D) mu = ntuple(α -> mu[i..., α, 1], D) end end diff --git a/src/operators.jl b/src/operators.jl index edc781225..38e8f320e 100644 --- a/src/operators.jl +++ b/src/operators.jl @@ -237,7 +237,12 @@ function momentum!(F, u, t, setup) diffusion!(F, u, setup) convection!(F, u, setup) bodyforce!(F, u, t, setup) - isnothing(closure_model) || (F .+= closure_model(u)) + if !isnothing(closure_model) + m = closure_model(u) + for α = 1:D + F[α] .+= m[α] + end + end F end From 6461f51bf0d1f7c7a1672722cdc0344be4cb07c8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Wed, 8 Nov 2023 13:07:35 +0100 Subject: [PATCH 115/379] Add projection --- scratch/train_model.jl | 52 +++++++++----------- src/closures/create_les_data.jl | 86 ++++++++++++++------------------- 2 files changed, 59 insertions(+), 79 deletions(-) diff --git a/scratch/train_model.jl b/scratch/train_model.jl index 204de9114..3bf50c30c 100644 --- a/scratch/train_model.jl +++ b/scratch/train_model.jl @@ -38,40 +38,30 @@ ArrayType = CuArray; CUDA.allowscalar(false); device = cu -# Setup -n = 128 -lims = T(0), T(1) -Re = T(6_000) -tburn = T(0.05) -tsim = T(0.05) - -# Build LES setup and assemble operators -x = LinRange(lims..., n + 1) -y = LinRange(lims..., n + 1) -setup = Setup(x, y; Re, ArrayType); - -# Number of simulations -ntrain = 10 -nvalid = 2 -ntest = 5 +# Parameters +params = (; + D = 2, + Re = T(6_000), + lims = (T(0), T(1)), + nles = 128, + compression = 4, + tburn = T(0.05), + tsim = T(0.05), + Δt = T(1e-4), + ArrayType, +) # Create LES data from DNS -params = - (; D = 2, Re, lims, nles = n, compression = 4, tburn, tsim, Δt = T(1e-4), ArrayType) -data_train = create_les_data(T; params..., nsim = ntrain); -data_valid = create_les_data(T; params..., nsim = nvalid); -data_test = create_les_data(T; params..., nsim = ntest); - -length(data_train.u) -length(data_train.u[1]) -length(data_train.u[1][1]) -size(data_train.u[1][1][1]) +data_train = create_les_data(T; params..., nsim = 10); +data_valid = create_les_data(T; params..., nsim = 2); +data_test = create_les_data(T; params..., nsim = 5); +# Inspect data o = Observable(data_train.u[1][end][1]) heatmap(o) for i = 1:501 o[] = data_train.u[1][i][1] - # o[] = data_train.cF[1][i][1] + # o[] = data_train.c[1][i][1] sleep(0.001) end @@ -81,7 +71,10 @@ end # # Load previous LES data # data_train, data_valid, data_test = load("output/forced/data.jld2", "data_train", "data_valid", "data_test") -nt = length(data_train.u[1]) - 1 +# Build LES setup and assemble operators +x = LinRange(params.lims..., params.nles + 1) +y = LinRange(params.lims..., params.nles + 1) +setup = Setup(x, y; params.Re, ArrayType); # Uniform periodic grid pressure_solver = SpectralPressureSolver(setup); @@ -126,8 +119,7 @@ io_valid = create_io_arrays(data_valid, setup); io_test = create_io_arrays(data_test, setup); # Prepare training -θ = T(1.0e-1) * device(θ₀) -# θ = cu(θ₀) +θ = T(1.0e-1) * device(θ₀); opt = Optimisers.setup(Adam(T(1.0e-3)), θ); callbackstate = Point2f[]; randloss = create_randloss(mean_squared_error, closure, io_train...; nuse = 50, device); diff --git a/src/closures/create_les_data.jl b/src/closures/create_les_data.jl index 95abe830d..f20b3013d 100644 --- a/src/closures/create_les_data.jl +++ b/src/closures/create_les_data.jl @@ -32,55 +32,52 @@ function gaussian_force( force end -_filter_saver(dns, les, comp; nupdate = 1) = processor(function (state) +_filter_saver(dns, les, comp, pressure_solver; nupdate = 1) = processor(function (state) (; dimension, x) = dns.grid + T = eltype(x[1]) D = dimension() F = zero.(state[].u) - # pbar = zero(volume_average(state[].p, les, comp)) - ubar = zero.(face_average(state[].u, les, comp)) - Fbar = zero.(ubar) - Fubar = zero.(ubar) - cF = zero.(ubar) + G = zero.(state[].u) + Φu = zero.(face_average(state[].u, les, comp)) + q = zero(pressure_additional_solve(pressure_solver, Φu, state[].t, les)) + M = zero(q) + ΦF = zero.(Φu) + FΦ = zero.(Φu) + GΦ = zero.(Φu) + c = zero.(Φu) _t = fill(zero(eltype(x[1])), 0) - _u = fill(Array.(ubar), 0) - # _p = fill(Array.(ubar), 0) - _F = fill(Array.(ubar), 0) - # _FG = fill(Array.(ubar), 0) - _cF = fill(Array.(ubar), 0) - # _cFG = fill(Array.(ubar), 0) + _u = fill(Array.(Φu), 0) + _c = fill(Array.(Φu), 0) on(state) do (; u, p, t) - face_average!(ubar, u, les, comp) - apply_bc_u!(ubar, t, les) - # pbar = Kp * p momentum!(F, u, t, dns) - # G = pressuregradient(p, dns) - # FG = F .+ G - face_average!(Fbar, F, les, comp) - # FGbar = Ku .* FG - momentum!(Fubar, ubar, t, les) - # Gubar = pressuregradient(pbar, les) - # FGubar = Fubar + Gubar + pressuregradient!(G, p, dns) for α = 1:D - cF[α] .= Fbar[α] .- Fubar[α] + F[α] .-= G[α] + end + face_average!(Φu, u, les, comp) + apply_bc_u!(Φu, t, les) + face_average!(ΦF, F, les, comp) + momentum!(FΦ, Φu, t, les) + apply_bc_u!(FΦ, t, les; dudt = true) + divergence!(M, FΦ, les) + @. M *= les.grid.Ω + + pressure_poisson!(pressure_solver, q, M) + apply_bc_p!(q, t, les) + pressuregradient!(GΦ, q, les) + for α = 1:D + FΦ[α] .-= GΦ[α] + c[α] .= ΦF[α] .- FΦ[α] end - # cFG = FGbar .- FGVbar push!(_t, t) - push!(_u, Array.(ubar)) - # push!(_p, Array(pbar)) - # push!(_F, Array.(Fubar)) - # push!(_FG, Array.(FGbar)) - push!(_cF, Array.(cF)) - # push!(_cFG, Array.(cFG)) + push!(_u, Array.(Φu)) + push!(_c, Array.(c)) end state[] = state[] # Save initial conditions (; t = _t, u = _u, - # p = _p, - # F = _F, - # FG = _FG, - cF = _cF, - # cFG = _cFG, + c = _c, ) end; nupdate) @@ -125,6 +122,7 @@ function create_les_data( # Since the grid is uniform and identical for x and y, we may use a specialized # spectral pressure solver pressure_solver = SpectralPressureSolver(dns) + pressure_solver_les = SpectralPressureSolver(les) # Number of time steps to save nt = round(Int, tsim / Δt) @@ -135,12 +133,7 @@ function create_les_data( filtered = (; Δt, u = fill(fill(ntuple(α -> zeros(T, N...), D), 0), 0), - # p = fill(fill(zeros(T, N...), 0), 0), - # F = fill(fill(ntuple(α -> zeros(T, N...), D), 0), 0), - # FG = fill(fill(ntuple(α -> zeros(T, N...), D), 0), 0), - cF = fill(fill(ntuple(α -> zeros(T, N...), D), 0), 0), - # cFG = fill(fill(ntuple(α -> zeros(T, N...), D), 0), 0), - # force = fill(fill(ntuple(α -> zeros(T, N...), D), 0), 0), + c = fill(fill(ntuple(α -> zeros(T, N...), D), 0), 0), ) # @info "Generating $(Base.summarysize(filtered) / 1e6) Mb of LES data" @@ -186,7 +179,7 @@ function create_les_data( (T(0), tsim); Δt, processors = ( - _filter_saver(_dns, _les, compression), + _filter_saver(_dns, _les, compression, pressure_solver_les), # step_logger(; nupdate = 10), ), pressure_solver, @@ -195,12 +188,7 @@ function create_les_data( # Store result for current IC push!(filtered.u, f.u) - # push!(filtered.p, f.p) - # push!(filtered.F, f.F) - # push!(filtered.FG, f.FG) - push!(filtered.cF, f.cF) - # push!(filtered.cFG, f.cFG) - # push!(filtered.force, f.force) + push!(filtered.c, f.c) end filtered @@ -222,7 +210,7 @@ function create_io_arrays(data, setup) ifield = ntuple(Returns(:), D) for i = 1:nsample, j = 1:nt+1, α = 1:D copyto!(view(u, ifield..., α, j, i), view(data.u[i][j][α], setup.grid.Iu[α])) - copyto!(view(c, ifield..., α, j, i), view(data.cF[i][j][α], setup.grid.Iu[α])) + copyto!(view(c, ifield..., α, j, i), view(data.c[i][j][α], setup.grid.Iu[α])) end reshape(u, (N .- 2)..., D, :), reshape(c, (N .- 2)..., D, :) end From ba86d1adff7218a2c784a8bed2ad39b048826988 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Wed, 8 Nov 2023 17:33:27 +0100 Subject: [PATCH 116/379] Add type --- src/closures/create_les_data.jl | 25 +++++++++++++------------ src/closures/fno.jl | 8 ++++++-- 2 files changed, 19 insertions(+), 14 deletions(-) diff --git a/src/closures/create_les_data.jl b/src/closures/create_les_data.jl index f20b3013d..4f9be6482 100644 --- a/src/closures/create_les_data.jl +++ b/src/closures/create_les_data.jl @@ -83,18 +83,19 @@ end; nupdate) """ create_les_data( - T; - D = 2, - Re = T(2_000), - lims = (T(0), T(1)), - nles = 64, - compression = 4, - nsim = 10, - tburn = T(0.1), - tsim = T(0.1), - Δt = T(1e-4), - ArrayType = Array, -) + T; + D = 2, + Re = T(2_000), + lims = (T(0), T(1)), + nles = 64, + compression = 4, + nsim = 10, + tburn = T(0.1), + tsim = T(0.1), + Δt = T(1e-4), + ArrayType = Array, + ) + Create filtered DNS data. """ diff --git a/src/closures/fno.jl b/src/closures/fno.jl index c9472cde6..e040f4cda 100644 --- a/src/closures/fno.jl +++ b/src/closures/fno.jl @@ -6,7 +6,7 @@ initial parameters and `closure(V, θ)` predicts the commutator error. """ function fno(setup, kmax, c, σ, ψ; rng = Random.default_rng(), kwargs...) (; grid) = setup - (; dimension, N) = grid + (; dimension, x, N) = grid D = dimension() @@ -18,11 +18,15 @@ function fno(setup, kmax, c, σ, ψ; rng = Random.default_rng(), kwargs...) # Make sure there are two velocity fields in input and output c = [2; c] + # Weight initializer + T = eltype(x[1]) + init_weight(rng::AbstractRNG, dims...) = glorot_uniform(rng, T, dims...) + # Create FNO closure model NN = Chain( # Some Fourier layers ( - FourierLayer(dimension, kmax[i], c[i] => c[i+1]; σ = σ[i]) for i ∈ eachindex(σ) + FourierLayer(dimension, kmax[i], c[i] => c[i+1]; σ = σ[i], init_weight) for i ∈ eachindex(σ) )..., # Put channels in first dimension From 3a04ce1b965d6616fa99b793ab88cd9fc1e81d30 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Thu, 9 Nov 2023 09:37:20 +0100 Subject: [PATCH 117/379] Add plot --- scratch/filter_plot.jl | 129 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 129 insertions(+) create mode 100644 scratch/filter_plot.jl diff --git a/scratch/filter_plot.jl b/scratch/filter_plot.jl new file mode 100644 index 000000000..102bc3bc0 --- /dev/null +++ b/scratch/filter_plot.jl @@ -0,0 +1,129 @@ +# Little LSP hack to get function signatures, go #src +# to definition etc. #src +if isdefined(@__MODULE__, :LanguageServer) #src + include("../src/IncompressibleNavierStokes.jl") #src + using .IncompressibleNavierStokes #src +end #src + +using GLMakie +using IncompressibleNavierStokes + +# Floating point precision +T = Float64 + +# Array type +ArrayType = Array +## using CUDA; ArrayType = CuArray +## using AMDGPU; ArrayType = ROCArray +## using oneAPI; ArrayType = oneArray +## using Metal; ArrayType = MtlArray + +using CUDA; T = Float32; ArrayType = CuArray; CUDA.allowscalar(false) + +# Viscosity model +Re = T(10_000) + +# A 2D grid is a Cartesian product of two vectors +n = 1024 +lims = T(0), T(1) +x = LinRange(lims..., n + 1), LinRange(lims..., n + 1) + +# Build setup and assemble operators +setup = Setup(x...; Re, ArrayType); + +# Since the grid is uniform and identical for x and y, we may use a specialized +# spectral pressure solver +pressure_solver = SpectralPressureSolver(setup); + +u₀, p₀ = random_field(setup, T(0); pressure_solver); +u, p = u₀, p₀ + +set_theme!(; + # theme_black(); + # colormap = :plasma, + # colormap = :lajolla, + # colormap = :inferno, + colormap = :hot, + # colormap = :magma, + # colormap = :Oranges, + # colormap = :Reds, + # colormap = :YlOrRd_9, + # colormap = :seaborn_icefire_gradient, +) + +function filter_plot(state, setup, les, comp; resolution = (1200, 600)) + (; boundary_conditions, grid) = setup + (; dimension, xlims, x, xp, Ip) = grid + D = dimension() + xf = Array.(getindex.(setup.grid.xp, Ip.indices)) + xfbar = Array.(getindex.(les.grid.xp, les.grid.Ip.indices)) + (; u, p, t) = state[] + ω = IncompressibleNavierStokes.vorticity(u, setup) + ωp = IncompressibleNavierStokes.interpolate_ω_p(ω, setup) + _f = Array(ωp)[Ip] + f = @lift begin + sleep(0.001) + (; u, p, t) = $state + IncompressibleNavierStokes.apply_bc_u!(u, t, setup) + IncompressibleNavierStokes.vorticity!(ω, u, setup) + IncompressibleNavierStokes.interpolate_ω_p!(ωp, ω, setup) + copyto!(_f, view(ωp, Ip)) + end + ubar = zero.(IncompressibleNavierStokes.face_average(u, les, comp)) + ωbar = IncompressibleNavierStokes.vorticity(ubar, les) + ωpbar = IncompressibleNavierStokes.interpolate_ω_p(ωbar, les) + _g = Array(ωpbar)[les.grid.Ip] + g = @lift begin + (; u, p, t) = $state + IncompressibleNavierStokes.face_average!(ubar, u, les, comp) + IncompressibleNavierStokes.apply_bc_u!(ubar, t, les) + IncompressibleNavierStokes.vorticity!(ωbar, ubar, les) + IncompressibleNavierStokes.interpolate_ω_p!(ωpbar, ωbar, les) + copyto!(_g, view(ωpbar, les.grid.Ip)) + end + lims = @lift IncompressibleNavierStokes.get_lims($f) + fig = Figure(; resolution) + ax, hm = heatmap(fig[1, 1], xf..., f; colorrange = lims) + ax.title = "$(setup.grid.N)" + ax.aspect = DataAspect() + ax.xlabel = "x" + ax.ylabel = "y" + limits!(ax, xlims[1]..., xlims[2]...) + ax, hm = heatmap(fig[1, 2], xfbar..., g; colorrange = lims) + ax.title = "$(les.grid.N)" + ax.aspect = DataAspect() + ax.xlabel = "x" + ax.ylabel = "y" + limits!(ax, xlims[1]..., xlims[2]...) + Colorbar(fig[1, 3], hm) + display(fig) + fig +end + +comp = 8 +les = Setup(x[1][1:comp:end], x[2][1:comp:end]; Re, ArrayType) + +# Solve unsteady problem +u, p, outputs = solve_unsteady( + setup, + copy.(u₀), copy(p₀), + # u, p, + (T(0), T(1e0)); + Δt = T(1e-4), + pressure_solver, + inplace = true, + processors = ( + # field_plotter(setup; nupdate = 10, docolorbar = false, displayupdates = false), + animator( + setup, + "vorticity.mp4"; + plotter = processor(state -> filter_plot(state, setup, les, comp; resolution = (1200, 600)); nupdate = 10), + nupdate = 50, + ), + # energy_history_plotter(setup; nupdate = 20, displayfig = false), + # energy_spectrum_plotter(setup; nupdate = 10, displayfig = false), + ## vtk_writer(setup; nupdate = 10, dir = "output/$name", filename = "solution"), + ## field_saver(setup; nupdate = 10), + step_logger(; nupdate = 10), + ), +); From c3f2fd9c1de1cd0d5835305f8cb8468f104ab617 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Sun, 12 Nov 2023 13:14:17 +0100 Subject: [PATCH 118/379] Minor changes --- scratch/filter_plot.jl | 92 ++++++++++++++++++++++++++---------------- 1 file changed, 58 insertions(+), 34 deletions(-) diff --git a/scratch/filter_plot.jl b/scratch/filter_plot.jl index 102bc3bc0..1107b1b20 100644 --- a/scratch/filter_plot.jl +++ b/scratch/filter_plot.jl @@ -18,7 +18,10 @@ ArrayType = Array ## using oneAPI; ArrayType = oneArray ## using Metal; ArrayType = MtlArray -using CUDA; T = Float32; ArrayType = CuArray; CUDA.allowscalar(false) +using CUDA; +T = Float32; +ArrayType = CuArray; +CUDA.allowscalar(false); # Viscosity model Re = T(10_000) @@ -38,19 +41,6 @@ pressure_solver = SpectralPressureSolver(setup); u₀, p₀ = random_field(setup, T(0); pressure_solver); u, p = u₀, p₀ -set_theme!(; - # theme_black(); - # colormap = :plasma, - # colormap = :lajolla, - # colormap = :inferno, - colormap = :hot, - # colormap = :magma, - # colormap = :Oranges, - # colormap = :Reds, - # colormap = :YlOrRd_9, - # colormap = :seaborn_icefire_gradient, -) - function filter_plot(state, setup, les, comp; resolution = (1200, 600)) (; boundary_conditions, grid) = setup (; dimension, xlims, x, xp, Ip) = grid @@ -83,42 +73,76 @@ function filter_plot(state, setup, les, comp; resolution = (1200, 600)) end lims = @lift IncompressibleNavierStokes.get_lims($f) fig = Figure(; resolution) - ax, hm = heatmap(fig[1, 1], xf..., f; colorrange = lims) - ax.title = "$(setup.grid.N)" - ax.aspect = DataAspect() - ax.xlabel = "x" - ax.ylabel = "y" - limits!(ax, xlims[1]..., xlims[2]...) - ax, hm = heatmap(fig[1, 2], xfbar..., g; colorrange = lims) - ax.title = "$(les.grid.N)" - ax.aspect = DataAspect() - ax.xlabel = "x" - ax.ylabel = "y" - limits!(ax, xlims[1]..., xlims[2]...) - Colorbar(fig[1, 3], hm) + ax, hm = heatmap( + fig[1, 1], + xf..., + f; + colorrange = lims, + axis = (; + title = "$(setup.grid.N .- 2)", + aspect = DataAspect(), + xlabel = "x", + ylabel = "y", + limits = (xlims[1]..., xlims[2]...), + ), + ) + ax, hm = heatmap( + fig[1, 2], + xfbar..., + g; + colorrange = lims, + axis = (; + title = "$(les.grid.N .- 2)", + aspect = DataAspect(), + xlabel = "x", + # ylabel = "y", + # yticksvisible = false, + # yticklabelsvisible = false, + limits = (xlims[1]..., xlims[2]...), + ), + ) + # Colorbar(fig[1, 3], hm) display(fig) fig end +set_theme!( + theme_black(); + # colormap = :plasma, + # colormap = :lajolla, + # colormap = :inferno, + # colormap = :hot, + # colormap = :magma, + # colormap = :Oranges, + # colormap = :Reds, + # colormap = :YlOrRd_9, + colormap = :seaborn_icefire_gradient, +) + comp = 8 les = Setup(x[1][1:comp:end], x[2][1:comp:end]; Re, ArrayType) # Solve unsteady problem u, p, outputs = solve_unsteady( setup, - copy.(u₀), copy(p₀), + copy.(u₀), + copy(p₀), # u, p, - (T(0), T(1e0)); + (T(0), T(2e0)); Δt = T(1e-4), pressure_solver, inplace = true, processors = ( # field_plotter(setup; nupdate = 10, docolorbar = false, displayupdates = false), - animator( - setup, - "vorticity.mp4"; - plotter = processor(state -> filter_plot(state, setup, les, comp; resolution = (1200, 600)); nupdate = 10), - nupdate = 50, + # animator( + # setup, + # "filtered.mp4"; + # plotter = processor(state -> filter_plot(state, setup, les, comp; resolution = (1200, 600))), + # nupdate = 100, + # ), + processor( + state -> filter_plot(state, setup, les, comp; resolution = (1200, 600)); + nupdate = 10, ), # energy_history_plotter(setup; nupdate = 20, displayfig = false), # energy_spectrum_plotter(setup; nupdate = 10, displayfig = false), From 4ad1a97098a00084d0d2881a5f816bf2daf65c32 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Sun, 12 Nov 2023 13:14:38 +0100 Subject: [PATCH 119/379] Add convergence plot --- examples/TaylorGreenVortex2D.jl | 161 ++++++++++++++------------------ 1 file changed, 70 insertions(+), 91 deletions(-) diff --git a/examples/TaylorGreenVortex2D.jl b/examples/TaylorGreenVortex2D.jl index 362e9c94c..f7e7196a9 100644 --- a/examples/TaylorGreenVortex2D.jl +++ b/examples/TaylorGreenVortex2D.jl @@ -8,98 +8,77 @@ end #src # # Taylor-Green vortex - 2D # # In this example we consider the Taylor-Green vortex. - -# We start by loading packages. -# A [Makie](https://github.com/JuliaPlots/Makie.jl) plotting backend is needed -# for plotting. `GLMakie` creates an interactive window (useful for real-time -# plotting), but does not work when building this example on GitHub. -# `CairoMakie` makes high-quality static vector-graphics plots. +# In 2D, it has an analytical solution, given by +# +# ```math +# \begin{split} +# u^1(x, y, t) & = - \sin(x) \cos(y) \mathrm{e}^{-2 t / Re} \\ +# u^2(x, y, t) & = + \cos(x) \sin(y) \mathrm{e}^{-2 t / Re} +# \end{split} +# ``` +# +# This allows us to test the convergence of our solver. #md using CairoMakie using GLMakie #!md using IncompressibleNavierStokes - -# Case name for saving results -name = "TaylorGreenVortex2D" - -# Floating point type -T = Float64 - -# Array type -ArrayType = Array -## using CUDA; ArrayType = CuArray -## using AMDGPU; ArrayType = ROCArray -## using oneAPI; ArrayType = oneArray -## using Metal; ArrayType = MtlArray - -# Reynolds number -Re = T(2_000) - -# A 2D grid is a Cartesian product of two vectors -n = 128 -lims = T(0), T(2π) -x = LinRange(lims..., n + 1), LinRange(lims..., n + 1) -plot_grid(x...) - -# Build setup and assemble operators -setup = Setup(x...; Re, ArrayType); - -# Since the grid is uniform and identical for x and y, we may use a specialized -# spectral pressure solver -pressure_solver = SpectralPressureSolver(setup) - -# Initial conditions -u₀, p₀ = create_initial_conditions( - setup, - (dim, x, y) -> dim() == 1 ? -sin(x) * cos(y) : cos(x) * sin(y); - pressure_solver, -); - -# Solve steady state problem -## u, p = solve_steady_state(setup, u₀, p₀; npicard = 2); - -# Iteration processors -processors = ( - ## field_plotter(setup; nupdate = 1), - energy_history_plotter(setup; nupdate = 1), - ## energy_spectrum_plotter(setup; nupdate = 1), - ## animator(setup, "vorticity.mkv"; nupdate = 4), - ## vtk_writer(setup; nupdate = 10, dir = "output/$name", filename = "solution"), - ## field_saver(setup; nupdate = 10), - step_logger(; nupdate = 1), -); - -# Solve unsteady problem -u, p, outputs = solve_unsteady( - setup, - u₀, - p₀, - (T(0), T(5)); - Δt = T(0.01), - processors, - pressure_solver, - inplace = true, -); - -# ## Post-process -# -# We may visualize or export the computed fields `(u, p)` - -# Export to VTK -save_vtk(setup, u, p, "output/solution") - -# Plot pressure -plot_pressure(setup, p) - -# Plot velocity -plot_velocity(setup, u) - -# Plot vorticity -plot_vorticity(setup, u) - -# Plot streamfunction -## plot_streamfunction(setup, u) -nothing - -# Energy history -outputs[1] +using LinearAlgebra + +function compute_convergence(; D, nlist, lims, Re, tlims, Δt, uref, ArrayType = Array) + T = typeof(lims[1]) + e = zeros(T, length(nlist)) + for (i, n) in enumerate(nlist) + @info "Computing error for n = $n" + x = ntuple(α -> LinRange(lims..., n + 1), D) + setup = Setup(x...; Re, ArrayType) + (; Ip) = setup.grid + pressure_solver = SpectralPressureSolver(setup) + u₀, p₀ = create_initial_conditions( + setup, + (dim, x...) -> uref(dim, x..., tlims[1]), + tlims[1]; + pressure_solver, + ) + ut, pt = create_initial_conditions( + setup, + (dim, x...) -> uref(dim, x..., tlims[2]), + tlims[2]; + pressure_solver, + ) + u, p, outputs = solve_unsteady(setup, u₀, p₀, tlims; Δt, pressure_solver) + for α = 1:D + e[i] += norm(u[α][Ip] - ut[α][Ip]) / norm(ut[α][Ip]) + end + e[i] /= D + end + e +end + +solution(Re) = + (dim, x, y, t) -> (dim() == 1 ? -sin(x) * cos(y) : cos(x) * sin(y)) * exp(-2t / Re) + +Re = 2.0e3 +nlist = [2, 4, 8, 16, 32, 64, 128, 256] +e = compute_convergence(; + D = 2, + nlist, + lims = (0.0, 2π), + Re, + tlims = (0.0, 2.0), + Δt = 0.01, + uref = solution(Re), +) + +# Plot convergence +fig = Figure() +ax = Axis( + fig[1, 1]; + xscale = log10, + yscale = log10, + xticks = nlist, + xlabel = "n", + title = "Relative error", +) +scatterlines!(nlist, e; label = "Data") +lines!(collect(extrema(nlist)), n -> n^-2.0; linestyle = :dash, label = "n^-2") +axislegend() From 778bac491caa7514967f43a220f5bfb2e8a235ff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Sun, 12 Nov 2023 13:16:48 +0100 Subject: [PATCH 120/379] Format files --- src/closures/create_les_data.jl | 93 ++++++++++++++++----------------- src/closures/fno.jl | 3 +- 2 files changed, 48 insertions(+), 48 deletions(-) diff --git a/src/closures/create_les_data.jl b/src/closures/create_les_data.jl index 4f9be6482..de83891f6 100644 --- a/src/closures/create_les_data.jl +++ b/src/closures/create_les_data.jl @@ -32,54 +32,53 @@ function gaussian_force( force end -_filter_saver(dns, les, comp, pressure_solver; nupdate = 1) = processor(function (state) - (; dimension, x) = dns.grid - T = eltype(x[1]) - D = dimension() - F = zero.(state[].u) - G = zero.(state[].u) - Φu = zero.(face_average(state[].u, les, comp)) - q = zero(pressure_additional_solve(pressure_solver, Φu, state[].t, les)) - M = zero(q) - ΦF = zero.(Φu) - FΦ = zero.(Φu) - GΦ = zero.(Φu) - c = zero.(Φu) - _t = fill(zero(eltype(x[1])), 0) - _u = fill(Array.(Φu), 0) - _c = fill(Array.(Φu), 0) - on(state) do (; u, p, t) - momentum!(F, u, t, dns) - pressuregradient!(G, p, dns) - for α = 1:D - F[α] .-= G[α] +_filter_saver(dns, les, comp, pressure_solver; nupdate = 1) = processor( + function (state) + (; dimension, x) = dns.grid + T = eltype(x[1]) + D = dimension() + F = zero.(state[].u) + G = zero.(state[].u) + Φu = zero.(face_average(state[].u, les, comp)) + q = zero(pressure_additional_solve(pressure_solver, Φu, state[].t, les)) + M = zero(q) + ΦF = zero.(Φu) + FΦ = zero.(Φu) + GΦ = zero.(Φu) + c = zero.(Φu) + _t = fill(zero(eltype(x[1])), 0) + _u = fill(Array.(Φu), 0) + _c = fill(Array.(Φu), 0) + on(state) do (; u, p, t) + momentum!(F, u, t, dns) + pressuregradient!(G, p, dns) + for α = 1:D + F[α] .-= G[α] + end + face_average!(Φu, u, les, comp) + apply_bc_u!(Φu, t, les) + face_average!(ΦF, F, les, comp) + momentum!(FΦ, Φu, t, les) + apply_bc_u!(FΦ, t, les; dudt = true) + divergence!(M, FΦ, les) + @. M *= les.grid.Ω + + pressure_poisson!(pressure_solver, q, M) + apply_bc_p!(q, t, les) + pressuregradient!(GΦ, q, les) + for α = 1:D + FΦ[α] .-= GΦ[α] + c[α] .= ΦF[α] .- FΦ[α] + end + push!(_t, t) + push!(_u, Array.(Φu)) + push!(_c, Array.(c)) end - face_average!(Φu, u, les, comp) - apply_bc_u!(Φu, t, les) - face_average!(ΦF, F, les, comp) - momentum!(FΦ, Φu, t, les) - apply_bc_u!(FΦ, t, les; dudt = true) - divergence!(M, FΦ, les) - @. M *= les.grid.Ω - - pressure_poisson!(pressure_solver, q, M) - apply_bc_p!(q, t, les) - pressuregradient!(GΦ, q, les) - for α = 1:D - FΦ[α] .-= GΦ[α] - c[α] .= ΦF[α] .- FΦ[α] - end - push!(_t, t) - push!(_u, Array.(Φu)) - push!(_c, Array.(c)) - end - state[] = state[] # Save initial conditions - (; - t = _t, - u = _u, - c = _c, - ) -end; nupdate) + state[] = state[] # Save initial conditions + (; t = _t, u = _u, c = _c) + end; + nupdate, +) """ create_les_data( diff --git a/src/closures/fno.jl b/src/closures/fno.jl index e040f4cda..0f33effef 100644 --- a/src/closures/fno.jl +++ b/src/closures/fno.jl @@ -26,7 +26,8 @@ function fno(setup, kmax, c, σ, ψ; rng = Random.default_rng(), kwargs...) NN = Chain( # Some Fourier layers ( - FourierLayer(dimension, kmax[i], c[i] => c[i+1]; σ = σ[i], init_weight) for i ∈ eachindex(σ) + FourierLayer(dimension, kmax[i], c[i] => c[i+1]; σ = σ[i], init_weight) for + i ∈ eachindex(σ) )..., # Put channels in first dimension From d399d3de80af3657d367a84eed3604c2088c842d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Sun, 12 Nov 2023 18:49:04 +0100 Subject: [PATCH 121/379] Create path before animating --- src/processors/animator.jl | 1 + 1 file changed, 1 insertion(+) diff --git a/src/processors/animator.jl b/src/processors/animator.jl index 6bd41cbed..4793bbf0d 100644 --- a/src/processors/animator.jl +++ b/src/processors/animator.jl @@ -15,6 +15,7 @@ Additional `kwargs` are passed to Makie's `VideoStream`. """ animator(setup, path; nupdate = 1, plotter = field_plotter(setup), kwargs...) = processor( function (state) + ispath(dirname(path)) || mkpath(dirname(path)) _state = Observable(state[]) fig = plotter.initialize(_state) stream = VideoStream(fig; kwargs...) From 0a0e8c97576feb7773537e6749fbb713d8707842 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Sun, 12 Nov 2023 21:24:10 +0100 Subject: [PATCH 122/379] Dimension agnostic training --- scratch/train_model.jl | 49 ++++++++++++++++++++++++++---------------- 1 file changed, 31 insertions(+), 18 deletions(-) diff --git a/scratch/train_model.jl b/scratch/train_model.jl index 3bf50c30c..da7d61a7d 100644 --- a/scratch/train_model.jl +++ b/scratch/train_model.jl @@ -72,9 +72,8 @@ end # data_train, data_valid, data_test = load("output/forced/data.jld2", "data_train", "data_valid", "data_test") # Build LES setup and assemble operators -x = LinRange(params.lims..., params.nles + 1) -y = LinRange(params.lims..., params.nles + 1) -setup = Setup(x, y; params.Re, ArrayType); +x = ntuple(α -> LinRange(params.lims..., params.nles + 1), params.D) +setup = Setup(x...; params.Re, ArrayType); # Uniform periodic grid pressure_solver = SpectralPressureSolver(setup); @@ -86,7 +85,7 @@ closure, θ₀ = cnn( [2, 2, 2, 2], # Channels - [5, 5, 5, 2], + [5, 5, 5, params.D], # Activations [leakyrelu, leakyrelu, leakyrelu, identity], @@ -157,35 +156,49 @@ Array(θ) # θθ = load("output/theta_fno.jld2") # copyto!(θ, θθ["theta"]) +function relerr(u, uref, setup) + (; dimension, Ip) = setup.grid + D = dimension() + a, b = T(0), T(0) + for α = 1:D + a += sum(abs2, u[α][Ip] - uref[α][Ip]) + b += sum(abs2, uref[α][Ip]) + end + sqrt(a) / sqrt(b) +end + +u = device(data_test.u[1][end]) u₀ = device(data_test.u[1][1]) p₀ = pressure_additional_solve(pressure_solver, u₀, T(0), setup) -u, p, outputs = solve_unsteady( +u_nm, p_nm, outputs = solve_unsteady( setup, copy.(u₀), copy(p₀), - (T(0), T(0.1)); - Δt = T(1e-4), + (T(0), params.tsim); + Δt = data_test.Δt, pressure_solver, - processors = (field_plotter(setup; nupdate = 10), step_logger(; nupdate = 1)), + processors = ( + # field_plotter(setup; nupdate = 10), + step_logger(; nupdate = 1), + ), ) -u, p, outputs = solve_unsteady( +u_cnn, p_cnn, outputs = solve_unsteady( (; setup..., closure_model = create_neural_closure(closure, θ, setup)), copy.(u₀), copy(p₀), - (T(0), T(0.1)); - Δt = T(1e-4), + (T(0), params.tsim); + Δt = data_test.Δt, pressure_solver, - processors = (field_plotter(setup; nupdate = 10), step_logger(; nupdate = 1)), + processors = ( + # field_plotter(setup; nupdate = 10), + step_logger(; nupdate = 1), + ), ) -relative_error(closure(device(data_train.V[:, 1, :]), θ), device(data_train.cF[:, 1, :])) -relative_error( - closure(device(data_train.V[:, end, :]), θ), - device(data_train.cF[:, end, :]), -) -relative_error(closure(u_test, θ), c_test) +relerr(u_nm, u, setup) +relerr(u_cnn, u, setup) function energy_history(setup, state) (; Ωp) = setup.grid From 3bb859c0c1c0492af69082aad0d27316cdf5e1ec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Sun, 12 Nov 2023 21:24:50 +0100 Subject: [PATCH 123/379] Improve initial spectrum --- src/create_initial_conditions.jl | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/create_initial_conditions.jl b/src/create_initial_conditions.jl index e90a8869a..f67cb8e74 100644 --- a/src/create_initial_conditions.jl +++ b/src/create_initial_conditions.jl @@ -81,9 +81,9 @@ function create_spectrum(N; A, σ, s, backend) a .*= A / sqrt(τ^2 * 2σ^2) for α = 1:D kα = k[α] - @. a *= randn(T) * exp(-(kα - s)^2 / 2σ^2) + @. a *= exp(-max(abs(kα) - s, s)^2 / 2σ^2) end - @. a *= exp(im * τ * rand(T)) + @. a *= randn(T) * exp(im * τ * rand(T)) for α = 1:D a = cat(a, reverse(a; dims = α); dims = α) end @@ -92,7 +92,7 @@ end """ random_field( - setup, t; + setup, t = 0; A = setup.grid.N[1] * 10_000, σ = 30, s = 5, @@ -108,7 +108,7 @@ Create random field. """ function random_field( setup, - t; + t = zero(eltype(setup.grid.x[1])); A = convert(eltype(setup.grid.x[1]), setup.grid.N[1] * 7_500), σ = convert(eltype(setup.grid.x[1]), 30), s = convert(eltype(setup.grid.x[1]), 5), From 8c29e6de82827c9658b5c8e10a35070e6b8d8b19 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Sun, 12 Nov 2023 21:26:11 +0100 Subject: [PATCH 124/379] fix: D instead of 2 --- src/closures/cnn.jl | 2 +- src/closures/fno.jl | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/closures/cnn.jl b/src/closures/cnn.jl index a14ec0419..da301400d 100644 --- a/src/closures/cnn.jl +++ b/src/closures/cnn.jl @@ -17,7 +17,7 @@ function cnn(setup, r, c, σ, b; channel_augmenter = identity, rng = Random.defa @assert c[end] == D # Add input channel size - c = [2; c] + c = [D; c] # Create convolutional closure model NN = Chain( diff --git a/src/closures/fno.jl b/src/closures/fno.jl index 0f33effef..0c3654516 100644 --- a/src/closures/fno.jl +++ b/src/closures/fno.jl @@ -16,7 +16,7 @@ function fno(setup, kmax, c, σ, ψ; rng = Random.default_rng(), kwargs...) @assert length(kmax) == length(c) == length(σ) # Make sure there are two velocity fields in input and output - c = [2; c] + c = [D; c] # Weight initializer T = eltype(x[1]) From cf533e3633604a524ab57abc1e68db1fd8938339 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Sun, 12 Nov 2023 21:26:40 +0100 Subject: [PATCH 125/379] Pass IC params to data generator --- src/closures/create_les_data.jl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/closures/create_les_data.jl b/src/closures/create_les_data.jl index de83891f6..056f9cbb9 100644 --- a/src/closures/create_les_data.jl +++ b/src/closures/create_les_data.jl @@ -110,6 +110,7 @@ function create_les_data( tsim = T(0.1), Δt = T(1e-4), ArrayType = Array, + ic_params = (;), ) ndns = compression * nles xdns = ntuple(α -> LinRange(lims..., ndns + 1), D) @@ -143,7 +144,7 @@ function create_les_data( # @info "Generating data for simulation $isim of $nsim" # Initial conditions - u₀, p₀ = random_field(dns, T(0); pressure_solver) + u₀, p₀ = random_field(dns, T(0); pressure_solver, ic_params...) # Random body force # force_dns = From 38ddc01a4238e09b64fa7cb3a32cd770f8412be8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Thu, 16 Nov 2023 17:11:55 +0100 Subject: [PATCH 126/379] Fix expression --- src/create_initial_conditions.jl | 2 +- src/processors/animator.jl | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/create_initial_conditions.jl b/src/create_initial_conditions.jl index f67cb8e74..3d1a8442f 100644 --- a/src/create_initial_conditions.jl +++ b/src/create_initial_conditions.jl @@ -81,7 +81,7 @@ function create_spectrum(N; A, σ, s, backend) a .*= A / sqrt(τ^2 * 2σ^2) for α = 1:D kα = k[α] - @. a *= exp(-max(abs(kα) - s, s)^2 / 2σ^2) + @. a *= exp(-max(abs(kα) - s, 0)^2 / 2σ^2) end @. a *= randn(T) * exp(im * τ * rand(T)) for α = 1:D diff --git a/src/processors/animator.jl b/src/processors/animator.jl index 4793bbf0d..5d0970709 100644 --- a/src/processors/animator.jl +++ b/src/processors/animator.jl @@ -15,7 +15,7 @@ Additional `kwargs` are passed to Makie's `VideoStream`. """ animator(setup, path; nupdate = 1, plotter = field_plotter(setup), kwargs...) = processor( function (state) - ispath(dirname(path)) || mkpath(dirname(path)) + ispath(dirname(path)) || mkpath(dirname(path)) _state = Observable(state[]) fig = plotter.initialize(_state) stream = VideoStream(fig; kwargs...) From cdcb33d798ad3ddea20cb17026ef101e1d021881 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Fri, 17 Nov 2023 13:24:18 +0100 Subject: [PATCH 127/379] Add setup --- scratch/multigrid.jl | 168 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 168 insertions(+) create mode 100644 scratch/multigrid.jl diff --git a/scratch/multigrid.jl b/scratch/multigrid.jl new file mode 100644 index 000000000..720dd2e5a --- /dev/null +++ b/scratch/multigrid.jl @@ -0,0 +1,168 @@ +# Little LSP hack to get function signatures, go #src +# to definition etc. #src +if isdefined(@__MODULE__, :LanguageServer) #src + include("../src/IncompressibleNavierStokes.jl") #src + using .IncompressibleNavierStokes #src +end #src + +# # Train closure model +# +# Here, we consider a periodic box ``[0, 1]^2``. It is discretized with a +# uniform Cartesian grid with square cells. + +using GLMakie +using IncompressibleNavierStokes +using JLD2 +using LinearAlgebra +using Lux +using NNlib +using Optimisers +using Random +using Zygote + +# Floating point precision +T = Float64 + +# Array type +ArrayType = Array +device = identity +## using CUDA; ArrayType = CuArray +## using AMDGPU; ArrayType = ROCArray +## using oneAPI; ArrayType = oneArray +## using Metal; ArrayType = MtlArray + +using LuxCUDA +using CUDA; +T = Float32; +ArrayType = CuArray; +CUDA.allowscalar(false); +device = cu + +# Parameters +get_params(nles) = (; + D = 2, + Re = T(10_000), + lims = (T(0), T(1)), + tburn = T(0.0), + tsim = T(1.0), + Δt = T(5e-4), + nles, + compression = 1024 ÷ nles, + ArrayType, + # ic_params = (; A = T(20_000_000), σ = T(5.0), s = T(3)), +) + +nles = [8, 16, 32, 64, 128] + +# Create LES data from DNS +data_train = [create_les_data(T; get_params(nles)..., nsim = 5) for nles in nles]; +data_valid = [create_les_data(T; get_params(nles)..., nsim = 1) for nles in nles]; +data_test = [create_les_data(T; get_params(nles)..., nsim = 1) for nles in nles]; + +# Inspect data +g = 5 +j = 1 +α = 1 +data_train[g].u[j][1][α] +o = Observable(data_train[g].u[j][1][α]) +heatmap(o) +for i = 1:1:length(data_train[g].u[j]) + o[] = data_train[g].u[j][i][α] + sleep(0.001) +end + +# # Save filtered DNS data +# jldsave("output/forced/data.jld2"; data_train, data_valid, data_test) + +# # Load previous LES data +# data_train, data_valid, data_test = load("output/forced/data.jld2", "data_train", "data_valid", "data_test") + +# Build LES setup and assemble operators +x = ntuple(α -> LinRange(params.lims..., params.nles + 1), params.D) +setup = Setup(x...; params.Re, ArrayType); + +# Uniform periodic grid +pressure_solver = SpectralPressureSolver(setup); + +closure, θ₀ = cnn( + setup, + + # Radius + [2, 2, 2, 2], + + # Channels + [5, 5, 5, params.D], + + # Activations + [leakyrelu, leakyrelu, leakyrelu, leakyrelu, identity], + + # Bias + [true, true, true, true, false]; +); +closure.NN + +# closure, θ₀ = fno( +# setup, +# +# # Cut-off wavenumbers +# [8, 8, 8, 8], +# +# # Channel sizes +# [8, 8, 8, 8], +# +# # Fourier layer activations +# [gelu, gelu, gelu, identity], +# +# # Dense activation +# gelu, +# ); +# closure.NN + +# Create input/output arrays +io_train = create_io_arrays(data_train, setup); +io_valid = create_io_arrays(data_valid, setup); +io_test = create_io_arrays(data_test, setup); + +size(io_train[1]) + +# Prepare training +θ = T(1.0e-1) * device(θ₀); +# θ = device(θ₀); +opt = Optimisers.setup(Adam(T(1.0e-3)), θ); +callbackstate = Point2f[]; +randloss = create_randloss(mean_squared_error, closure, io_train...; nuse = 50, device); + +# Warm-up +randloss(θ) +@time randloss(θ); +first(gradient(randloss, θ)); +@time first(gradient(randloss, θ)); +GC.gc() +CUDA.reclaim() + +# Training +# Note: The states `opt`, `θ`, and `callbackstate` +# will not be overwritten until training is finished. +# This allows for cancelling with "Control-C" should errors explode. +(; opt, θ, callbackstate) = train( + randloss, + opt, + θ; + niter = 50, + ncallback = 10, + callbackstate, + callback = create_callback(closure, device(io_valid)...; state = callbackstate), +); +GC.gc() +CUDA.reclaim() + +Array(θ) + +# # Save trained parameters +# jldsave("output/forced/theta_cnn.jld2"; theta = Array(θ)) +# jldsave("output/forced/theta_fno.jld2"; theta = Array(θ)) + +# # Load trained parameters +# θθ = load("output/theta_cnn.jld2") +# θθ = load("output/theta_fno.jld2") +# copyto!(θ, θθ["theta"]) From 0f149a47f86cee1a9835f564676249d3c99203dc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Fri, 17 Nov 2023 16:40:27 +0100 Subject: [PATCH 128/379] Use Tullio --- Project.toml | 4 ++++ src/IncompressibleNavierStokes.jl | 5 +++++ src/closures/fno.jl | 27 ++++++++++++--------------- 3 files changed, 21 insertions(+), 15 deletions(-) diff --git a/Project.toml b/Project.toml index 8061c4ca0..bc3664dd7 100644 --- a/Project.toml +++ b/Project.toml @@ -5,6 +5,7 @@ version = "0.4.1" [deps] Adapt = "79e6a3ab-5dfb-504d-930d-738a2a938a0e" +CUDA = "052768ef-5323-5732-b1bb-66c8b64840ba" ComponentArrays = "b0b7db55-cfe3-40fc-9ded-d10e2dbeff66" FFTW = "7a1cc6ca-52ef-59f5-83cd-3a7055c09341" IterativeSolvers = "42fd0dbc-a981-5370-80f2-aaf504508153" @@ -19,12 +20,14 @@ Printf = "de0858da-6303-5e67-8744-51eddeeeb8d7" Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" SparseArrays = "2f01184e-e22b-5df5-ae63-d93ebab69eaf" Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2" +Tullio = "bc48ee85-29a4-5162-ae0b-a64e1601d4bc" WriteVTK = "64499a7a-5c06-52f2-abe2-ccb03c286192" Zygote = "e88e6eb3-aa80-5325-afca-941959d7151f" [compat] Adapt = "3" ComponentArrays = "0.15" +CUDA = "5" FFTW = "1" IterativeSolvers = "0.9" KernelAbstractions = "0.9" @@ -38,6 +41,7 @@ Printf = "1" Random = "1" SparseArrays = "1" Statistics = "1" +Tullio = "3" WriteVTK = "1" Zygote = "0.6" julia = "1.7" diff --git a/src/IncompressibleNavierStokes.jl b/src/IncompressibleNavierStokes.jl index a442b0069..8464baa73 100644 --- a/src/IncompressibleNavierStokes.jl +++ b/src/IncompressibleNavierStokes.jl @@ -18,10 +18,15 @@ using Optimisers using Printf using Random using SparseArrays +using StaticArrays using Statistics +using Tullio using WriteVTK: CollectionFile, paraview_collection, vtk_grid, vtk_save using Zygote +# Must be loaded inside for Tullio to work correctly +using CUDA + # Workgroup size for kernels # Let this be constant for now const WORKGROUP = 64 diff --git a/src/closures/fno.jl b/src/closures/fno.jl index 0c3654516..ac9308b91 100644 --- a/src/closures/fno.jl +++ b/src/closures/fno.jl @@ -124,16 +124,15 @@ function ((; dimension, kmax, cout, cin, σ)::FourierLayer)(x, params, state) # Destructure params # The real and imaginary parts of R are stored in two separate channels W = params.spatial_weight - W = reshape(W, ntuple(Returns(1), N)..., cout, cin) R = params.spectral_weights R = selectdim(R, N + 3, 1) .+ im .* selectdim(R, N + 3, 2) # Spatial part (applied point-wise) - # Do matrix multiplication manually for now - # TODO: Make W*x more efficient with Tullio.jl - y = reshape(x, nx..., 1, cin, :) - y = sum(W .* y; dims = N + 2) - y = reshape(y, nx..., cout, :) + if N == 2 + @tullio y[i₁, i₂, b, s] := W[b, a] * x[i₁, i₂, a, s] + elseif N == 3 + @tullio y[i₁, i₂, i₃, b, s] := W[b, a] * x[i₁, i₂, i₃, a, s] + end # Spectral part (applied mode-wise) # @@ -144,17 +143,15 @@ function ((; dimension, kmax, cout, cin, σ)::FourierLayer)(x, params, state) # - multiply with weights mode-wise # - pad with zeros to restore original shape # - go back to real valued spatial representation - # - # We do matrix multiplications manually for now - # TODO: Make R*xhat more efficient with Tullio ikeep = ntuple(Returns(1:kmax+1), N) - nkeep = ntuple(Returns(kmax + 1), N) dims = ntuple(identity, N) - z = fft(x, dims) - z = z[ikeep..., :, :] - z = reshape(z, nkeep..., 1, cin, :) - z = sum(R .* z; dims = N + 2) - z = reshape(z, nkeep..., cout, :) + xhat = fft(x, dims) + xhat = xhat[ikeep..., :, :] + if N == 2 + @tullio z[k₁, k₂, b, s] := R[k₁, k₂, b, a] * xhat[k₁, k₂, a, s] + elseif N == 3 + @tullio z[k₁, k₂, k₃, b, s] := R[k₁, k₂, k₃, b, a] * xhat[k₁, k₂, k₃, a, s] + end z = pad_zeros(z, ntuple(i -> isodd(i) ? 0 : first(nx) - kmax - 1, 2N); dims) z = real.(ifft(z, dims)) From f220b358ae0825fd21e6ec4d73c4522c470e2c22 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Wed, 22 Nov 2023 10:11:48 +0100 Subject: [PATCH 129/379] Some fixes --- Project.toml | 4 +- examples/Actuator2D.jl | 7 +-- examples/BackwardFacingStep2D.jl | 15 +++-- examples/PlaneJets2D.jl | 4 +- examples/TaylorGreenVortex2D.jl | 39 +++++++----- scratch/multigrid.jl | 102 +++++++++++++++++++++++++------ scratch/train_model.jl | 99 +++++++++++++++++++++--------- src/closures/closure.jl | 2 +- src/operators.jl | 6 +- src/processors/processors.jl | 10 +-- test/runtests.jl | 5 +- 11 files changed, 205 insertions(+), 88 deletions(-) diff --git a/Project.toml b/Project.toml index bc3664dd7..ca1f9567a 100644 --- a/Project.toml +++ b/Project.toml @@ -26,8 +26,8 @@ Zygote = "e88e6eb3-aa80-5325-afca-941959d7151f" [compat] Adapt = "3" -ComponentArrays = "0.15" CUDA = "5" +ComponentArrays = "0.15" FFTW = "1" IterativeSolvers = "0.9" KernelAbstractions = "0.9" @@ -41,7 +41,7 @@ Printf = "1" Random = "1" SparseArrays = "1" Statistics = "1" -Tullio = "3" +Tullio = "0.3" WriteVTK = "1" Zygote = "0.6" julia = "1.7" diff --git a/examples/Actuator2D.jl b/examples/Actuator2D.jl index bccc53977..5357e411a 100644 --- a/examples/Actuator2D.jl +++ b/examples/Actuator2D.jl @@ -68,9 +68,8 @@ u, p = u₀, p₀ # Solve unsteady problem u, p, outputs = solve_unsteady( setup, - ## u₀, p₀, - u, - p, + copy.(u₀), copy(p₀), + # u, p, (0.0, 12.0); method = RK44P2(), Δt = 0.05, @@ -87,7 +86,7 @@ u, p, outputs = solve_unsteady( # ## Post-process # -# We may visualize or export the computed fields `(V, p)`. +# We may visualize or export the computed fields `(u, p)`. # We create a box to visualize the actuator. box = ( diff --git a/examples/BackwardFacingStep2D.jl b/examples/BackwardFacingStep2D.jl index e1ab85c7e..b95d55d9d 100644 --- a/examples/BackwardFacingStep2D.jl +++ b/examples/BackwardFacingStep2D.jl @@ -39,7 +39,7 @@ ArrayType = Array Re = T(3_000) # Boundary conditions: steady inflow on the top half -U(dim, x, y, t) = dim() == 1 && y ≥ 0 ? 24y * (one(x) / 2 - y) : zero(x) +U(dim, x, y, t) = dim() == 1 && y ≥ 0 ? 24y * (one(x) / 2 - y) : zero(x) + randn(typeof(x)) / 1_000 dUdt(dim, x, y, t) = zero(x) boundary_conditions = ( ## x left, x right @@ -58,8 +58,11 @@ plot_grid(x, y) # Build setup and assemble operators setup = Setup(x, y; Re, boundary_conditions, ArrayType); +pressure_solver = CGPressureSolverManual(setup); + # Initial conditions (extend inflow) -u₀, p₀ = create_initial_conditions(setup, (dim, x, y) -> U(dim, x, y, zero(x))); +u₀, p₀ = create_initial_conditions(setup, (dim, x, y) -> U(dim, x, y, zero(x)); pressure_solver); +u, p = copy.(u₀), copy(p₀) # Solve steady state problem ## u, p = solve_steady_state(setup, u₀, p₀); @@ -67,12 +70,13 @@ u₀, p₀ = create_initial_conditions(setup, (dim, x, y) -> U(dim, x, y, zero(x # Solve unsteady problem u, p, outputs = solve_unsteady( setup, - u₀, - p₀, + copy.(u₀), copy(p₀), + # u, p, (T(0), T(7)); Δt = T(0.002), + pressure_solver, processors = ( - field_plotter(setup; nupdate = 5), + field_plotter(setup; nupdate = 1), ## energy_history_plotter(setup; nupdate = 10), ## energy_spectrum_plotter(setup; nupdate = 10), ## animator(setup, "vorticity.mkv"; nupdate = 4), @@ -80,7 +84,6 @@ u, p, outputs = solve_unsteady( ## field_saver(setup; nupdate = 10), step_logger(; nupdate = 1), ), - inplace = true, ); # ## Post-process diff --git a/examples/PlaneJets2D.jl b/examples/PlaneJets2D.jl index 0ec9a24a3..9cd8937bf 100644 --- a/examples/PlaneJets2D.jl +++ b/examples/PlaneJets2D.jl @@ -111,7 +111,7 @@ mean_plotter(setup; nupdate = 1) = processor( umean = @lift begin (; u, p, t) = $state - up = IncompressibleNavierStokes.interpolate_u_p(setup, u) + up = IncompressibleNavierStokes.interpolate_u_p(u, setup) u1 = u[1] reshape(sum(u1[Ip]; dims = 1), :) ./ size(u1, 1) ./ V() end @@ -176,7 +176,7 @@ toto, p, outputs = solve_unsteady( Δt = 0.001, pressure_solver, processors = ( - field_plotter(setup; nupdate = 1), + # field_plotter(setup; nupdate = 1), ## energy_history_plotter(setup; nupdate = 1), ## energy_spectrum_plotter(setup; nupdate = 100), ## animator(setup, "vorticity.mkv"; nupdate = 4), diff --git a/examples/TaylorGreenVortex2D.jl b/examples/TaylorGreenVortex2D.jl index f7e7196a9..7dd5a2007 100644 --- a/examples/TaylorGreenVortex2D.jl +++ b/examples/TaylorGreenVortex2D.jl @@ -31,7 +31,6 @@ function compute_convergence(; D, nlist, lims, Re, tlims, Δt, uref, ArrayType = @info "Computing error for n = $n" x = ntuple(α -> LinRange(lims..., n + 1), D) setup = Setup(x...; Re, ArrayType) - (; Ip) = setup.grid pressure_solver = SpectralPressureSolver(setup) u₀, p₀ = create_initial_conditions( setup, @@ -46,10 +45,13 @@ function compute_convergence(; D, nlist, lims, Re, tlims, Δt, uref, ArrayType = pressure_solver, ) u, p, outputs = solve_unsteady(setup, u₀, p₀, tlims; Δt, pressure_solver) + (; Ip) = setup.grid + a, b = T(0), T(0) for α = 1:D - e[i] += norm(u[α][Ip] - ut[α][Ip]) / norm(ut[α][Ip]) + a += sum(abs2, u[α][Ip] - ut[α][Ip]) + b += sum(abs2, ut[α][Ip]) end - e[i] /= D + e[i] = sqrt(a) / sqrt(b) end e end @@ -70,15 +72,22 @@ e = compute_convergence(; ) # Plot convergence -fig = Figure() -ax = Axis( - fig[1, 1]; - xscale = log10, - yscale = log10, - xticks = nlist, - xlabel = "n", - title = "Relative error", -) -scatterlines!(nlist, e; label = "Data") -lines!(collect(extrema(nlist)), n -> n^-2.0; linestyle = :dash, label = "n^-2") -axislegend() +with_theme(; + # linewidth = 5, + # markersize = 20, + # fontsize = 20, +) do + fig = Figure() + ax = Axis( + fig[1, 1]; + xscale = log10, + yscale = log10, + xticks = nlist, + xlabel = "n", + title = "Relative error", + ) + scatterlines!(nlist, e; label = "Data") + lines!(collect(extrema(nlist)), n -> n^-2.0; linestyle = :dash, label = "n^-2") + axislegend() + fig +end diff --git a/scratch/multigrid.jl b/scratch/multigrid.jl index 720dd2e5a..90ff6de56 100644 --- a/scratch/multigrid.jl +++ b/scratch/multigrid.jl @@ -41,26 +41,26 @@ device = cu # Parameters get_params(nles) = (; D = 2, - Re = T(10_000), + Re = T(6_000), lims = (T(0), T(1)), - tburn = T(0.0), - tsim = T(1.0), - Δt = T(5e-4), + tburn = T(0.05), + tsim = T(0.05), + Δt = T(1e-4), nles, - compression = 1024 ÷ nles, + compression = 2048 ÷ nles, ArrayType, # ic_params = (; A = T(20_000_000), σ = T(5.0), s = T(3)), + ic_params = (; A = T(10_000_000)), ) -nles = [8, 16, 32, 64, 128] - # Create LES data from DNS -data_train = [create_les_data(T; get_params(nles)..., nsim = 5) for nles in nles]; -data_valid = [create_les_data(T; get_params(nles)..., nsim = 1) for nles in nles]; -data_test = [create_les_data(T; get_params(nles)..., nsim = 1) for nles in nles]; +data_train = [create_les_data(T; get_params(nles)..., nsim = 5) for nles in [32, 64, 128]]; +data_valid = [create_les_data(T; get_params(nles)..., nsim = 1) for nles in [128]]; +ntest = [8, 16, 32, 64, 128, 256, 512, 1024] +data_test = [create_les_data(T; get_params(nles)..., nsim = 1) for nles in ntest]; # Inspect data -g = 5 +g = 3 j = 1 α = 1 data_train[g].u[j][1][α] @@ -77,12 +77,80 @@ end # # Load previous LES data # data_train, data_valid, data_test = load("output/forced/data.jld2", "data_train", "data_valid", "data_test") -# Build LES setup and assemble operators -x = ntuple(α -> LinRange(params.lims..., params.nles + 1), params.D) -setup = Setup(x...; params.Re, ArrayType); - -# Uniform periodic grid -pressure_solver = SpectralPressureSolver(setup); +relerr_track(uref, setup) = processor(function (state) + (; dimension, x, Ip) = setup.grid + D = dimension() + T = eltype(x[1]) + e = Ref(T(0)) + on(state) do (; u, n) + a, b = T(0), T(0) + for α = 1:D + # @show size(uref[n + 1]) + a += sum(abs2, u[α][Ip] - uref[n+1][α][Ip]) + b += sum(abs2, uref[n+1][α][Ip]) + end + e[] += sqrt(a) / sqrt(b) / (length(uref) - 1) + end + e +end) + +e_nm = zeros(T, length(ntest)) +for (i, n) in enumerate(ntest) + params = get_params(n) + # Build LES setup and assemble operators + x = ntuple(α -> LinRange(params.lims..., params.nles + 1), params.D) + setup = Setup(x...; params.Re, ArrayType) + # Uniform periodic grid + pressure_solver = SpectralPressureSolver(setup) + u = device.(data_test[i].u[1]) + u₀ = device(data_test[i].u[1][1]) + p₀ = pressure_additional_solve(pressure_solver, u₀, T(0), setup) + u_nm, p_nm, outputs = solve_unsteady( + setup, + copy.(u₀), + copy(p₀), + (T(0), params.tsim); + Δt = data_test[i].Δt, + pressure_solver, + processors = ( + relerr_track(u, setup), + # step_logger(; nupdate = 1), + ), + ) + e_nm[i] = outputs[1][] +end +e_nm +e_cnn = ones(T, length(ntest)) +e_fno_share = ones(T, length(ntest)) +e_fno_spec = ones(T, length(ntest)) + +using CairoMakie +CairoMakie.activate!() + +# Plot convergence +with_theme(; +# linewidth = 5, +# markersize = 20, +# fontsize = 20, +) do + fig = Figure() + ax = Axis( + fig[1, 1]; + xscale = log10, + yscale = log10, + xticks = ntest, + xlabel = "n", + title = "Relative error (DNS: n = 2048)", + ) + scatterlines!(ntest, e; label = "No closure") + scatterlines!(ntest, e_cnn; label = "CNN") + scatterlines!(ntest, e_fno_spec; label = "FNO (retrained)") + scatterlines!(ntest, e_fno_share; label = "FNO (shared parameters)") + lines!(collect(extrema(ntest)), n -> 100n^-2.0; linestyle = :dash, label = "n^-2") + axislegend(; position = :lb) + fig +end +save("convergence.pdf", current_figure()) closure, θ₀ = cnn( setup, diff --git a/scratch/train_model.jl b/scratch/train_model.jl index da7d61a7d..c56f7dbe5 100644 --- a/scratch/train_model.jl +++ b/scratch/train_model.jl @@ -44,24 +44,30 @@ params = (; Re = T(6_000), lims = (T(0), T(1)), nles = 128, - compression = 4, + compression = 8, tburn = T(0.05), - tsim = T(0.05), + tsim = T(0.2), Δt = T(1e-4), ArrayType, + # ic_params = (; A = T(20_000_000), σ = T(5.0), s = T(3)), + ic_params = (; A = T(10_000_000)), ) # Create LES data from DNS data_train = create_les_data(T; params..., nsim = 10); -data_valid = create_les_data(T; params..., nsim = 2); +data_valid = create_les_data(T; params..., nsim = 1); data_test = create_les_data(T; params..., nsim = 5); # Inspect data -o = Observable(data_train.u[1][end][1]) +isim = 1 +α = 1 +# j = 13 +o = Observable(data_train.u[isim][1][α]) +# o = Observable(data_train.u[isim][1][α][:, :, j]) heatmap(o) -for i = 1:501 - o[] = data_train.u[1][i][1] - # o[] = data_train.c[1][i][1] +for i = 1:length(data_train.u[isim]) + o[] = data_train.u[isim][i][α] + # o[] = data_train.u[isim][i][α][:, :, j] sleep(0.001) end @@ -78,30 +84,28 @@ setup = Setup(x...; params.Re, ArrayType); # Uniform periodic grid pressure_solver = SpectralPressureSolver(setup); -closure, θ₀ = cnn( +closure, θ₀ = cnn(; setup, - - # Radius - [2, 2, 2, 2], - - # Channels - [5, 5, 5, params.D], - - # Activations - [leakyrelu, leakyrelu, leakyrelu, identity], - - # Bias - [true, true, true, false]; + radii = [2, 2, 2, 2], + channels = [32, 32, 32, params.D], + activations = [leakyrelu, leakyrelu, leakyrelu, identity], + bias = [true, true, true, false], ); +closure.NN +sample = io_train[1][:, :, :, 1:5] +closure(sample, θ₀) |> size + +θ.layer_5 +θ.layer_6 # closure, θ₀ = fno( # setup, # # # Cut-off wavenumbers -# [32, 32, 32, 32], +# [8, 8, 8, 8], # # # Channel sizes -# [24, 12, 8, 8], +# [16, 16, 16, 16], # # # Fourier layer activations # [gelu, gelu, gelu, identity], @@ -109,16 +113,18 @@ closure, θ₀ = cnn( # # Dense activation # gelu, # ); - -closure.NN +# closure.NN # Create input/output arrays io_train = create_io_arrays(data_train, setup); io_valid = create_io_arrays(data_valid, setup); io_test = create_io_arrays(data_test, setup); +size(io_train[1]) + # Prepare training θ = T(1.0e-1) * device(θ₀); +# θ = device(θ₀); opt = Optimisers.setup(Adam(T(1.0e-3)), θ); callbackstate = Point2f[]; randloss = create_randloss(mean_squared_error, closure, io_train...; nuse = 50, device); @@ -128,6 +134,8 @@ randloss(θ) @time randloss(θ); first(gradient(randloss, θ)); @time first(gradient(randloss, θ)); +GC.gc() +CUDA.reclaim() # Training # Note: The states `opt`, `θ`, and `callbackstate` @@ -137,8 +145,8 @@ first(gradient(randloss, θ)); randloss, opt, θ; - niter = 2000, - ncallback = 10, + niter = 1000, + ncallback = 20, callbackstate, callback = create_callback(closure, device(io_valid)...; state = callbackstate), ); @@ -167,9 +175,28 @@ function relerr(u, uref, setup) sqrt(a) / sqrt(b) end -u = device(data_test.u[1][end]) +relerr_track(uref, setup) = processor(function (state) + (; dimension, x, Ip) = setup.grid + D = dimension() + T = eltype(x[1]) + e = Ref(T(0)) + on(state) do (; u, n) + a, b = T(0), T(0) + for α = 1:D + # @show size(uref[n + 1]) + a += sum(abs2, u[α][Ip] - uref[n+1][α][Ip]) + b += sum(abs2, uref[n+1][α][Ip]) + end + e[] += sqrt(a) / sqrt(b) / (length(uref) - 1) + end + e +end) + +u, u₀, p₀ = nothing, nothing, nothing +u = device.(data_test.u[1]) u₀ = device(data_test.u[1][1]) p₀ = pressure_additional_solve(pressure_solver, u₀, T(0), setup) +length(u) u_nm, p_nm, outputs = solve_unsteady( setup, @@ -179,10 +206,22 @@ u_nm, p_nm, outputs = solve_unsteady( Δt = data_test.Δt, pressure_solver, processors = ( + relerr_track(u, setup), # field_plotter(setup; nupdate = 10), + # field_plotter( + # setup; + # nupdate = 10, + # fieldname = :λ₂field, + # levels = LinRange(-T(2), T(3), 5), + # # levels = LinRange(-1.0f0, 3.0f0, 5), + # # levels = LinRange(-2.0f0, 2.0f0, 5), + # # levels=5, + # docolorbar = false, + # ), step_logger(; nupdate = 1), ), ) +relerr_nm = outputs[1][] u_cnn, p_cnn, outputs = solve_unsteady( (; setup..., closure_model = create_neural_closure(closure, θ, setup)), @@ -192,13 +231,15 @@ u_cnn, p_cnn, outputs = solve_unsteady( Δt = data_test.Δt, pressure_solver, processors = ( + relerr_track(u, setup), # field_plotter(setup; nupdate = 10), step_logger(; nupdate = 1), ), ) +relerr_cnn = outputs[1][] -relerr(u_nm, u, setup) -relerr(u_cnn, u, setup) +relerr_nm +relerr_cnn function energy_history(setup, state) (; Ωp) = setup.grid diff --git a/src/closures/closure.jl b/src/closures/closure.jl index ed50b0cd4..e3f765889 100644 --- a/src/closures/closure.jl +++ b/src/closures/closure.jl @@ -5,7 +5,7 @@ function create_neural_closure(m, θ, setup) u = stack(ntuple(α -> u[α][Iu[α]], D)) u = reshape(u, size(u)..., 1) # One sample mu = m(u, θ) - mu = pad_circular(u, 1) + mu = pad_circular(mu, 1) sz..., _ = size(mu) i = ntuple(Returns(:), D) mu = ntuple(α -> mu[i..., α, 1], D) diff --git a/src/operators.jl b/src/operators.jl index 38e8f320e..d8fb16661 100644 --- a/src/operators.jl +++ b/src/operators.jl @@ -1,7 +1,7 @@ """ δ = Offset{D}() -Carsesian index unit vector in `D = 2` or `D = 3` dimensions. +Cartesian index unit vector in `D = 2` or `D = 3` dimensions. Calling `δ(α)` returns a Cartesian index with `1` in the dimension `α` and zeros elsewhere. @@ -201,10 +201,10 @@ function bodyforce!(F, u, t, setup) (; dimension, Δ, Δu, Nu, Iu, x, xp) = grid D = dimension() δ = Offset{D}() - @kernel function _bodyforce!(F, force, valα::Val{α}, t, I0) where {α} + @kernel function _bodyforce!(F, force, ::Val{α}, t, I0) where {α} I = @index(Global, Cartesian) I = I + I0 - F[α][I] += force(valα, ntuple(β -> α == β ? x[β][1+I[β]] : xp[β][I[β]], D)..., t) + F[α][I] += force(Dimension(α), ntuple(β -> α == β ? x[β][1+I[β]] : xp[β][I[β]], D)..., t) end for α = 1:D I0 = first(Iu[α]) diff --git a/src/processors/processors.jl b/src/processors/processors.jl index 378b869dc..957d1c399 100644 --- a/src/processors/processors.jl +++ b/src/processors/processors.jl @@ -1,4 +1,4 @@ -raw""" +""" processor( initialize; finalize = (initialized, stepper) -> initialized, @@ -16,14 +16,14 @@ function initialize(step_observer) s = 0 println("Let's sum up the time steps") @lift begin - (; n) = $step_observer - println("The summand is $n") + (; n) = \$step_observer + println("The summand is \$n") s = s + n end s end -finalize(s, stepper) = println("The final sum (at time t=$(stepper.t)) is $s") +finalize(s, stepper) = println("The final sum (at time t=\$(stepper.t)) is \$s") p = Processor(initialize; finalize, nupdate = 5) ``` @@ -80,7 +80,7 @@ vtk_writer( @lift begin (; grid) = setup (; dimension, xp, yp) = grid - (; V, p, t) = $step_observer + (; u, p, t) = $step_observer N = dimension() if N == 2 diff --git a/test/runtests.jl b/test/runtests.jl index 29f8517be..db8378bd2 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -24,9 +24,6 @@ using Test @testset "Aqua" begin @info "Testing code with Aqua" - Aqua.test_all( - IncompressibleNavierStokes; - ambiguities = false, - ) + Aqua.test_all(IncompressibleNavierStokes; ambiguities = false) end end From 451e0e21a8032e5f8bfd97c9eb946e325da51eb9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Wed, 22 Nov 2023 10:16:33 +0100 Subject: [PATCH 130/379] Add input/output manipulation for staggered grid --- scratch/train_model.jl | 10 ++--- src/closures/cnn.jl | 83 +++++++++++++++++++++++++++++++++++++++++- src/closures/fno.jl | 67 +++++++++++++++++++++++++++++++++- 3 files changed, 151 insertions(+), 9 deletions(-) diff --git a/scratch/train_model.jl b/scratch/train_model.jl index c56f7dbe5..3542229a2 100644 --- a/scratch/train_model.jl +++ b/scratch/train_model.jl @@ -98,20 +98,20 @@ closure(sample, θ₀) |> size θ.layer_5 θ.layer_6 -# closure, θ₀ = fno( +# closure, θ₀ = fno(; # setup, # # # Cut-off wavenumbers -# [8, 8, 8, 8], +# k = [8, 8, 8, 8], # # # Channel sizes -# [16, 16, 16, 16], +# c = [16, 16, 16, 16], # # # Fourier layer activations -# [gelu, gelu, gelu, identity], +# σ = [gelu, gelu, gelu, identity], # # # Dense activation -# gelu, +# ψ = gelu, # ); # closure.NN diff --git a/src/closures/cnn.jl b/src/closures/cnn.jl index da301400d..f95be102c 100644 --- a/src/closures/cnn.jl +++ b/src/closures/cnn.jl @@ -1,10 +1,27 @@ """ - cnn(setup, r, c, σ, b; kwargs...) + cnn(; + setup, + radii, + channels, + activations, + use_bias, + channel_augmenter = identity, + rng = Random.default_rng(), + ) Create CNN closure model. Return a tuple `(closure, θ)` where `θ` are the initial parameters and `closure(u, θ)` predicts the commutator error. """ -function cnn(setup, r, c, σ, b; channel_augmenter = identity, rng = Random.default_rng()) +function cnn(; + setup, + radii, + channels, + activations, + use_bias, + channel_augmenter = identity, + rng = Random.default_rng(), +) + r, c, σ, b = radii, channels, activations, use_bias (; grid) = setup (; dimension, x) = grid D = dimension() @@ -21,6 +38,34 @@ function cnn(setup, r, c, σ, b; channel_augmenter = identity, rng = Random.defa # Create convolutional closure model NN = Chain( + function (u) + sz..., _, _ = size(u) + # for α = 1:D + # v = selectdim(u, D + 1, α) + # v = (v + circshift(v, ntuple(β -> α == β ? -1 : 0, D + 1))) / 2 + # end + if D == 2 + a = selectdim(u, 3, 1) + b = selectdim(u, 3, 2) + a = (a + circshift(a, (-1, 0, 0))) / 2 + b = (b + circshift(b, (0, -1, 0))) / 2 + a = reshape(a, sz..., 1, :) + b = reshape(b, sz..., 1, :) + cat(a, b; dims = 3) + elseif D == 3 + a = selectdim(u, 4, 1) + b = selectdim(u, 4, 2) + c = selectdim(u, 4, 3) + a = (a + circshift(a, (-1, 0, 0, 0))) / 2 + b = (b + circshift(b, (0, -1, 0, 0))) / 2 + c = (c + circshift(c, (0, 0, -1, 0))) / 2 + a = reshape(a, sz..., 1, :) + b = reshape(b, sz..., 1, :) + c = reshape(c, sz..., 1, :) + cat(a, b, c; dims = 4) + end + end, + # Add padding so that output has same shape as commutator error u -> pad_circular(u, sum(r)), @@ -34,6 +79,40 @@ function cnn(setup, r, c, σ, b; channel_augmenter = identity, rng = Random.defa init_weight = glorot_uniform_T, ) for i ∈ eachindex(r) )..., + + # Differentiate output to velocity points + function (u) + sz..., _, _ = size(u) + # for α = 1:D + # v = selectdim(u, D + 1, α) + # v = (v + circshift(v, ntuple(β -> α == β ? -1 : 0, D + 1))) / 2 + # end + if D == 2 + a = selectdim(u, 3, 1) + b = selectdim(u, 3, 2) + # a = (a + circshift(a, (1, 0, 0, 0))) / 2 + # b = (b + circshift(b, (0, 1, 0, 0))) / 2 + a = circshift(a, (1, 0, 0)) - a + b = circshift(b, (0, 1, 0)) - b + a = reshape(a, sz..., 1, :) + b = reshape(b, sz..., 1, :) + cat(a, b; dims = 3) + elseif D == 3 + a = selectdim(u, 4, 1) + b = selectdim(u, 4, 2) + c = selectdim(u, 4, 3) + # a = (a + circshift(a, (1, 0, 0, 0))) / 2 + # b = (b + circshift(b, (0, 1, 0, 0))) / 2 + # c = (c + circshift(c, (0, 0, 1, 0))) / 2 + a = circshift(a, (1, 0, 0, 0)) - a + b = circshift(b, (0, 1, 0, 0)) - b + c = circshift(c, (0, 0, 1, 0)) - c + a = reshape(a, sz..., 1, :) + b = reshape(b, sz..., 1, :) + c = reshape(c, sz..., 1, :) + cat(a, b, c; dims = 4) + end + end, ) # Create parameter vector (empty state) diff --git a/src/closures/fno.jl b/src/closures/fno.jl index ac9308b91..a5f8a23e4 100644 --- a/src/closures/fno.jl +++ b/src/closures/fno.jl @@ -1,10 +1,10 @@ """ - fno(setup, kmax, c, σ, ψ; rng = Random.default_rng(), kwargs...) + fno(; setup, kmax, c, σ, ψ, rng = Random.default_rng(), kwargs...) Create FNO closure model. Return a tuple `(closure, θ)` where `θ` are the initial parameters and `closure(V, θ)` predicts the commutator error. """ -function fno(setup, kmax, c, σ, ψ; rng = Random.default_rng(), kwargs...) +function fno(; setup, kmax, c, σ, ψ, rng = Random.default_rng(), kwargs...) (; grid) = setup (; dimension, x, N) = grid @@ -24,6 +24,35 @@ function fno(setup, kmax, c, σ, ψ; rng = Random.default_rng(), kwargs...) # Create FNO closure model NN = Chain( + # Put inputs in pressure points + function (u) + sz..., _, _ = size(u) + # for α = 1:D + # v = selectdim(u, D + 1, α) + # v = (v + circshift(v, ntuple(β -> α == β ? -1 : 0, D + 1))) / 2 + # end + if D == 2 + a = selectdim(u, 3, 1) + b = selectdim(u, 3, 2) + a = (a + circshift(a, (-1, 0, 0))) / 2 + b = (b + circshift(b, (0, -1, 0))) / 2 + a = reshape(a, sz..., 1, :) + b = reshape(b, sz..., 1, :) + cat(a, b; dims = 3) + elseif D == 3 + a = selectdim(u, 4, 1) + b = selectdim(u, 4, 2) + c = selectdim(u, 4, 3) + a = (a + circshift(a, (-1, 0, 0, 0))) / 2 + b = (b + circshift(b, (0, -1, 0, 0))) / 2 + c = (c + circshift(c, (0, 0, -1, 0))) / 2 + a = reshape(a, sz..., 1, :) + b = reshape(b, sz..., 1, :) + c = reshape(c, sz..., 1, :) + cat(a, b, c; dims = 4) + end + end, + # Some Fourier layers ( FourierLayer(dimension, kmax[i], c[i] => c[i+1]; σ = σ[i], init_weight) for @@ -39,6 +68,40 @@ function fno(setup, kmax, c, σ, ψ; rng = Random.default_rng(), kwargs...) # Put channels back after spatial dimensions u -> permutedims(u, ((2:D+1)..., 1, D + 2)), + + # Differentiate output to velocity points + function (u) + sz..., _, _ = size(u) + # for α = 1:D + # v = selectdim(u, D + 1, α) + # v = (v + circshift(v, ntuple(β -> α == β ? -1 : 0, D + 1))) / 2 + # end + if D == 2 + a = selectdim(u, 3, 1) + b = selectdim(u, 3, 2) + # a = (a + circshift(a, (1, 0, 0, 0))) / 2 + # b = (b + circshift(b, (0, 1, 0, 0))) / 2 + a = circshift(a, (1, 0, 0)) - a + b = circshift(b, (0, 1, 0)) - b + a = reshape(a, sz..., 1, :) + b = reshape(b, sz..., 1, :) + cat(a, b; dims = 3) + elseif D == 3 + a = selectdim(u, 4, 1) + b = selectdim(u, 4, 2) + c = selectdim(u, 4, 3) + # a = (a + circshift(a, (1, 0, 0, 0))) / 2 + # b = (b + circshift(b, (0, 1, 0, 0))) / 2 + # c = (c + circshift(c, (0, 0, 1, 0))) / 2 + a = circshift(a, (1, 0, 0, 0)) - a + b = circshift(b, (0, 1, 0, 0)) - b + c = circshift(c, (0, 0, 1, 0)) - c + a = reshape(a, sz..., 1, :) + b = reshape(b, sz..., 1, :) + c = reshape(c, sz..., 1, :) + cat(a, b, c; dims = 4) + end + end, ) # Create parameter vector (empty state) From cdb36fe039c61e83d6beb26e081767a3a64ffa88 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Wed, 22 Nov 2023 18:42:52 +0100 Subject: [PATCH 131/379] Fix FFT normalization --- scratch/multigrid.jl | 11 ++++++----- src/create_initial_conditions.jl | 6 +++--- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/scratch/multigrid.jl b/scratch/multigrid.jl index 90ff6de56..48ffff9b4 100644 --- a/scratch/multigrid.jl +++ b/scratch/multigrid.jl @@ -50,17 +50,17 @@ get_params(nles) = (; compression = 2048 ÷ nles, ArrayType, # ic_params = (; A = T(20_000_000), σ = T(5.0), s = T(3)), - ic_params = (; A = T(10_000_000)), + # ic_params = (; A = T(10)), ) # Create LES data from DNS data_train = [create_les_data(T; get_params(nles)..., nsim = 5) for nles in [32, 64, 128]]; data_valid = [create_les_data(T; get_params(nles)..., nsim = 1) for nles in [128]]; -ntest = [8, 16, 32, 64, 128, 256, 512, 1024] +ntest = [8, 16, 32, 64, 128, 256, 512] data_test = [create_les_data(T; get_params(nles)..., nsim = 1) for nles in ntest]; # Inspect data -g = 3 +g = 4 j = 1 α = 1 data_train[g].u[j][1][α] @@ -124,7 +124,7 @@ e_cnn = ones(T, length(ntest)) e_fno_share = ones(T, length(ntest)) e_fno_spec = ones(T, length(ntest)) -using CairoMakie +using CairoMakie CairoMakie.activate!() # Plot convergence @@ -142,7 +142,7 @@ with_theme(; xlabel = "n", title = "Relative error (DNS: n = 2048)", ) - scatterlines!(ntest, e; label = "No closure") + scatterlines!(ntest, e_nm; label = "No closure") scatterlines!(ntest, e_cnn; label = "CNN") scatterlines!(ntest, e_fno_spec; label = "FNO (retrained)") scatterlines!(ntest, e_fno_share; label = "FNO (shared parameters)") @@ -150,6 +150,7 @@ with_theme(; axislegend(; position = :lb) fig end + save("convergence.pdf", current_figure()) closure, θ₀ = cnn( diff --git a/src/create_initial_conditions.jl b/src/create_initial_conditions.jl index 3d1a8442f..5744c3091 100644 --- a/src/create_initial_conditions.jl +++ b/src/create_initial_conditions.jl @@ -78,7 +78,7 @@ function create_spectrum(N; A, σ, s, backend) # k = AT.(Array{Complex{T}}.(k)) # k = AT.(k) τ = T(2π) - a .*= A / sqrt(τ^2 * 2σ^2) + a .*= prod(N) * A / sqrt(τ^2 * 2σ^2) for α = 1:D kα = k[α] @. a *= exp(-max(abs(kα) - s, 0)^2 / 2σ^2) @@ -93,7 +93,7 @@ end """ random_field( setup, t = 0; - A = setup.grid.N[1] * 10_000, + A = 10, σ = 30, s = 5, pressure_solver = DirectPressureSolver(setup), @@ -109,7 +109,7 @@ Create random field. function random_field( setup, t = zero(eltype(setup.grid.x[1])); - A = convert(eltype(setup.grid.x[1]), setup.grid.N[1] * 7_500), + A = convert(eltype(setup.grid.x[1]), 10), σ = convert(eltype(setup.grid.x[1]), 30), s = convert(eltype(setup.grid.x[1]), 5), pressure_solver = DirectPressureSolver(setup), From 648a5435bce4eb43d29e4e74daade560e86f969a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Fri, 24 Nov 2023 10:39:18 +0100 Subject: [PATCH 132/379] Copy IC by default --- src/solvers/solve_unsteady.jl | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/solvers/solve_unsteady.jl b/src/solvers/solve_unsteady.jl index a3d32dddf..ed0c9fae9 100644 --- a/src/solvers/solve_unsteady.jl +++ b/src/solvers/solve_unsteady.jl @@ -53,8 +53,14 @@ function solve_unsteady( cfl = 1, n_adapt_Δt = 1, inplace = true, + docopy = true, processors = (), ) + if docopy + u₀ = copy.(u₀) + p₀ = copy(p₀) + end + t_start, t_end = tlims isadaptive = isnothing(Δt) if !isadaptive From c2817e331595afc4b7e5274eb5b54ad3662def46 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Fri, 24 Nov 2023 17:57:52 +0100 Subject: [PATCH 133/379] Add dependency --- Project.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/Project.toml b/Project.toml index ef6c0fd73..7505d1f76 100644 --- a/Project.toml +++ b/Project.toml @@ -19,6 +19,7 @@ Optimisers = "3bd65402-5787-11e9-1adc-39752487f4e2" Printf = "de0858da-6303-5e67-8744-51eddeeeb8d7" Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" SparseArrays = "2f01184e-e22b-5df5-ae63-d93ebab69eaf" +StaticArrays = "90137ffa-7385-5640-81b9-e52037218182" Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2" Tullio = "bc48ee85-29a4-5162-ae0b-a64e1601d4bc" WriteVTK = "64499a7a-5c06-52f2-abe2-ccb03c286192" From 09823b8130fcc89d7f9e46a9af1f32f2f6f93fe4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Fri, 24 Nov 2023 19:40:09 +0100 Subject: [PATCH 134/379] Update processors --- README.md | 2 +- docs/src/api/api.md | 7 +- examples/Actuator2D.jl | 18 +-- examples/Actuator3D.jl | 18 +-- examples/BackwardFacingStep2D.jl | 20 ++-- examples/BackwardFacingStep3D.jl | 18 +-- examples/DecayingTurbulence2D.jl | 29 +++-- examples/DecayingTurbulence3D.jl | 18 +-- examples/LidDrivenCavity2D.jl | 18 +-- examples/LidDrivenCavity3D.jl | 18 +-- examples/PlanarMixing2D.jl | 18 +-- examples/PlaneJets2D.jl | 144 ++++++++++++------------ examples/ShearLayer2D.jl | 22 ++-- examples/TaylorGreenVortex3D.jl | 29 ++--- scratch/filter_plot.jl | 6 +- scratch/filtered.jl | 180 ------------------------------ scratch/multigrid.jl | 42 +++---- scratch/train_model.jl | 37 +++--- src/IncompressibleNavierStokes.jl | 4 +- src/closures/create_les_data.jl | 10 +- src/operators.jl | 1 + src/processors/animator.jl | 20 ++-- src/processors/processors.jl | 146 +++++++++++------------- src/processors/real_time_plot.jl | 141 ++++++++++++----------- src/solvers/solve_unsteady.jl | 40 ++----- test/postprocess2D.jl | 4 +- test/postprocess3D.jl | 4 +- test/simulation2D.jl | 2 +- test/simulation3D.jl | 2 +- test/solvers.jl | 2 +- 30 files changed, 403 insertions(+), 617 deletions(-) delete mode 100644 scratch/filtered.jl diff --git a/README.md b/README.md index db7960207..c4f25222a 100644 --- a/README.md +++ b/README.md @@ -100,7 +100,7 @@ u, p, outputs = solve_unsteady( Δt = 0.05, processors = ( animator(setup, "vorticity.mp4"; nupdate = 4), - step_logger(), + timelogger(), ), ) ``` diff --git a/docs/src/api/api.md b/docs/src/api/api.md index f0ee272f6..c91e9cbf2 100644 --- a/docs/src/api/api.md +++ b/docs/src/api/api.md @@ -52,10 +52,11 @@ random_field ## Processors ```@docs -step_logger +timelogger vtk_writer -field_saver -field_plotter +fieldsaver +realtimeplotter +fieldplot energy_history_plotter energy_spectrum_plotter animator diff --git a/examples/Actuator2D.jl b/examples/Actuator2D.jl index 5357e411a..35e3b4307 100644 --- a/examples/Actuator2D.jl +++ b/examples/Actuator2D.jl @@ -74,13 +74,17 @@ u, p, outputs = solve_unsteady( method = RK44P2(), Δt = 0.05, processors = ( - field_plotter(setup; nupdate = 1), - ## energy_history_plotter(setup; nupdate = 10), - ## energy_spectrum_plotter(setup; nupdate = 10), - ## animator(setup, "vorticity.mkv"; nupdate = 4), - ## vtk_writer(setup; nupdate = 10, dir = "output/$name", filename = "solution"), - ## field_saver(setup; nupdate = 10), - step_logger(; nupdate = 1), + rtp = realtimeplotter(; + setup, + plot = fieldplot, + ## plot = energy_history_plot, + ## plot = energy_spectrum_plot, + nupdate = 1, + ), + ## anim = animator(; setup, path = "vorticity.mkv", nupdate = 20), + ## vtk = vtk_writer(; setup, nupdate = 10, dir = "output/$name", filename = "solution"), + ## field = fieldsaver(; setup, nupdate = 10), + log = timelogger(; nupdate = 1), ), ); diff --git a/examples/Actuator3D.jl b/examples/Actuator3D.jl index 3da5aa49a..7525d6218 100644 --- a/examples/Actuator3D.jl +++ b/examples/Actuator3D.jl @@ -90,13 +90,17 @@ u, p, outputs = solve_unsteady( method = RK44P2(), Δt = T(0.05), processors = ( - field_plotter(setup; nupdate = 5), - ## energy_history_plotter(setup; nupdate = 10), - ## energy_spectrum_plotter(setup; nupdate = 10), - ## animator(setup, "vorticity.mkv"; nupdate = 4), - ## vtk_writer(setup; nupdate = 2, dir = "output/$name", filename = "solution"), - ## field_saver(setup; nupdate = 10), - step_logger(; nupdate = 1), + rtp = realtimeplotter(; + setup, + plot = fieldplot, + ## plot = energy_history_plot, + ## plot = energy_spectrum_plot, + nupdate = 1, + ), + ## anim = animator(; setup, path = "vorticity.mkv", nupdate = 20), + ## vtk = vtk_writer(; setup, nupdate = 10, dir = "output/$name", filename = "solution"), + ## field = fieldsaver(; setup, nupdate = 10), + log = timelogger(; nupdate = 1), ), ); diff --git a/examples/BackwardFacingStep2D.jl b/examples/BackwardFacingStep2D.jl index b95d55d9d..032601bc5 100644 --- a/examples/BackwardFacingStep2D.jl +++ b/examples/BackwardFacingStep2D.jl @@ -70,19 +70,23 @@ u, p = copy.(u₀), copy(p₀) # Solve unsteady problem u, p, outputs = solve_unsteady( setup, - copy.(u₀), copy(p₀), + u₀, p₀, # u, p, (T(0), T(7)); Δt = T(0.002), pressure_solver, processors = ( - field_plotter(setup; nupdate = 1), - ## energy_history_plotter(setup; nupdate = 10), - ## energy_spectrum_plotter(setup; nupdate = 10), - ## animator(setup, "vorticity.mkv"; nupdate = 4), - ## vtk_writer(setup; nupdate = 20, dir = "output/$name", filename = "solution"), - ## field_saver(setup; nupdate = 10), - step_logger(; nupdate = 1), + rtp = realtimeplotter(; + setup, + plot = fieldplot, + ## plot = energy_history_plot, + ## plot = energy_spectrum_plot, + nupdate = 1, + ), + ## anim = animator(; setup, path = "vorticity.mkv", nupdate = 20), + ## vtk = vtk_writer(; setup, nupdate = 10, dir = "output/$name", filename = "solution"), + ## field = fieldsaver(; setup, nupdate = 10), + log = timelogger(; nupdate = 1), ), ); diff --git a/examples/BackwardFacingStep3D.jl b/examples/BackwardFacingStep3D.jl index b0abb1066..136ec9e8a 100644 --- a/examples/BackwardFacingStep3D.jl +++ b/examples/BackwardFacingStep3D.jl @@ -76,13 +76,17 @@ u, p, outputs = solve_unsteady( (T(0), T(7)); Δt = T(0.01), processors = ( - field_plotter(setup; nupdate = 50), - ## energy_history_plotter(setup; nupdate = 10), - ## energy_spectrum_plotter(setup; nupdate = 10), - ## animator(setup, "vorticity.mkv"; nupdate = 4), - ## vtk_writer(setup; nupdate = 20, dir = "output/$name", filename = "solution"), - ## field_saver(setup; nupdate = 10), - step_logger(; nupdate = 10), + rtp = realtimeplotter(; + setup, + plot = fieldplot, + ## plot = energy_history_plot, + ## plot = energy_spectrum_plot, + nupdate = 1, + ), + ## anim = animator(; setup, path = "vorticity.mkv", nupdate = 20), + ## vtk = vtk_writer(; setup, nupdate = 10, dir = "output/$name", filename = "solution"), + ## field = fieldsaver(; setup, nupdate = 10), + log = timelogger(; nupdate = 1), ), ) #md current_figure() diff --git a/examples/DecayingTurbulence2D.jl b/examples/DecayingTurbulence2D.jl index 55a6a9ea6..346a9fbdb 100644 --- a/examples/DecayingTurbulence2D.jl +++ b/examples/DecayingTurbulence2D.jl @@ -62,15 +62,18 @@ u, p, outputs = solve_unsteady( (T(0), T(1)); Δt = T(1e-3), pressure_solver, - inplace = true, processors = ( - field_plotter(setup; nupdate = 20), - energy_history_plotter(setup; nupdate = 20, displayfig = false), - energy_spectrum_plotter(setup; nupdate = 20, displayfig = false), - ## animator(setup, "vorticity.mp4"; nupdate = 16), - ## vtk_writer(setup; nupdate = 10, dir = "output/$name", filename = "solution"), - ## field_saver(setup; nupdate = 10), - step_logger(; nupdate = 100), + rtp = realtimeplotter(; + setup, + plot = fieldplot, + ## plot = energy_history_plot, + ## plot = energy_spectrum_plot, + nupdate = 1, + ), + ## anim = animator(; setup, path = "vorticity.mkv", nupdate = 20), + ## vtk = vtk_writer(; setup, nupdate = 10, dir = "output/$name", filename = "solution"), + ## field = fieldsaver(; setup, nupdate = 10), + log = timelogger(; nupdate = 1), ), ); @@ -78,14 +81,8 @@ u, p, outputs = solve_unsteady( # # We may visualize or export the computed fields `(u, p)` -# Field plot -outputs[1] - -# Energy history plot -outputs[2] - -# Energy spectrum plot -outputs[3] +# Plot +outputs.rtp # Export to VTK save_vtk(setup, u, p, "output/solution") diff --git a/examples/DecayingTurbulence3D.jl b/examples/DecayingTurbulence3D.jl index b0bb3af10..e3598ace8 100644 --- a/examples/DecayingTurbulence3D.jl +++ b/examples/DecayingTurbulence3D.jl @@ -66,13 +66,17 @@ u, p, outputs = solve_unsteady( Δt = T(0.001), pressure_solver, processors = ( - field_plotter(setup; nupdate = 10), - energy_history_plotter(setup; nupdate = 10), - energy_spectrum_plotter(setup; nupdate = 10), - ## animator(setup, "vorticity.mp4"; nupdate = 4), - ## vtk_writer(setup; nupdate = 10, dir = "output/$name", filename = "solution"), - ## field_saver(setup; nupdate = 10), - step_logger(; nupdate = 1), + rtp = realtimeplotter(; + setup, + plot = fieldplot, + ## plot = energy_history_plot, + ## plot = energy_spectrum_plot, + nupdate = 1, + ), + ## anim = animator(; setup, path = "vorticity.mkv", nupdate = 20), + ## vtk = vtk_writer(; setup, nupdate = 10, dir = "output/$name", filename = "solution"), + ## field = fieldsaver(; setup, nupdate = 10), + log = timelogger(; nupdate = 1), ), ); diff --git a/examples/LidDrivenCavity2D.jl b/examples/LidDrivenCavity2D.jl index c38137420..0d35f13f8 100644 --- a/examples/LidDrivenCavity2D.jl +++ b/examples/LidDrivenCavity2D.jl @@ -115,13 +115,17 @@ u, p = u₀, p₀ # later returned by `solve_unsteady`. processors = ( - field_plotter(setup; nupdate = 50), - ## energy_history_plotter(setup; nupdate = 1), - ## energy_spectrum_plotter(setup; nupdate = 100), - ## animator(setup, "vorticity.mkv"; nupdate = 4), - ## vtk_writer(setup; nupdate = 100, dir = "output/$name", filename = "solution"), - ## field_saver(setup; nupdate = 10), - step_logger(; nupdate = 1000), + rtp = realtimeplotter(; + setup, + plot = fieldplot, + ## plot = energy_history_plot, + ## plot = energy_spectrum_plot, + nupdate = 50, + ), + ## anim = animator(; setup, path = "vorticity.mkv", nupdate = 20), + ## vtk = vtk_writer(; setup, nupdate = 10, dir = "output/$name", filename = "solution"), + ## field = fieldsaver(; setup, nupdate = 10), + log = timelogger(; nupdate = 1000), ); # By default, a standard fourth order Runge-Kutta method is used. If we don't diff --git a/examples/LidDrivenCavity3D.jl b/examples/LidDrivenCavity3D.jl index d4c8a6f74..a150f4247 100644 --- a/examples/LidDrivenCavity3D.jl +++ b/examples/LidDrivenCavity3D.jl @@ -76,13 +76,17 @@ u, p, outputs = solve_unsteady( (T(0), T(0.2)); Δt = T(0.001), processors = ( - field_plotter(setup; nupdate = 1), - ## energy_history_plotter(setup; nupdate = 1), - ## energy_spectrum_plotter(setup; nupdate = 100), - ## animator(setup, "vorticity.mkv"; nupdate = 4), - ## vtk_writer(setup; nupdate = 5, dir = "output/$name", filename = "solution"), - ## field_saver(setup; nupdate = 10), - step_logger(; nupdate = 1), + rtp = realtimeplotter(; + setup, + plot = fieldplot, + ## plot = energy_history_plot, + ## plot = energy_spectrum_plot, + nupdate = 1, + ), + ## anim = animator(; setup, path = "vorticity.mkv", nupdate = 20), + ## vtk = vtk_writer(; setup, nupdate = 10, dir = "output/$name", filename = "solution"), + ## field = fieldsaver(; setup, nupdate = 10), + log = timelogger(; nupdate = 1), ), ); diff --git a/examples/PlanarMixing2D.jl b/examples/PlanarMixing2D.jl index 9f25b5302..d9e613e60 100644 --- a/examples/PlanarMixing2D.jl +++ b/examples/PlanarMixing2D.jl @@ -67,13 +67,17 @@ u, p, outputs = solve_unsteady( method = RK44P2(), Δt = 0.1, processors = ( - field_plotter(setup; nupdate = 1), - ## energy_history_plotter(setup; nupdate = 1), - ## energy_spectrum_plotter(setup; nupdate = 100), - ## animator(setup, "vorticity.mkv"; nupdate = 4), - ## vtk_writer(setup; nupdate = 10, dir = "output/$name", filename = "solution"), - ## field_saver(setup; nupdate = 10), - step_logger(; nupdate = 1), + rtp = realtimeplotter(; + setup, + plot = fieldplot, + ## plot = energy_history_plot, + ## plot = energy_spectrum_plot, + nupdate = 1, + ), + ## anim = animator(; setup, path = "vorticity.mkv", nupdate = 20), + ## vtk = vtk_writer(; setup, nupdate = 10, dir = "output/$name", filename = "solution"), + ## field = fieldsaver(; setup, nupdate = 10), + log = timelogger(; nupdate = 1), ), ); diff --git a/examples/PlaneJets2D.jl b/examples/PlaneJets2D.jl index 9cd8937bf..a3f512721 100644 --- a/examples/PlaneJets2D.jl +++ b/examples/PlaneJets2D.jl @@ -105,66 +105,62 @@ u₀, p₀ = create_initial_conditions( ); # Real time plot: Streamwise average and spectrum -mean_plotter(setup; nupdate = 1) = processor( - function (state) - (; Ip) = setup.grid - - umean = @lift begin - (; u, p, t) = $state - up = IncompressibleNavierStokes.interpolate_u_p(u, setup) - u1 = u[1] - reshape(sum(u1[Ip]; dims = 1), :) ./ size(u1, 1) ./ V() - end - - K = size(Ip, 1) ÷ 2 - k = 1:(K-1) - - # Find energy spectrum where y = 0 - n₀ = size(Ip, 2) ÷ 2 - E₀ = @lift begin - (; u, p, t) = $state - u_y = u[1][:, n₀] - abs.(fft(u_y .^ 2))[k.+1] - end - - # Find energy spectrum where y = 1 - n₁ = argmin(n -> abs(yin[n] .- 1), 1:Nuy_in) - E₁ = @lift begin - (; V, p, t) = $state - u = V[indu] - u_y = reshape(u, size(yu))[:, n₁] - abs.(fft(u_y .^ 2))[k.+1] - end - - fig = Figure() - ax = Axis( - fig[1, 1]; - title = "Mean streamwise flow", - xlabel = "y", - ylabel = L"\langle u \rangle / U_0", - ) - lines!(ax, yu[1, :], umean) - ax = Axis( - fig[1, 2]; - title = "Streamwise energy spectrum", - xscale = log10, - yscale = log10, - xlabel = L"k_x", - ylabel = L"\hat{U}_{cl} / U_0", - ) - # ylims!(ax, (10^(0.0), 10^4.0)) - ksub = k[10:end] - lines!(ax, ksub, 1000 .* ksub .^ (-3 / 5); label = L"k^{-3/5}") - lines!(ax, ksub, 1e7 .* ksub .^ -3; label = L"k^{-3}") - scatter!(ax, k, E₀; label = "y = $(yin[n₀])") - scatter!(ax, k, E₁; label = "y = $(yin[n₁])") - axislegend(ax; position = :lb) - - display(fig) - fig - end; - nupdate, -) +function meanplot(; setup, state) + (; Ip) = setup.grid + + umean = @lift begin + (; u, p, t) = $state + up = IncompressibleNavierStokes.interpolate_u_p(u, setup) + u1 = u[1] + reshape(sum(u1[Ip]; dims = 1), :) ./ size(u1, 1) ./ V() + end + + K = size(Ip, 1) ÷ 2 + k = 1:(K-1) + + # Find energy spectrum where y = 0 + n₀ = size(Ip, 2) ÷ 2 + E₀ = @lift begin + (; u, p, t) = $state + u_y = u[1][:, n₀] + abs.(fft(u_y .^ 2))[k.+1] + end + + # Find energy spectrum where y = 1 + n₁ = argmin(n -> abs(yin[n] .- 1), 1:Nuy_in) + E₁ = @lift begin + (; V, p, t) = $state + u = V[indu] + u_y = reshape(u, size(yu))[:, n₁] + abs.(fft(u_y .^ 2))[k.+1] + end + + fig = Figure() + ax = Axis( + fig[1, 1]; + title = "Mean streamwise flow", + xlabel = "y", + ylabel = L"\langle u \rangle / U_0", + ) + lines!(ax, yu[1, :], umean) + ax = Axis( + fig[1, 2]; + title = "Streamwise energy spectrum", + xscale = log10, + yscale = log10, + xlabel = L"k_x", + ylabel = L"\hat{U}_{cl} / U_0", + ) + # ylims!(ax, (10^(0.0), 10^4.0)) + ksub = k[10:end] + lines!(ax, ksub, 1000 .* ksub .^ (-3 / 5); label = L"k^{-3/5}") + lines!(ax, ksub, 1e7 .* ksub .^ -3; label = L"k^{-3}") + scatter!(ax, k, E₀; label = "y = $(yin[n₀])") + scatter!(ax, k, E₁; label = "y = $(yin[n₁])") + axislegend(ax; position = :lb) + + fig +end # Solve unsteady problem toto, p, outputs = solve_unsteady( @@ -176,26 +172,26 @@ toto, p, outputs = solve_unsteady( Δt = 0.001, pressure_solver, processors = ( - # field_plotter(setup; nupdate = 1), - ## energy_history_plotter(setup; nupdate = 1), - ## energy_spectrum_plotter(setup; nupdate = 100), - ## animator(setup, "vorticity.mkv"; nupdate = 4), - ## vtk_writer(setup; nupdate = 10, dir = "output/$name", filename = "solution"), - ## field_saver(setup; nupdate = 10), - mean_plotter(setup), - step_logger(; nupdate = 1), + rtp = realtimeplotter(; + setup, + ## plot = fieldplot, + ## plot = energy_history_plot, + ## plot = energy_spectrum_plot, + plot = meanplot, + nupdate = 1, + ), + ## anim = animator(; setup, path = "vorticity.mkv", nupdate = 4), + ## vtk = vtk_writer(; setup, nupdate = 10, dir = "output/$name", filename = "solution"), + ## field = fieldsaver(; setup, nupdate = 10), + log = timelogger(; nupdate = 1), ), ); # ## Post-process # -# We may visualize or export the computed fields `(V, p)` - -outputs[1] - -#- +# We may visualize or export the computed fields `(u, p)` -outputs[2] +outputs.rtp # Export to VTK save_vtk(setup, toto, p, "output/solution") diff --git a/examples/ShearLayer2D.jl b/examples/ShearLayer2D.jl index 41b869b40..5f4079fd5 100644 --- a/examples/ShearLayer2D.jl +++ b/examples/ShearLayer2D.jl @@ -59,8 +59,6 @@ u₀, p₀ = create_initial_conditions( pressure_solver, ); -# Iteration processors - # Solve unsteady problem u, p, outputs = solve_unsteady( setup, @@ -70,13 +68,17 @@ u, p, outputs = solve_unsteady( Δt = T(0.01), pressure_solver, processors = ( - field_plotter(setup; nupdate = 1), - ## energy_history_plotter(setup; nupdate = 1), - ## energy_spectrum_plotter(setup; nupdate = 100), - ## animator(setup, "vorticity.mkv"; nupdate = 4), - ## vtk_writer(setup; nupdate = 10, dir = "output/$name", filename = "solution"), - ## field_saver(setup; nupdate = 10), - step_logger(; nupdate = 1), + rtp = realtimeplotter(; + setup, + plot = fieldplot, + ## plot = energy_history_plot, + ## plot = energy_spectrum_plot, + nupdate = 1, + ), + ## anim = animator(; setup, path = "vorticity.mkv", nupdate = 20), + ## vtk = vtk_writer(; setup, nupdate = 10, dir = "output/$name", filename = "solution"), + ## field = fieldsaver(; setup, nupdate = 10), + log = timelogger(; nupdate = 1), ), ); @@ -84,7 +86,7 @@ u, p, outputs = solve_unsteady( # # We may visualize or export the computed fields `(u, p)` -outputs[1] +outputs.rtp # Export to VTK save_vtk(setup, u, p, "output/solution") diff --git a/examples/TaylorGreenVortex3D.jl b/examples/TaylorGreenVortex3D.jl index eb6fec39c..a4b0ce569 100644 --- a/examples/TaylorGreenVortex3D.jl +++ b/examples/TaylorGreenVortex3D.jl @@ -75,24 +75,17 @@ u, p, outputs = solve_unsteady( (T(0), T(5)); Δt = T(0.01), processors = ( - ## field_plotter(setup; nupdate = 1), - energy_history_plotter(setup; nupdate = 1), - ## energy_spectrum_plotter(setup; nupdate = 100), - ## animator( - ## setup, - ## "vorticity3D.mp4"; - ## plotter = field_plotter( - ## setup; - ## fieldname = :Dfield, - ## levels = LinRange(-T(3), T(1), 5), - ## docolorbar = false, - ## resolution = (1024, 1024), - ## ), - ## nupdate = 20, - ## ), - ## vtk_writer(setup; nupdate = 10, dir = "output/$name", filename = "solution"), - ## field_saver(setup; nupdate = 10), - step_logger(; nupdate = 1), + rtp = realtimeplotter(; + setup, + plot = fieldplot, + ## plot = energy_history_plot, + ## plot = energy_spectrum_plot, + nupdate = 1, + ), + ## anim = animator(; setup, path = "vorticity.mkv", nupdate = 20), + ## vtk = vtk_writer(; setup, nupdate = 10, dir = "output/$name", filename = "solution"), + ## field = fieldsaver(; setup, nupdate = 10), + log = timelogger(; nupdate = 1), ), pressure_solver, ); diff --git a/scratch/filter_plot.jl b/scratch/filter_plot.jl index 1107b1b20..3f66bc4b0 100644 --- a/scratch/filter_plot.jl +++ b/scratch/filter_plot.jl @@ -133,7 +133,7 @@ u, p, outputs = solve_unsteady( pressure_solver, inplace = true, processors = ( - # field_plotter(setup; nupdate = 10, docolorbar = false, displayupdates = false), + # realtimeplotter(; setup, nupdate = 10, docolorbar = false, displayupdates = false), # animator( # setup, # "filtered.mp4"; @@ -147,7 +147,7 @@ u, p, outputs = solve_unsteady( # energy_history_plotter(setup; nupdate = 20, displayfig = false), # energy_spectrum_plotter(setup; nupdate = 10, displayfig = false), ## vtk_writer(setup; nupdate = 10, dir = "output/$name", filename = "solution"), - ## field_saver(setup; nupdate = 10), - step_logger(; nupdate = 10), + ## fieldsaver(setup; nupdate = 10), + timelogger(; nupdate = 10), ), ); diff --git a/scratch/filtered.jl b/scratch/filtered.jl deleted file mode 100644 index ae6965e00..000000000 --- a/scratch/filtered.jl +++ /dev/null @@ -1,180 +0,0 @@ -# Little LSP hack to get function signatures, go #src -# to definition etc. #src -if isdefined(@__MODULE__, :LanguageServer) #src - include("../src/IncompressibleNavierStokes.jl") #src - using .IncompressibleNavierStokes #src -end #src - -using GLMakie -using IncompressibleNavierStokes -using LinearAlgebra -using Lux - -# Floating point precision -T = Float32 - -# To use CPU: Do not move any arrays -device = identity - -# To use GPU, use `cu` to move arrays to the GPU. -# Note: `cu` converts to Float32 -using CUDA -using LuxCUDA -device = cu - -# Reynolds number -Re = T(2_000) - -# A 2D grid is a Cartesian product of two vectors -s = 2 -n_coarse = 256 -n = s * n_coarse - -lims = T(0), T(1) -x = LinRange(lims..., n + 1) -y = LinRange(lims..., n + 1) -plot_grid(x, y) - -x_coarse = x[1:s:end] -y_coarse = y[1:s:end] -plot_grid(x_coarse, y_coarse) - -# Build setup and assemble operators -setup = Setup(x, y; Re); -devsetup = device(setup); -setup_coarse = Setup(x_coarse, y_coarse; Re); - -# Filter -(; KV, Kp) = operator_filter(setup.grid, setup.boundary_conditions, s); - -# Since the grid is uniform and identical for x and y, we may use a specialized -# spectral pressure solver -pressure_solver = SpectralPressureSolver(setup); -pressure_solver_coarse = SpectralPressureSolver(setup_coarse); - -# Initial conditions -V₀, p₀ = random_field(setup; A = T(10_000_000), σ = T(30), s = 5, pressure_solver); - -filter_saver(setup, KV, Kp; nupdate = 1, bc_vectors = get_bc_vectors(setup, T(0))) = - processor( - function (step_observer) - T = eltype(setup.grid.x) - _V = fill(zeros(T, 0), 0) - _F = fill(zeros(T, 0), 0) - _FG = fill(zeros(T, 0), 0) - _p = fill(zeros(T, 0), 0) - _t = fill(zero(T), 0) - @lift begin - (; V, p, t) = $step_observer - F, = momentum(V, V, p, t, setup; bc_vectors, nopressure = true) - FG, = momentum(V, V, p, t, setup; bc_vectors, nopressure = false) - push!(_V, KV * Array(V)) - push!(_F, KV * Array(F)) - push!(_FG, KV * Array(FG)) - push!(_p, Kp * Array(p)) - push!(_t, t) - end - (; V = _V, F = _F, FG = _FG, p = _p, t = _t) - end; - nupdate, - ) - -# Iteration processors -processors = ( - filter_saver(devsetup, KV, Kp; bc_vectors = device(get_bc_vectors(setup, T(0)))), - field_plotter(devsetup; type = image, nupdate = 1), - # energy_history_plotter(devsetup; nupdate = 20, displayfig = false), - # energy_spectrum_plotter(devsetup; nupdate = 20, displayfig = false), - # animator(devsetup; nupdate = 16), - ## vtk_writer(setup; nupdate = 10, dir = "output/$name", filename = "solution"), - ## field_saver(setup; nupdate = 10), - step_logger(; nupdate = 10), -); - -processors_coarse = ( - field_plotter(device(setup_coarse); type = image, nupdate = 1), - step_logger(; nupdate = 10), -); - -# Time interval -t_start, t_end = tlims = T(0), T(0.1) - -# Solve unsteady problem -V, p, outputs = solve_unsteady( - setup, - V₀, - p₀, - # V, p, - tlims; - Δt = T(2e-4), - processors, - pressure_solver, - inplace = true, - device, -); - -# V₀, p₀ = V, p -# V, p = V₀, p₀ - -Vbar = KV * V -pbar = Kp * p - -Vbar_nomodel, pbar_nomodel, outputs_lam = solve_unsteady( - setup_coarse, - KV * V₀, - Kp * p₀, - tlims; - Δt = T(2e-4), - processors = processors_coarse, - pressure_solver = pressure_solver_coarse, - inplace = true, - device, -); - -Vbar_smag, pbar_smag, outputs_smag = solve_unsteady( - (; setup_coarse..., viscosity_model = SmagorinskyModel(; C_s = T(0.173))), - KV * V₀, - Kp * p₀, - tlims; - Δt = T(2e-4), - processors = processors_coarse, - pressure_solver = pressure_solver_coarse, - inplace = true, - device, -); - -norm(Vbar_nomodel - Vbar) / norm(Vbar) -norm(Vbar_smag - Vbar) / norm(Vbar) - -# Filtered quantities -filtered = outputs[1] -filtered.V[end] - -closure, θ = - cnn(setup_coarse, [5, 5, 5], [2, 5, 5, 2], [tanh, tanh, identity], [true, true, false];) - -@time closure(Vbar, θ); - -cuVbar = cu(Vbar) -cuθ = cu(θ) -@time closure(cuVbar, cuθ); - -size(Vbar) -size(closure(Vbar, θ)) - -using Zygote - -@time gradient(θ -> sum(closure(Vbar, θ)), θ); -@time gradient(θ -> sum(closure(cuVbar, θ)), cuθ); - -Vbar_cnn, pbar_cnn, outputs_cnn = solve_unsteady( - (; setup_coarse..., closure_model = V -> closure(V, 1.0f-3 * cuθ)), - KV * V₀, - Kp * p₀, - tlims; - Δt = T(2e-4), - processors = processors_coarse, - pressure_solver = pressure_solver_coarse, - inplace = true, - device, -); diff --git a/scratch/multigrid.jl b/scratch/multigrid.jl index 48ffff9b4..098921fff 100644 --- a/scratch/multigrid.jl +++ b/scratch/multigrid.jl @@ -77,7 +77,7 @@ end # # Load previous LES data # data_train, data_valid, data_test = load("output/forced/data.jld2", "data_train", "data_valid", "data_test") -relerr_track(uref, setup) = processor(function (state) +relerr_track(uref, setup) = processor() do state (; dimension, x, Ip) = setup.grid D = dimension() T = eltype(x[1]) @@ -92,32 +92,24 @@ relerr_track(uref, setup) = processor(function (state) e[] += sqrt(a) / sqrt(b) / (length(uref) - 1) end e -end) +end e_nm = zeros(T, length(ntest)) for (i, n) in enumerate(ntest) params = get_params(n) - # Build LES setup and assemble operators x = ntuple(α -> LinRange(params.lims..., params.nles + 1), params.D) setup = Setup(x...; params.Re, ArrayType) - # Uniform periodic grid pressure_solver = SpectralPressureSolver(setup) u = device.(data_test[i].u[1]) u₀ = device(data_test[i].u[1][1]) p₀ = pressure_additional_solve(pressure_solver, u₀, T(0), setup) - u_nm, p_nm, outputs = solve_unsteady( - setup, - copy.(u₀), - copy(p₀), - (T(0), params.tsim); - Δt = data_test[i].Δt, - pressure_solver, - processors = ( - relerr_track(u, setup), - # step_logger(; nupdate = 1), - ), - ) - e_nm[i] = outputs[1][] + tlims = (T(0), params.tsim) + (; Δt) = data_test[i] + processors = (; relerr = relerr_track(u, setup)) + _, _, o = solve_unsteady(setup, u₀, p₀, tlims; Δt, pressure_solver, processors) + e_nm[i] = o.relerr[] + _, _, o = solve_unsteady(setup, u₀, p₀, tlims; Δt, pressure_solver, processors) + e_cnn[i] = o.relerr[] end e_nm e_cnn = ones(T, length(ntest)) @@ -155,18 +147,10 @@ save("convergence.pdf", current_figure()) closure, θ₀ = cnn( setup, - - # Radius - [2, 2, 2, 2], - - # Channels - [5, 5, 5, params.D], - - # Activations - [leakyrelu, leakyrelu, leakyrelu, leakyrelu, identity], - - # Bias - [true, true, true, true, false]; + radii = [2, 2, 2, 2], + channels = [5, 5, 5, params.D], + activations = [leakyrelu, leakyrelu, leakyrelu, identity], + use_bias = [true, true, true, false], ); closure.NN diff --git a/scratch/train_model.jl b/scratch/train_model.jl index 3542229a2..837e0825c 100644 --- a/scratch/train_model.jl +++ b/scratch/train_model.jl @@ -92,6 +92,7 @@ closure, θ₀ = cnn(; bias = [true, true, true, false], ); closure.NN + sample = io_train[1][:, :, :, 1:5] closure(sample, θ₀) |> size @@ -175,7 +176,7 @@ function relerr(u, uref, setup) sqrt(a) / sqrt(b) end -relerr_track(uref, setup) = processor(function (state) +relerr_track(uref, setup) = processor() do state (; dimension, x, Ip) = setup.grid D = dimension() T = eltype(x[1]) @@ -190,7 +191,7 @@ relerr_track(uref, setup) = processor(function (state) e[] += sqrt(a) / sqrt(b) / (length(uref) - 1) end e -end) +end u, u₀, p₀ = nothing, nothing, nothing u = device.(data_test.u[1]) @@ -200,43 +201,31 @@ length(u) u_nm, p_nm, outputs = solve_unsteady( setup, - copy.(u₀), - copy(p₀), + u₀, + p₀, (T(0), params.tsim); Δt = data_test.Δt, pressure_solver, processors = ( - relerr_track(u, setup), - # field_plotter(setup; nupdate = 10), - # field_plotter( - # setup; - # nupdate = 10, - # fieldname = :λ₂field, - # levels = LinRange(-T(2), T(3), 5), - # # levels = LinRange(-1.0f0, 3.0f0, 5), - # # levels = LinRange(-2.0f0, 2.0f0, 5), - # # levels=5, - # docolorbar = false, - # ), - step_logger(; nupdate = 1), + relerr = relerr_track(u, setup), + log = timelogger(; nupdate = 1), ), ) -relerr_nm = outputs[1][] +relerr_nm = outputs.relerr[] u_cnn, p_cnn, outputs = solve_unsteady( (; setup..., closure_model = create_neural_closure(closure, θ, setup)), - copy.(u₀), - copy(p₀), + u₀, + p₀, (T(0), params.tsim); Δt = data_test.Δt, pressure_solver, processors = ( - relerr_track(u, setup), - # field_plotter(setup; nupdate = 10), - step_logger(; nupdate = 1), + relerr = relerr_track(u, setup), + log = timelogger(; nupdate = 1), ), ) -relerr_cnn = outputs[1][] +relerr_cnn = outputs.relerr[] relerr_nm relerr_cnn diff --git a/src/IncompressibleNavierStokes.jl b/src/IncompressibleNavierStokes.jl index 8464baa73..1b8e5486c 100644 --- a/src/IncompressibleNavierStokes.jl +++ b/src/IncompressibleNavierStokes.jl @@ -113,8 +113,8 @@ export AbstractViscosityModel, LaminarModel, MixingLengthModel, SmagorinskyModel export NoRegConvectionModel, C2ConvectionModel, C4ConvectionModel, LerayConvectionModel # Processors -export processor, step_logger, vtk_writer, field_saver -export field_plotter, energy_history_plotter, energy_spectrum_plotter +export processor, timelogger, vtk_writer, fieldsaver, realtimeplotter +export fieldplot, energy_history_plot, energy_spectrum_plot export animator # Setup diff --git a/src/closures/create_les_data.jl b/src/closures/create_les_data.jl index 056f9cbb9..4efc9f7f0 100644 --- a/src/closures/create_les_data.jl +++ b/src/closures/create_les_data.jl @@ -32,8 +32,8 @@ function gaussian_force( force end -_filter_saver(dns, les, comp, pressure_solver; nupdate = 1) = processor( - function (state) +_filter_saver(dns, les, comp, pressure_solver; nupdate = 1) = + processor() do state (; dimension, x) = dns.grid T = eltype(x[1]) D = dimension() @@ -76,9 +76,7 @@ _filter_saver(dns, les, comp, pressure_solver; nupdate = 1) = processor( end state[] = state[] # Save initial conditions (; t = _t, u = _u, c = _c) - end; - nupdate, -) + end """ create_les_data( @@ -167,7 +165,7 @@ function create_les_data( p₀, (T(0), tburn); Δt, - # processors = (step_logger(; nupdate = 10),), + # processors = (timelogger(; nupdate = 10),), pressure_solver, ) diff --git a/src/operators.jl b/src/operators.jl index d8fb16661..47d17a852 100644 --- a/src/operators.jl +++ b/src/operators.jl @@ -462,6 +462,7 @@ function Dfield!(d, G, p, setup; ϵ = eps(eltype(p))) d end + """ Dfield(p, setup) diff --git a/src/processors/animator.jl b/src/processors/animator.jl index 5d0970709..79e147c1e 100644 --- a/src/processors/animator.jl +++ b/src/processors/animator.jl @@ -1,5 +1,5 @@ """ - animator(setup, path; nupdate = 1, plotter = field_plotter(setup); kwargs...) + animator(; setup, path, plot = fieldplot, nupdate = 1, kwargs...) Animate a plot of the solution every `update` iteration. The animation is saved to `path`, which should have one @@ -13,18 +13,16 @@ of the following extensions: The plot is determined by a `plotter` processor. Additional `kwargs` are passed to Makie's `VideoStream`. """ -animator(setup, path; nupdate = 1, plotter = field_plotter(setup), kwargs...) = processor( - function (state) +animator(; setup, path, plot = fieldplot, nupdate = 1, kwargs...) = + processor((stream, state) -> save(path, stream)) do outerstate ispath(dirname(path)) || mkpath(dirname(path)) - _state = Observable(state[]) - fig = plotter.initialize(_state) + state = Observable(outerstate[]) + fig = plot(; setup, state) stream = VideoStream(fig; kwargs...) - @lift begin - _state[] = $state + on(outerstate) do outerstate + outerstate.n % nupdate == 0 || return + state[] = outerstate recordframe!(stream) end stream - end; - finalize = (stream, step_observer) -> save(path, stream), - nupdate, -) + end diff --git a/src/processors/processors.jl b/src/processors/processors.jl index 957d1c399..73e082017 100644 --- a/src/processors/processors.jl +++ b/src/processors/processors.jl @@ -1,139 +1,125 @@ """ - processor( - initialize; - finalize = (initialized, stepper) -> initialized, - nupdate = 1 - ) + processor(initialize, finalize = (initialized, state) -> initialized) -Process results from time stepping. Before time stepping, the `initialize` function is called on an observable of the time `stepper`, returning `initialized`. The observable is updated after every `nupdate` time step, triggering updates where `@lift` is used inside `initialize`. +Process results from time stepping. Before time stepping, the `initialize` +function is called on an observable of the time stepper `state`, returning +`initialized`. The observable is updated every time step. -After timestepping, the `finalize` function is called on `initialized` and the final stepper. +After timestepping, the `finalize` function is called on `initialized` and the +final `state`. See the following example: ```example -function initialize(step_observer) +function initialize(state) s = 0 println("Let's sum up the time steps") - @lift begin - (; n) = \$step_observer - println("The summand is \$n") + on(state) do (; n, t) + println("The summand is \$n, the time is \$t") s = s + n end s end -finalize(s, stepper) = println("The final sum (at time t=\$(stepper.t)) is \$s") -p = Processor(initialize; finalize, nupdate = 5) +finalize(i, state) = println("The final sum (at time t=\$(state.t)) is \$s") +p = processor(initialize, finalize) ``` -When solved for 20 time steps from t=0 to t=2 the displayed output is +When solved for 6 time steps from t=0 to t=2 the displayed output is ``` Let's sum up the time steps -The summand is 0 -The summand is 5 -The summand is 10 -The summand is 15 -The summand is 20 -The final sum (at time t=2.0) is 50 +The summand is 0, the time is 0.0 +The summand is 1, the time is 0.4 +The summand is 2, the time is 0.8 +The summand is 3, the time is 1.2 +The summand is 4, the time is 1.6 +The summand is 5, the time is 2.0 +The final sum (at time t=2.0) is 15 ``` """ -processor(initialize; finalize = (initialized, stepper) -> initialized, nupdate = 1) = - (; initialize, finalize, nupdate) +processor(initialize, finalize = (initialized, state) -> initialized) = + (; initialize, finalize) """ - step_logger(; nupdate = 1) + timelogger(; nupdate = 1) Create processor that logs time step information. """ -step_logger(; nupdate = 1) = processor(function (step_observer) - @lift begin - (; t, n) = $step_observer +step_logger(; nupdate = 1) = processor(function (state) + on(state) do (; t, n) + n % nupdate == 0 || return @printf "Iteration %d\tt = %g\n" n t end nothing -end; nupdate) +end) """ - vtk_writer( - setup; + vtk_writer(; + setup, nupdate = 1, dir = "output", filename = "solution", fields = (:velocity, :pressure, :vorticity), ) -Create processor that writes the solution every `nupdate` time steps to a VTK file. The resulting Paraview data -collection file is stored in `"\$dir/\$filename.pvd"`. +Create processor that writes the solution every `nupdate` time steps to a VTK +file. The resulting Paraview data collection file is stored in +`"\$dir/\$filename.pvd"`. """ -vtk_writer( - setup; +vtk_writer(; + setup, nupdate = 1, dir = "output", filename = "solution", fields = (:velocity, :pressure, :vorticity), -) = processor( - function (step_observer) +) = + processor((pvd, state) -> vtk_save(pvd)) do state + (; grid) = setup + (; dimension, xp) = grid + D = dimension() ispath(dir) || mkpath(dir) pvd = paraview_collection(joinpath(dir, filename)) - @lift begin - (; grid) = setup - (; dimension, xp, yp) = grid - (; u, p, t) = $step_observer - - N = dimension() - if N == 2 - coords = (xp, yp) - elseif N == 3 - (; zp) = grid - coords = (xp, yp, zp) - end - - # Move arrays to CPU before writing - V isa Array || (V = Array(V)) - p isa Array || (p = Array(p)) - + xp = Array.(xp) + on(state) do (; u, p, t, n) + n % nupdate == 0 || return tformat = replace(string(t), "." => "p") - vtk_grid("$(dir)/$(filename)_t=$tformat", coords...) do vtk + vtk_grid("$(dir)/$(filename)_t=$tformat", xp...) do vtk if :velocity ∈ fields - vels = get_velocity(setup, V, t) - if N == 2 + up = interpolate_u_p(u, setup) + if D == 2 # ParaView prefers 3D vectors. Add zero z-component. - wp = zeros(size(vels[1])) - vels = (vels..., wp) + up = (up..., zero(up[1])) end - vtk["velocity"] = vels + vtk["velocity"] = Array.(up) + end + :pressure ∈ fields && (vtk["pressure"] = Array(p)) + if :vorticity ∈ fields + ω = interpolate_ω_p(vorticity(u, setup), setup) + ω = D == 2 ? Array(ω) : Array.(ω) + vtk["vorticity"] = ω end - - :pressure ∈ fields && (vtk["pressure"] = p) - :vorticity ∈ fields && - (vtk["vorticity"] = reshape(get_vorticity(setup, V, t), :)) - pvd[t] = vtk end end pvd - end; - finalize = (pvd, step_observer) -> vtk_save(pvd), - nupdate, -) + end """ - field_saver(setup; nupdate = 1) + fieldsaver(; setup, nupdate = 1) Create processor that stores the solution every `nupdate` time step to the vector of vectors `V` and `p`. The solution times are stored in the vector `t`. """ -field_saver(setup; nupdate = 1) = processor(function (step_observer) - T = eltype(setup.grid.x) - _V = fill(zeros(T, 0), 0) - _p = fill(zeros(T, 0), 0) - _t = fill(zero(T), 0) - @lift begin - (; V, p, t) = $step_observer - push!(_V, Array(V)) - push!(_p, Array(p)) - push!(_t, t) +field_saver(; setup, nupdate = 1) = + processor() do state + T = eltype(setup.grid.x[1]) + (; u, p) = state[] + fields = (; u = fill(Array.(u), 0), p = fill(Array(p), 0), t = zeros(T, 0)) + on(state) do (; u, p, t, n) + n % nupdate == 0 || return + push!(fields.u, Array.(u)) + push!(fields.p, Array(p)) + push!(fields.t, t) + end + fields end - (; V = _V, p = _p, t = _t) -end; nupdate) diff --git a/src/processors/real_time_plot.jl b/src/processors/real_time_plot.jl index 1cded6595..0e4925d68 100644 --- a/src/processors/real_time_plot.jl +++ b/src/processors/real_time_plot.jl @@ -1,16 +1,48 @@ """ - field_plotter( - setup; - fieldname = :vorticity, - type = nothing, - sleeptime = 0.001, - alpha = 0.05, + realtimeplotter(; + setup, + plot = fieldplot, + nupdate = 1, + displayfig = true, + displayupdates = false, + sleeptime = nothing, + kwargs..., ) -Plot the solution every time the state `o` is updated. +Processor for plotting the solution every `nupdate` time step. The `sleeptime` is slept at every update, to give Makie time to update the plot. Set this to `nothing` to skip sleeping. +""" +realtimeplotter(; + setup, + plot = fieldplot, + nupdate = 1, + displayfig = true, + displayupdates = false, + sleeptime = nothing, + kwargs..., +) = + processor() do outerstate + state = Observable(outerstate[]) + fig = plot(; setup, state = outerstate, kwargs...) + displayfig && display(fig) + on(outerstate) do outerstate + outerstate.n % nupdate == 0 || return + state[] = outerstate + displayupdates && display(fig) + isnothing(sleeptime) || sleep(sleeptime) + end + fig + end + +""" + fieldplot(; + setup, + fieldname = :vorticity, + type = nothing, + alpha = 0.05, + ) Available fieldnames are: @@ -32,22 +64,17 @@ Available plot `type`s for 3D are: The `alpha` value gets passed to `contour` in 3D. """ -field_plotter(setup; nupdate = 1, kwargs...) = - processor(state -> field_plot(setup.grid.dimension, setup, state; kwargs...); nupdate) +fieldplot(; setup, kwargs...) = fieldplot(setup.grid.dimension; setup, kwargs...) -function field_plot( - ::Dimension{2}, +function fieldplot( + ::Dimension{2}; setup, - state; + state, fieldname = :vorticity, type = heatmap, - # sleeptime = 0.001, - sleeptime = nothing, equal_axis = true, - displayfig = true, - displayupdates = false, docolorbar = true, - resolution = (800, 600), + size = nothing, kwargs..., ) (; boundary_conditions, grid) = setup @@ -69,7 +96,6 @@ function field_plot( end _f = Array(_f)[Ip] field = @lift begin - isnothing(sleeptime) || sleep(sleeptime) (; u, p, t) = $state f = if fieldname == :velocity interpolate_u_p!(up, u, setup) @@ -106,53 +132,45 @@ function field_plot( lims end - fig = Figure(; resolution) - if type ∈ (heatmap, image) - ax, hm = type(fig[1, 1], xf..., field; colorrange = lims, kwargs...) + kwargs = (; colorrange = lims, kwargs...) elseif type ∈ (contour, contourf) - ax, hm = type( - fig[1, 1], - xf..., - field; + kwargs = (; extendlow = :auto, extendhigh = :auto, levels = @lift(LinRange($(lims)..., 10)), colorrange = lims, kwargs..., ) - else - error("Unknown plot type") end - ax.title = titlecase(string(fieldname)) - equal_axis && (ax.aspect = DataAspect()) - ax.xlabel = "x" - ax.ylabel = "y" - limits!(ax, xlims[1]..., xlims[2]...) - docolorbar && Colorbar(fig[1, 2], hm) + axis = (; + title = titlecase(string(fieldname)), + xlabel = "x", + ylabel = "y", + limits = (xlims[1]..., xlims[2]...), + ) + equal_axis && (axis = (axis..., aspect = DataAspect())) - displayfig && display(fig) - displayupdates && on(state) do _ - display(fig) - end + size = isnothing(size) ? (;) : (; size) + fig = Figure(; size...) + ax, hm = type(fig[1, 1], xf..., field; axis, kwargs...) + docolorbar && Colorbar(fig[1, 2], hm) fig end -function field_plot( +function fieldplot( ::Dimension{3}, setup, - state; + state, fieldname = :Dfield, - sleeptime = 0.001, alpha = convert(eltype(setup.grid.x[1]), 0.1), isorange = convert(eltype(setup.grid.x[1]), 0.5), equal_axis = true, levels = 3, - displayfig = true, - docolorbar = true, - resolution = (800, 600), + docolorbar = false, + size = (800, 600), kwargs..., ) (; boundary_conditions, grid) = setup @@ -178,7 +196,6 @@ function field_plot( end field = @lift begin - isnothing(sleeptime) || sleep(sleeptime) (; u, p, t) = $state f = if fieldname == :velocity up = interpolate_u_p(u, setup) @@ -206,7 +223,7 @@ function field_plot( isnothing(levels) && (levels = @lift(LinRange($(lims)..., 10))) aspect = equal_axis ? (; aspect = :data) : (;) - fig = Figure(; resolution) + fig = Figure(; size) # ax = Axis3(fig[1, 1]; title = titlecase(string(fieldname)), aspect...) hm = contour( fig[1, 1], @@ -225,42 +242,35 @@ function field_plot( ) docolorbar && Colorbar(fig[1, 2], hm) - - displayfig && display(fig) - fig end -""" - energy_history_plotter(setup) - -Create energy history plot, with a history point added every time `step_observer` is updated. -""" energy_history_plotter(setup; nupdate = 1, kwargs...) = processor(state -> energy_history_plot(setup, state; kwargs...); nupdate) -function energy_history_plot(setup, state; displayfig = true) +""" + energy_history_plot(; setup, state) + +Create energy history plot. +""" +function energy_history_plot(; setup, state) _points = Point2f[] points = @lift begin (; u, p, t) = $state - E = kinetic_energy(setup, u) + E = kinetic_energy(u, setup) push!(_points, Point2f(t, E)) end fig = lines(points; axis = (; xlabel = "t", ylabel = "Kinetic energy")) - displayfig && display(fig) on(_ -> autolimits!(fig.axis), points) fig end """ - energy_spectrum_plotter(setup; nupdate = 1) + energy_spectrum_plot(; setup, state) -Create energy spectrum plot, redrawn every time `step_observer` is updated. +Create energy spectrum plot. """ -energy_spectrum_plotter(setup; nupdate = 1, kwargs...) = - processor(state -> energy_spectrum_plot(setup, state; kwargs...); nupdate) - -function energy_spectrum_plot(setup, state; displayfig = true) +function energy_spectrum_plot(; setup, state) (; dimension, xp, Ip) = setup.grid T = eltype(xp[1]) D = dimension() @@ -279,8 +289,8 @@ function energy_spectrum_plot(setup, state; displayfig = true) e = sum(up -> up[Ip] .^ 2, up) Array(reshape(abs.(fft(e)[ntuple(α -> kx[α] .+ 1, D)...]), :)) end - espec = Figure() - ax = Axis(espec[1, 1]; xlabel = "k", ylabel = "e(k)", xscale = log10, yscale = log10) + fig = Figure() + ax = Axis(fig[1, 1]; xlabel = "k", ylabel = "e(k)", xscale = log10, yscale = log10) ## ylims!(ax, (1e-20, 1)) scatter!(ax, k, ehat; label = "Kinetic energy") krange = LinRange(extrema(k)..., 100) @@ -288,11 +298,10 @@ function energy_spectrum_plot(setup, state; displayfig = true) D == 3 && lines!(ax, krange, 1e6 * krange .^ (-5 / 3); label = "\$k^{-5/3}\$", color = :red) axislegend(ax) - displayfig && display(espec) on(ehat) do _ autolimits!(ax) end - espec + fig end # # Make sure the figure is fully rendered before allowing code to continue diff --git a/src/solvers/solve_unsteady.jl b/src/solvers/solve_unsteady.jl index ed0c9fae9..e313d2e3d 100644 --- a/src/solvers/solve_unsteady.jl +++ b/src/solvers/solve_unsteady.jl @@ -7,7 +7,7 @@ cfl = 1, n_adapt_Δt = 1, inplace = true, - processors = (), + processors = (;), ) Solve unsteady problem using `method`. @@ -17,26 +17,9 @@ an integer. If `Δt = nothing`, the time step is chosen every `n_adapt_Δt` iteration with CFL-number `cfl` . -Each `processor` is called after every `processor.nupdate` time step. +The `processors` are called after every time step. -All arrays and operators are passed through the `device` function. -This allows for performing computations on a different device than the host (CPU). -To compute on an Nvidia GPU using CUDA, change - -``` -solve_unsteady(setup, u₀, p₀, tlims; kwargs...) -``` - -to the following: - -``` -using CUDA -solve_unsteady( - setup, u₀, p₀, tlims; - device = cu, - kwargs... -) -``` +Return a named tuple with the outputs of `processors` with the same field names. Note that the `state` observable passed to the `processor.initialize` function contains vector living on the device, and you may have to move them back to @@ -54,7 +37,7 @@ function solve_unsteady( n_adapt_Δt = 1, inplace = true, docopy = true, - processors = (), + processors = (;), ) if docopy u₀ = copy.(u₀) @@ -79,9 +62,8 @@ function solve_unsteady( isadaptive && (Δt = get_timestep(stepper, cfl)) # Initialize processors for iteration results - state = get_state(stepper) - states = map(ps -> Observable(state), processors) - initialized = map((ps, o) -> ps.initialize(o), processors, states) + state = Observable(get_state(stepper)) + initialized = (; (k => v.initialize(state) for (k, v) in pairs(processors))...) while stepper.t < t_end if isadaptive @@ -102,14 +84,12 @@ function solve_unsteady( end # Process iteration results with each processor - for (ps, o) ∈ zip(processors, states) - # Only update each `nupdate`-th iteration - stepper.n % ps.nupdate == 0 && (o[] = get_state(stepper)) - end + state[] = get_state(stepper) end - (; u, p, t, n) = stepper - finalized = map((ps, i) -> ps.finalize(i, get_state(stepper)), processors, initialized) + finalized = (; + (k => processors[k].finalize(initialized[k], state) for k in keys(processors))... + ) # Final state (; u, p) = stepper diff --git a/test/postprocess2D.jl b/test/postprocess2D.jl index 2881196f7..e33a8c501 100644 --- a/test/postprocess2D.jl +++ b/test/postprocess2D.jl @@ -28,7 +28,7 @@ # Iteration processors processors = ( - field_plotter(setup; nupdate = 1, displayfig = false), + realtimeplotter(; setup, nupdate = 1, displayfig = false), vtk_writer(setup; nupdate = 5, dir = "output", filename = "solution2D"), animator( setup, @@ -36,7 +36,7 @@ nupdate = 10, plotter = field_plotter(setup; displayfig = false), ), - step_logger(), + timelogger(), ) # Solve unsteady problem diff --git a/test/postprocess3D.jl b/test/postprocess3D.jl index 16d2fbe0f..ec1de19ed 100644 --- a/test/postprocess3D.jl +++ b/test/postprocess3D.jl @@ -32,7 +32,7 @@ # Iteration processors processors = ( - field_plotter(setup; nupdate = 5, displayfig = false), + realtimeplotter(; setup, nupdate = 5, displayfig = false), vtk_writer(setup; nupdate = 5, dir = "output", filename = "solution3D"), animator( setup, @@ -40,7 +40,7 @@ nupdate = 10, plotter = field_plotter(setup; displayfig = false), ), - step_logger(), + timelogger(), ) # Solve unsteady problem diff --git a/test/simulation2D.jl b/test/simulation2D.jl index f12c865ac..8e2583a6d 100644 --- a/test/simulation2D.jl +++ b/test/simulation2D.jl @@ -40,7 +40,7 @@ end # Iteration processors - processors = (step_logger(),) + processors = (timelogger(),) @testset "Unsteady problem" begin V, p, outputs = solve_unsteady(setup, V₀, p₀, tlims; Δt = 0.01, processors) diff --git a/test/simulation3D.jl b/test/simulation3D.jl index f1851c65e..50842446e 100644 --- a/test/simulation3D.jl +++ b/test/simulation3D.jl @@ -39,7 +39,7 @@ end # Iteration processors - processors = (step_logger(),) + processors = (timelogger(),) @testset "Unsteady problem" begin V, p, outputs = solve_unsteady(setup, V₀, p₀, tlims; Δt = 0.01, processors) diff --git a/test/solvers.jl b/test/solvers.jl index b4fdb4515..1bec7dfe2 100644 --- a/test/solvers.jl +++ b/test/solvers.jl @@ -90,7 +90,7 @@ Δt = 0.01, pressure_solver, inplace = true, - processors = (step_logger(),), + processors = (timelogger(),), ) @test_broken norm(V - V_exact) / norm(V_exact) < 1e-3 end From 140ba47d9e51ad1c72916abbf2005aecf5a9003e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Sat, 25 Nov 2023 11:12:40 +0100 Subject: [PATCH 135/379] fix: wrong name --- src/processors/processors.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/processors/processors.jl b/src/processors/processors.jl index 73e082017..94fa8fff2 100644 --- a/src/processors/processors.jl +++ b/src/processors/processors.jl @@ -46,7 +46,7 @@ processor(initialize, finalize = (initialized, state) -> initialized) = Create processor that logs time step information. """ -step_logger(; nupdate = 1) = processor(function (state) +timelogger(; nupdate = 1) = processor(function (state) on(state) do (; t, n) n % nupdate == 0 || return @printf "Iteration %d\tt = %g\n" n t From 7b7e69ad21a614b241a1675990b94f140c77cb79 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Sat, 25 Nov 2023 17:01:28 +0100 Subject: [PATCH 136/379] Add projection option --- src/create_initial_conditions.jl | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/create_initial_conditions.jl b/src/create_initial_conditions.jl index 5744c3091..f0d6ea055 100644 --- a/src/create_initial_conditions.jl +++ b/src/create_initial_conditions.jl @@ -4,6 +4,7 @@ initial_velocity, t = 0; pressure_solver = CGPressureSolverManual(setup), + project = true, ) Create initial vectors `(u, p)` at starting time `t`. @@ -15,6 +16,7 @@ function create_initial_conditions( initial_velocity, t = convert(eltype(setup.grid.x[1]), 0); pressure_solver = CGPressureSolverManual(setup), + project = true, ) (; grid) = setup (; dimension, N, Iu, Ip, x, xp, Ω) = grid @@ -42,7 +44,7 @@ function create_initial_conditions( maxdiv = maximum(abs, divergence(u, setup)) # TODO: Maybe eps(T)^(3//4) - if maxdiv > 1e-12 + if project && maxdiv > 1e-12 @warn "Initial velocity field not (discretely) divergence free: $maxdiv.\n" * "Performing additional projection." From c0ca54b386fcd694e984ac277b67eb528bbba3e7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Sat, 25 Nov 2023 17:03:14 +0100 Subject: [PATCH 137/379] Update processors --- src/processors/animator.jl | 9 ++++--- src/processors/real_time_plot.jl | 46 +++++++++++++++++++------------- 2 files changed, 33 insertions(+), 22 deletions(-) diff --git a/src/processors/animator.jl b/src/processors/animator.jl index 79e147c1e..3841e0df5 100644 --- a/src/processors/animator.jl +++ b/src/processors/animator.jl @@ -11,14 +11,15 @@ of the following extensions: - ".gif" The plot is determined by a `plotter` processor. -Additional `kwargs` are passed to Makie's `VideoStream`. +Additional `kwargs` are passed to `plot`. """ -animator(; setup, path, plot = fieldplot, nupdate = 1, kwargs...) = +animator(; setup, path, plot = fieldplot, nupdate = 1, framerate = 24, visible = true, kwargs...) = processor((stream, state) -> save(path, stream)) do outerstate ispath(dirname(path)) || mkpath(dirname(path)) state = Observable(outerstate[]) - fig = plot(; setup, state) - stream = VideoStream(fig; kwargs...) + fig = plot(state; setup, kwargs...) + visible && display(fig) + stream = VideoStream(fig; framerate, visible) on(outerstate) do outerstate outerstate.n % nupdate == 0 || return state[] = outerstate diff --git a/src/processors/real_time_plot.jl b/src/processors/real_time_plot.jl index 0e4925d68..e249d5810 100644 --- a/src/processors/real_time_plot.jl +++ b/src/processors/real_time_plot.jl @@ -13,6 +13,8 @@ Processor for plotting the solution every `nupdate` time step. The `sleeptime` is slept at every update, to give Makie time to update the plot. Set this to `nothing` to skip sleeping. + +Additional `kwargs` are passed to the `plot` function. """ realtimeplotter(; setup, @@ -25,7 +27,7 @@ realtimeplotter(; ) = processor() do outerstate state = Observable(outerstate[]) - fig = plot(; setup, state = outerstate, kwargs...) + fig = plot(state; setup, kwargs...) displayfig && display(fig) on(outerstate) do outerstate outerstate.n % nupdate == 0 || return @@ -37,13 +39,17 @@ realtimeplotter(; end """ - fieldplot(; + fieldplot( + state; setup, fieldname = :vorticity, type = nothing, - alpha = 0.05, + kwargs..., ) +Plot `state` field in pressure points. +If `state` is `Observable`, then the plot is interactive. + Available fieldnames are: - `:velocity`, @@ -64,12 +70,17 @@ Available plot `type`s for 3D are: The `alpha` value gets passed to `contour` in 3D. """ -fieldplot(; setup, kwargs...) = fieldplot(setup.grid.dimension; setup, kwargs...) +fieldplot(state; setup, kwargs...) = fieldplot( + setup.grid.dimension, + state isa Observable ? state : Observable(state); + setup, + kwargs..., +) function fieldplot( - ::Dimension{2}; + ::Dimension{2}, + state; setup, - state, fieldname = :vorticity, type = heatmap, equal_axis = true, @@ -162,15 +173,15 @@ end function fieldplot( ::Dimension{3}, + state; setup, - state, fieldname = :Dfield, alpha = convert(eltype(setup.grid.x[1]), 0.1), isorange = convert(eltype(setup.grid.x[1]), 0.5), equal_axis = true, levels = 3, docolorbar = false, - size = (800, 600), + size = nothing, kwargs..., ) (; boundary_conditions, grid) = setup @@ -222,8 +233,9 @@ function fieldplot( isnothing(levels) && (levels = @lift(LinRange($(lims)..., 10))) - aspect = equal_axis ? (; aspect = :data) : (;) - fig = Figure(; size) + # aspect = equal_axis ? (; aspect = :data) : (;) + size = isnothing(size) ? (;) : (; size) + fig = Figure(; size...) # ax = Axis3(fig[1, 1]; title = titlecase(string(fieldname)), aspect...) hm = contour( fig[1, 1], @@ -233,7 +245,6 @@ function fieldplot( levels, colorrange = lims, # colorrange = extrema(levels), - shading = false, alpha, isorange, # highclip = :red, @@ -245,15 +256,13 @@ function fieldplot( fig end -energy_history_plotter(setup; nupdate = 1, kwargs...) = - processor(state -> energy_history_plot(setup, state; kwargs...); nupdate) - """ - energy_history_plot(; setup, state) + energy_history_plot(state; setup) Create energy history plot. """ -function energy_history_plot(; setup, state) +function energy_history_plot(state; setup) + @assert state isa Observable "Energy history requires observable state." _points = Point2f[] points = @lift begin (; u, p, t) = $state @@ -266,11 +275,12 @@ function energy_history_plot(; setup, state) end """ - energy_spectrum_plot(; setup, state) + energy_spectrum_plot(state; setup) Create energy spectrum plot. """ -function energy_spectrum_plot(; setup, state) +function energy_spectrum_plot(state; setup, naverage = 5^setup.grid.dimension()) + state isa Observable || (state = Observable(state)) (; dimension, xp, Ip) = setup.grid T = eltype(xp[1]) D = dimension() From 87a8fc5d2639877a7302e02bef1f9463223f7d98 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Sat, 25 Nov 2023 17:03:58 +0100 Subject: [PATCH 138/379] Add smoother in energy spectrum --- src/processors/real_time_plot.jl | 45 +++++++++++++++++++++++--------- 1 file changed, 33 insertions(+), 12 deletions(-) diff --git a/src/processors/real_time_plot.jl b/src/processors/real_time_plot.jl index e249d5810..e841e3401 100644 --- a/src/processors/real_time_plot.jl +++ b/src/processors/real_time_plot.jl @@ -282,35 +282,56 @@ Create energy spectrum plot. function energy_spectrum_plot(state; setup, naverage = 5^setup.grid.dimension()) state isa Observable || (state = Observable(state)) (; dimension, xp, Ip) = setup.grid + backend = get_backend(xp[1]) T = eltype(xp[1]) D = dimension() K = size(Ip) .÷ 2 kx = ntuple(α -> 1:K[α]-1, D) - k = KernelAbstractions.zeros(get_backend(xp[1]), T, length.(kx)...) + k = KernelAbstractions.zeros(backend, T, length.(kx)...) for α = 1:D kα = reshape(kx[α], ntuple(Returns(1), α - 1)..., :, ntuple(Returns(1), D - α)...) k .+= kα .^ 2 end k .= sqrt.(k) - k = Array(reshape(k, :)) + k = reshape(k, :) + + # Make averaging matrix + i = sortperm(k) + nbin, r = divrem(length(i), naverage) + ib = KernelAbstractions.zeros(backend, Int, nbin * naverage) + ia = KernelAbstractions.zeros(backend, Int, nbin * naverage) + for j = 1:naverage + copyto!(view(ia, (j-1) * nbin + 1:j * nbin), collect(1:nbin)) + ib[(j-1) * nbin + 1:j * nbin] = i[j:naverage:end-r] + end + vals = KernelAbstractions.ones(backend, T, nbin * naverage) / naverage + A = sparse(ia, ib, vals, nbin, length(i)) + k = Array(A * k) + + up = interpolate_u_p(state[].u, setup) ehat = @lift begin (; u, p, t) = $state - up = interpolate_u_p(u, setup) + interpolate_u_p!(up, u, setup) e = sum(up -> up[Ip] .^ 2, up) - Array(reshape(abs.(fft(e)[ntuple(α -> kx[α] .+ 1, D)...]), :)) + Array(A * reshape(abs.(fft(e)[ntuple(α -> kx[α] .+ 1, D)...]) ./ size(e, 1), :)) end fig = Figure() - ax = Axis(fig[1, 1]; xlabel = "k", ylabel = "e(k)", xscale = log10, yscale = log10) - ## ylims!(ax, (1e-20, 1)) - scatter!(ax, k, ehat; label = "Kinetic energy") + ax = Axis( + fig[1, 1]; + xlabel = "k", + ylabel = "e(k)", + xscale = log10, + yscale = log10, + limits = (extrema(k)..., T(1e-8), T(1)), + ) + lines!(ax, k, ehat; label = "Kinetic energy") krange = LinRange(extrema(k)..., 100) - D == 2 && lines!(ax, krange, 1e7 * krange .^ (-3); label = "k⁻³", color = :red) + D == 2 && lines!(ax, krange, 1e4 * krange .^ (-3); label = "k⁻³", color = :red) D == 3 && - lines!(ax, krange, 1e6 * krange .^ (-5 / 3); label = "\$k^{-5/3}\$", color = :red) + lines!(ax, krange, 1e2 * krange .^ (-5 / 3); label = L"$k^{-5/3}$", color = :red) axislegend(ax) - on(ehat) do _ - autolimits!(ax) - end + # autolimits!(ax) + on(e -> autolimits!(ax), ehat) fig end From 90a9da50b84b4c4b5dd5cf4c740889eaf8318df8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Sat, 25 Nov 2023 23:02:07 +0100 Subject: [PATCH 139/379] Add Laplace matrix --- docs/make.jl | 8 +-- src/operators.jl | 90 +++++++++++++++++++++++- src/solvers/pressure/pressure_poisson.jl | 18 ++--- src/solvers/pressure/pressure_solvers.jl | 19 +++-- 4 files changed, 116 insertions(+), 19 deletions(-) diff --git a/docs/make.jl b/docs/make.jl index e8d5a0573..75175db88 100644 --- a/docs/make.jl +++ b/docs/make.jl @@ -22,16 +22,16 @@ bib = CitationBibliography(joinpath(@__DIR__, "references.bib")) # # Generate examples # examples = [ # "Tutorial: Lid-Driven Cavity (2D)" => "LidDrivenCavity2D", -# "Actuator (2D)" => "Actuator2D", +# "Convergence: Taylor-Green Vortex (2D)" => "TaylorGreenVortex2D", +# "Unsteady inflow: Actuator (2D)" => "Actuator2D", # # "Actuator (3D)" => "Actuator3D", -# "Backward Facing Step (2D)" => "BackwardFacingStep2D", +# "Walls: Backward Facing Step (2D)" => "BackwardFacingStep2D", # # "Backward Facing Step (3D)" => "BackwardFacingStep3D", # "Decaying Turbulunce (2D)" => "DecayingTurbulence2D", # # "Decaying Turbulunce (3D)" => "DecayingTurbulence3D", # # "Lid-Driven Cavity (3D)" => "LidDrivenCavity3D", # "Planar Mixing (2D)" => "PlanarMixing2D", -# "Shear Layer (2D)" => "ShearLayer2D", -# # "Taylor-Green Vortex (2D)" => "TaylorGreenVortex2D", +# # "Shear Layer (2D)" => "ShearLayer2D", # # "Taylor-Green Vortex (3D)" => "TaylorGreenVortex3D", # ] # diff --git a/src/operators.jl b/src/operators.jl index 47d17a852..4f2a543d4 100644 --- a/src/operators.jl +++ b/src/operators.jl @@ -204,7 +204,8 @@ function bodyforce!(F, u, t, setup) @kernel function _bodyforce!(F, force, ::Val{α}, t, I0) where {α} I = @index(Global, Cartesian) I = I + I0 - F[α][I] += force(Dimension(α), ntuple(β -> α == β ? x[β][1+I[β]] : xp[β][I[β]], D)..., t) + F[α][I] += + force(Dimension(α), ntuple(β -> α == β ? x[β][1+I[β]] : xp[β][I[β]], D)..., t) end for α = 1:D I0 = first(Iu[α]) @@ -321,6 +322,93 @@ function laplacian!(L, p, setup) L end +function laplacian_mat(setup) + (; grid, boundary_conditions) = setup + (; dimension, x, N, Np, Ip, Δ, Δu, Ω) = grid + backend = get_backend(x[1]) + T = eltype(x[1]) + D = dimension() + δ = Offset{D}() + Ia = first(Ip) + Ib = last(Ip) + I = KernelAbstractions.zeros(backend, CartesianIndex{D}, 0) + J = KernelAbstractions.zeros(backend, CartesianIndex{D}, 0) + val = KernelAbstractions.zeros(backend, T, 0) + I0 = Ia - oneunit(Ia) + for α = 1:D + a, b = boundary_conditions[α] + # i = Ip[ntuple(β -> α == β ? (Ia[α]+1:Ib[α]-1) : (:), D)...][:] + # ia = Ip[ntuple(β -> α == β ? (Ia[α]:Ia[α]) : (:), D)...][:] + # ib = Ip[ntuple(β -> α == β ? (Ib[α]:Ib[α]) : (:), D)...][:] + i = Ip[ntuple(β -> α == β ? (2:Np[α]-1) : (:), D)...][:] + ia = Ip[ntuple(β -> α == β ? (1:1) : (:), D)...][:] + ib = Ip[ntuple(β -> α == β ? (Np[α]:Np[α]) : (:), D)...][:] + for (aa, bb, j) in [(a, nothing, ia), (nothing, nothing, i), (nothing, b, ib)] + ja = if isnothing(aa) + j .- δ(α) + elseif aa isa PressureBC + error() + elseif aa isa PeriodicBC + ib + elseif aa isa SymmetricBC + ia + elseif aa isa DirichletBC + # The weight of the "left" is zero, but still needs a J in Ip, so + # just set it to ia + ia + end + jb = if isnothing(bb) + j .+ δ(α) + elseif bb isa PressureBC + error() + elseif bb isa PeriodicBC + ia + elseif bb isa SymmetricBC + ib + elseif bb isa DirichletBC + # The weight of the "right" bc is zero, but still needs a J in Ip, so + # just set it to ib + ib + end + J = [J; ja; j; jb] + I = [I; j; j; j] + # val = vcat( + # val, + # map(I -> Ω[I] / Δ[α][I[α]] / Δu[α][I[α]-1], j), + # map(I -> -Ω[I] / Δ[α][I[α]] * (1 / Δu[α][I[α]] + 1 / Δu[α][I[α]-1]), j), + # map(I -> Ω[I] / Δ[α][I[α]] / Δu[α][I[α]], j), + # ) + val = vcat( + val, + @.(Ω[j] / Δ[α][getindex.(j, α)] / Δu[α][getindex.(j, α)-1]), + @.( + -Ω[j] / Δ[α][getindex.(j, α)] * + (1 / Δu[α][getindex.(j, α)] + 1 / Δu[α][getindex.(j, α)-1]) + ), + @.(Ω[j] / Δ[α][getindex.(j, α)] / Δu[α][getindex.(j, α)]), + ) + end + end + # Go back to CPU, otherwise get following error: + # ERROR: CUDA error: an illegal memory access was encountered (code 700, ERROR_ILLEGAL_ADDRESS) + I = Array(I) + J = Array(J) + I = I .- I0 + J = J .- I0 + # linear = copyto!(KernelAbstractions.zeros(backend, Int, Np), collect(LinearIndices(Ip))) + linear = LinearIndices(Ip) + I = linear[I] + J = linear[J] + + # Assemble on CPU, since CUDA overwrites instead of adding + L = sparse(I, J, Array(val)) + # II = copyto!(KernelAbstractions.zeros(backend, Int, length(I)), I) + # JJ = copyto!(KernelAbstractions.zeros(backend, Int, length(J)), J) + # sparse(II, JJ, val) + + Ω isa CuArray ? cu(L) : L +end + """ interpolate_u_p(u, setup) diff --git a/src/solvers/pressure/pressure_poisson.jl b/src/solvers/pressure/pressure_poisson.jl index d4f5f23b8..fff993ec8 100644 --- a/src/solvers/pressure/pressure_poisson.jl +++ b/src/solvers/pressure/pressure_poisson.jl @@ -12,7 +12,7 @@ function pressure_poisson end pressure_poisson(solver, f) = pressure_poisson!( solver, - KernelAbstractions.zeros(get_backend(f), typeof(solver.setup.Re), solver.setup.grid.N), + zero(f), f, ) @@ -29,13 +29,15 @@ See also [`pressure_poisson`](@ref). function pressure_poisson! end function pressure_poisson!(solver::DirectPressureSolver, p, f) - # Assume the Laplace matrix is known (A) and is possibly factorized - - f = view(f, :) - p = view(p, :) - - # Use pre-determined decomposition - p .= solver.A_fact \ f + (; setup, fact) = solver + (; Ip) = setup.grid + solver.f .= view(view(f, Ip), :) + # @infiltrate + solver.p .= fact \ solver.f + # ldiv!(pp, fact, ff) + pp = view(view(p, Ip), :) + pp .= solver.p + p end function pressure_poisson!(solver::CGPressureSolver, p, f) diff --git a/src/solvers/pressure/pressure_solvers.jl b/src/solvers/pressure/pressure_solvers.jl index 2d40f740f..3f27f7349 100644 --- a/src/solvers/pressure/pressure_solvers.jl +++ b/src/solvers/pressure/pressure_solvers.jl @@ -10,13 +10,20 @@ abstract type AbstractPressureSolver{T} end Direct pressure solver using a LU decomposition. """ -struct DirectPressureSolver{T,F<:Factorization{T}} <: AbstractPressureSolver{T} - A_fact::F +struct DirectPressureSolver{T,S,F,A} <: AbstractPressureSolver{T} + setup::S + fact::F + f::A + p::A function DirectPressureSolver(setup) - (; A) = setup.operators - T = eltype(A) - fact = factorize(setup.operators.A) - new{T,typeof(fact)}(fact) + (; x, Np) = setup.grid + T = eltype(x[1]) + backend = get_backend(x[1]) + f = KernelAbstractions.zeros(backend, T, prod(Np)) + p = KernelAbstractions.zeros(backend, T, prod(Np)) + L = laplacian_mat(setup) + fact = lu(L) + new{T,typeof(setup),typeof(fact),typeof(f)}(setup,fact, f, p) end end From 0d4b63692e553a7cdfcd9b8036bc43544f02f3b5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Sat, 25 Nov 2023 23:04:30 +0100 Subject: [PATCH 140/379] Format files --- examples/Actuator2D.jl | 3 +- examples/BackwardFacingStep2D.jl | 9 ++++-- examples/TaylorGreenVortex2D.jl | 6 ++-- scratch/multigrid.jl | 29 +++++++++--------- scratch/train_model.jl | 39 +++++++++++------------- src/operators.jl | 3 +- src/processors/animator.jl | 10 +++++- src/processors/real_time_plot.jl | 4 +-- src/solvers/pressure/pressure_poisson.jl | 6 +--- src/solvers/pressure/pressure_solvers.jl | 2 +- 10 files changed, 57 insertions(+), 54 deletions(-) diff --git a/examples/Actuator2D.jl b/examples/Actuator2D.jl index 35e3b4307..35b026354 100644 --- a/examples/Actuator2D.jl +++ b/examples/Actuator2D.jl @@ -68,7 +68,8 @@ u, p = u₀, p₀ # Solve unsteady problem u, p, outputs = solve_unsteady( setup, - copy.(u₀), copy(p₀), + copy.(u₀), + copy(p₀), # u, p, (0.0, 12.0); method = RK44P2(), diff --git a/examples/BackwardFacingStep2D.jl b/examples/BackwardFacingStep2D.jl index 032601bc5..4a80d10c1 100644 --- a/examples/BackwardFacingStep2D.jl +++ b/examples/BackwardFacingStep2D.jl @@ -39,7 +39,8 @@ ArrayType = Array Re = T(3_000) # Boundary conditions: steady inflow on the top half -U(dim, x, y, t) = dim() == 1 && y ≥ 0 ? 24y * (one(x) / 2 - y) : zero(x) + randn(typeof(x)) / 1_000 +U(dim, x, y, t) = + dim() == 1 && y ≥ 0 ? 24y * (one(x) / 2 - y) : zero(x) + randn(typeof(x)) / 1_000 dUdt(dim, x, y, t) = zero(x) boundary_conditions = ( ## x left, x right @@ -61,7 +62,8 @@ setup = Setup(x, y; Re, boundary_conditions, ArrayType); pressure_solver = CGPressureSolverManual(setup); # Initial conditions (extend inflow) -u₀, p₀ = create_initial_conditions(setup, (dim, x, y) -> U(dim, x, y, zero(x)); pressure_solver); +u₀, p₀ = + create_initial_conditions(setup, (dim, x, y) -> U(dim, x, y, zero(x)); pressure_solver); u, p = copy.(u₀), copy(p₀) # Solve steady state problem @@ -70,7 +72,8 @@ u, p = copy.(u₀), copy(p₀) # Solve unsteady problem u, p, outputs = solve_unsteady( setup, - u₀, p₀, + u₀, + p₀, # u, p, (T(0), T(7)); Δt = T(0.002), diff --git a/examples/TaylorGreenVortex2D.jl b/examples/TaylorGreenVortex2D.jl index 7dd5a2007..d2b15f882 100644 --- a/examples/TaylorGreenVortex2D.jl +++ b/examples/TaylorGreenVortex2D.jl @@ -73,9 +73,9 @@ e = compute_convergence(; # Plot convergence with_theme(; - # linewidth = 5, - # markersize = 20, - # fontsize = 20, +# linewidth = 5, +# markersize = 20, +# fontsize = 20, ) do fig = Figure() ax = Axis( diff --git a/scratch/multigrid.jl b/scratch/multigrid.jl index 098921fff..cfc956c68 100644 --- a/scratch/multigrid.jl +++ b/scratch/multigrid.jl @@ -77,22 +77,23 @@ end # # Load previous LES data # data_train, data_valid, data_test = load("output/forced/data.jld2", "data_train", "data_valid", "data_test") -relerr_track(uref, setup) = processor() do state - (; dimension, x, Ip) = setup.grid - D = dimension() - T = eltype(x[1]) - e = Ref(T(0)) - on(state) do (; u, n) - a, b = T(0), T(0) - for α = 1:D - # @show size(uref[n + 1]) - a += sum(abs2, u[α][Ip] - uref[n+1][α][Ip]) - b += sum(abs2, uref[n+1][α][Ip]) +relerr_track(uref, setup) = + processor() do state + (; dimension, x, Ip) = setup.grid + D = dimension() + T = eltype(x[1]) + e = Ref(T(0)) + on(state) do (; u, n) + a, b = T(0), T(0) + for α = 1:D + # @show size(uref[n + 1]) + a += sum(abs2, u[α][Ip] - uref[n+1][α][Ip]) + b += sum(abs2, uref[n+1][α][Ip]) + end + e[] += sqrt(a) / sqrt(b) / (length(uref) - 1) end - e[] += sqrt(a) / sqrt(b) / (length(uref) - 1) + e end - e -end e_nm = zeros(T, length(ntest)) for (i, n) in enumerate(ntest) diff --git a/scratch/train_model.jl b/scratch/train_model.jl index 837e0825c..c36b90b84 100644 --- a/scratch/train_model.jl +++ b/scratch/train_model.jl @@ -176,22 +176,23 @@ function relerr(u, uref, setup) sqrt(a) / sqrt(b) end -relerr_track(uref, setup) = processor() do state - (; dimension, x, Ip) = setup.grid - D = dimension() - T = eltype(x[1]) - e = Ref(T(0)) - on(state) do (; u, n) - a, b = T(0), T(0) - for α = 1:D - # @show size(uref[n + 1]) - a += sum(abs2, u[α][Ip] - uref[n+1][α][Ip]) - b += sum(abs2, uref[n+1][α][Ip]) +relerr_track(uref, setup) = + processor() do state + (; dimension, x, Ip) = setup.grid + D = dimension() + T = eltype(x[1]) + e = Ref(T(0)) + on(state) do (; u, n) + a, b = T(0), T(0) + for α = 1:D + # @show size(uref[n + 1]) + a += sum(abs2, u[α][Ip] - uref[n+1][α][Ip]) + b += sum(abs2, uref[n+1][α][Ip]) + end + e[] += sqrt(a) / sqrt(b) / (length(uref) - 1) end - e[] += sqrt(a) / sqrt(b) / (length(uref) - 1) + e end - e -end u, u₀, p₀ = nothing, nothing, nothing u = device.(data_test.u[1]) @@ -206,10 +207,7 @@ u_nm, p_nm, outputs = solve_unsteady( (T(0), params.tsim); Δt = data_test.Δt, pressure_solver, - processors = ( - relerr = relerr_track(u, setup), - log = timelogger(; nupdate = 1), - ), + processors = (relerr = relerr_track(u, setup), log = timelogger(; nupdate = 1)), ) relerr_nm = outputs.relerr[] @@ -220,10 +218,7 @@ u_cnn, p_cnn, outputs = solve_unsteady( (T(0), params.tsim); Δt = data_test.Δt, pressure_solver, - processors = ( - relerr = relerr_track(u, setup), - log = timelogger(; nupdate = 1), - ), + processors = (relerr = relerr_track(u, setup), log = timelogger(; nupdate = 1)), ) relerr_cnn = outputs.relerr[] diff --git a/src/operators.jl b/src/operators.jl index 4f2a543d4..c6e98d9c6 100644 --- a/src/operators.jl +++ b/src/operators.jl @@ -399,7 +399,7 @@ function laplacian_mat(setup) linear = LinearIndices(Ip) I = linear[I] J = linear[J] - + # Assemble on CPU, since CUDA overwrites instead of adding L = sparse(I, J, Array(val)) # II = copyto!(KernelAbstractions.zeros(backend, Int, length(I)), I) @@ -550,7 +550,6 @@ function Dfield!(d, G, p, setup; ϵ = eps(eltype(p))) d end - """ Dfield(p, setup) diff --git a/src/processors/animator.jl b/src/processors/animator.jl index 3841e0df5..c4c754141 100644 --- a/src/processors/animator.jl +++ b/src/processors/animator.jl @@ -13,7 +13,15 @@ of the following extensions: The plot is determined by a `plotter` processor. Additional `kwargs` are passed to `plot`. """ -animator(; setup, path, plot = fieldplot, nupdate = 1, framerate = 24, visible = true, kwargs...) = +animator(; + setup, + path, + plot = fieldplot, + nupdate = 1, + framerate = 24, + visible = true, + kwargs..., +) = processor((stream, state) -> save(path, stream)) do outerstate ispath(dirname(path)) || mkpath(dirname(path)) state = Observable(outerstate[]) diff --git a/src/processors/real_time_plot.jl b/src/processors/real_time_plot.jl index e841e3401..601023525 100644 --- a/src/processors/real_time_plot.jl +++ b/src/processors/real_time_plot.jl @@ -301,8 +301,8 @@ function energy_spectrum_plot(state; setup, naverage = 5^setup.grid.dimension()) ib = KernelAbstractions.zeros(backend, Int, nbin * naverage) ia = KernelAbstractions.zeros(backend, Int, nbin * naverage) for j = 1:naverage - copyto!(view(ia, (j-1) * nbin + 1:j * nbin), collect(1:nbin)) - ib[(j-1) * nbin + 1:j * nbin] = i[j:naverage:end-r] + copyto!(view(ia, (j-1)*nbin+1:j*nbin), collect(1:nbin)) + ib[(j-1)*nbin+1:j*nbin] = i[j:naverage:end-r] end vals = KernelAbstractions.ones(backend, T, nbin * naverage) / naverage A = sparse(ia, ib, vals, nbin, length(i)) diff --git a/src/solvers/pressure/pressure_poisson.jl b/src/solvers/pressure/pressure_poisson.jl index fff993ec8..feaf4e06b 100644 --- a/src/solvers/pressure/pressure_poisson.jl +++ b/src/solvers/pressure/pressure_poisson.jl @@ -10,11 +10,7 @@ See also [`pressure_poisson!`](@ref). """ function pressure_poisson end -pressure_poisson(solver, f) = pressure_poisson!( - solver, - zero(f), - f, -) +pressure_poisson(solver, f) = pressure_poisson!(solver, zero(f), f) """ pressure_poisson!(solver, p, f) diff --git a/src/solvers/pressure/pressure_solvers.jl b/src/solvers/pressure/pressure_solvers.jl index 3f27f7349..1e0d46167 100644 --- a/src/solvers/pressure/pressure_solvers.jl +++ b/src/solvers/pressure/pressure_solvers.jl @@ -23,7 +23,7 @@ struct DirectPressureSolver{T,S,F,A} <: AbstractPressureSolver{T} p = KernelAbstractions.zeros(backend, T, prod(Np)) L = laplacian_mat(setup) fact = lu(L) - new{T,typeof(setup),typeof(fact),typeof(f)}(setup,fact, f, p) + new{T,typeof(setup),typeof(fact),typeof(f)}(setup, fact, f, p) end end From e2261be1bc0868a3638f54059b9a22880a46bf75 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Sun, 26 Nov 2023 21:31:28 +0100 Subject: [PATCH 141/379] Update files --- examples/DecayingTurbulence2D.jl | 48 +++++++--------- examples/DecayingTurbulence3D.jl | 58 +++++++------------ examples/LidDrivenCavity2D.jl | 47 ++++++--------- examples/LidDrivenCavity3D.jl | 46 ++++----------- examples/PlaneJets2D.jl | 4 +- examples/TaylorGreenVortex2D.jl | 40 ++++++------- examples/TaylorGreenVortex3D.jl | 65 ++++++--------------- src/create_initial_conditions.jl | 2 +- src/operators.jl | 73 +++++++++++++++++++++--- src/processors/processors.jl | 37 ++++++++---- src/processors/real_time_plot.jl | 27 +++++++-- src/solvers/pressure/pressure_poisson.jl | 11 +++- src/solvers/pressure/pressure_solvers.jl | 14 +++-- src/solvers/solve_unsteady.jl | 14 +++-- 14 files changed, 251 insertions(+), 235 deletions(-) diff --git a/examples/DecayingTurbulence2D.jl b/examples/DecayingTurbulence2D.jl index 346a9fbdb..a6d046264 100644 --- a/examples/DecayingTurbulence2D.jl +++ b/examples/DecayingTurbulence2D.jl @@ -13,18 +13,12 @@ end #src # specific energy spectrum. Due to viscous dissipation, the turbulent features # eventually group to form larger visible eddies. -# We start by loading packages. -# A [Makie](https://github.com/JuliaPlots/Makie.jl) plotting backend is needed -# for plotting. `GLMakie` creates an interactive window (useful for real-time -# plotting), but does not work when building this example on GitHub. -# `CairoMakie` makes high-quality static vector-graphics plots. - #md using CairoMakie using GLMakie #!md using IncompressibleNavierStokes -# Case name for saving results -name = "DecayingTurbulence2D" +# Output directory +output = "output/DecayingTurbulence2D" # Floating point precision T = Float64 @@ -43,7 +37,6 @@ Re = T(10_000) n = 256 lims = T(0), T(1) x = LinRange(lims..., n + 1), LinRange(lims..., n + 1) -# plot_grid(x...) # Build setup and assemble operators setup = Setup(x...; Re, ArrayType); @@ -52,6 +45,7 @@ setup = Setup(x...; Re, ArrayType); # spectral pressure solver pressure_solver = SpectralPressureSolver(setup); +# Create random initial conditions u₀, p₀ = random_field(setup, T(0); pressure_solver); # Solve unsteady problem @@ -63,17 +57,18 @@ u, p, outputs = solve_unsteady( Δt = T(1e-3), pressure_solver, processors = ( - rtp = realtimeplotter(; + ## rtp = realtimeplotter(; setup, nupdate = 1), + ehist = realtimeplotter(; setup, - plot = fieldplot, - ## plot = energy_history_plot, - ## plot = energy_spectrum_plot, - nupdate = 1, + plot = energy_history_plot, + nupdate = 10, + displayfig = false, ), - ## anim = animator(; setup, path = "vorticity.mkv", nupdate = 20), - ## vtk = vtk_writer(; setup, nupdate = 10, dir = "output/$name", filename = "solution"), + espec = realtimeplotter(; setup, plot = energy_spectrum_plot, nupdate = 10), + ## anim = animator(; setup, path = "$output/solution.mkv", nupdate = 20), + ## vtk = vtk_writer(; setup, nupdate = 10, dir = output, filename = "solution"), ## field = fieldsaver(; setup, nupdate = 10), - log = timelogger(; nupdate = 1), + log = timelogger(; nupdate = 100), ), ); @@ -81,17 +76,16 @@ u, p, outputs = solve_unsteady( # # We may visualize or export the computed fields `(u, p)` -# Plot -outputs.rtp +state = (; u, p, t = T(1)); -# Export to VTK -save_vtk(setup, u, p, "output/solution") +# Energy history +outputs.ehist -# Plot pressure -plot_pressure(setup, p₀) +# Energy spectrum +outputs.espec -# Plot velocity -plot_velocity(setup, u) +# Export to VTK +save_vtk(setup, u, p, "$output/solution") -# Plot vorticity -plot_vorticity(setup, u) +# Plot field +fieldplot(state; setup) diff --git a/examples/DecayingTurbulence3D.jl b/examples/DecayingTurbulence3D.jl index e3598ace8..11b98edd0 100644 --- a/examples/DecayingTurbulence3D.jl +++ b/examples/DecayingTurbulence3D.jl @@ -13,18 +13,12 @@ end #src # specific energy spectrum. Due to viscous dissipation, the turbulent features # eventually group to form larger visible eddies. -# We start by loading packages. -# A [Makie](https://github.com/JuliaPlots/Makie.jl) plotting backend is needed for plotting. `GLMakie` creates an interactive window (useful for real-time -# plotting), but does not work when building this example on GitHub. -# `CairoMakie` makes high-quality static vector-graphics plots. - -using FFTW #md using CairoMakie using GLMakie #!md using IncompressibleNavierStokes -# Case name for saving results -name = "DecayingTurbulence3D" +# Output directory +output = "output/DecayingTurbulence3D" # Floating point precision T = Float64 @@ -37,7 +31,7 @@ ArrayType = Array ## using Metal; ArrayType = MtlArray # Reynolds number -Re = T(10_000) +Re = T(6_000) # A 3D grid is a Cartesian product of three vectors n = 32 @@ -45,7 +39,6 @@ lims = T(0), T(1) x = LinRange(lims..., n + 1) y = LinRange(lims..., n + 1) z = LinRange(lims..., n + 1) -plot_grid(x, y, z) # Build setup and assemble operators setup = Setup(x, y, z; Re, ArrayType); @@ -55,7 +48,7 @@ setup = Setup(x, y, z; Re, ArrayType); pressure_solver = SpectralPressureSolver(setup); # Initial conditions -u₀, p₀ = random_field(setup; pressure_solver) +u₀, p₀ = random_field(setup; pressure_solver); # Solve unsteady problem u, p, outputs = solve_unsteady( @@ -63,44 +56,33 @@ u, p, outputs = solve_unsteady( u₀, p₀, (T(0), T(1)); - Δt = T(0.001), + Δt = T(1e-3), pressure_solver, processors = ( - rtp = realtimeplotter(; + ## rtp = realtimeplotter(; setup, plot = fieldplot, nupdate = 10), + ehist = realtimeplotter(; setup, - plot = fieldplot, - ## plot = energy_history_plot, - ## plot = energy_spectrum_plot, - nupdate = 1, + plot = energy_history_plot, + nupdate = 10, + displayfig = false, ), - ## anim = animator(; setup, path = "vorticity.mkv", nupdate = 20), - ## vtk = vtk_writer(; setup, nupdate = 10, dir = "output/$name", filename = "solution"), + espec = realtimeplotter(; setup, plot = energy_spectrum_plot, nupdate = 10), + ## anim = animator(; setup, path = "$output/solution.mkv", nupdate = 20), + ## vtk = vtk_writer(; setup, nupdate = 10, dir = output, filename = "solution"), ## field = fieldsaver(; setup, nupdate = 10), - log = timelogger(; nupdate = 1), + log = timelogger(; nupdate = 100), ), ); -# Field plot -outputs[1] - -# Energy history -outputs[2] - -# Energy spectrum -outputs[3] - # ## Post-process # # We may visualize or export the computed fields `(u, p)` -# Export to VTK -save_vtk(setup, u, p, "output/solution") - -# Plot pressure -plot_pressure(setup, p) +# Energy history +outputs.ehist -# Plot velocity -plot_velocity(setup, u) +# Energy spectrum +outputs.espec -# Plot vorticity -plot_vorticity(setup, u) +# Export to VTK +save_vtk(setup, u, p, "$output/solution") diff --git a/examples/LidDrivenCavity2D.jl b/examples/LidDrivenCavity2D.jl index 0d35f13f8..92447e804 100644 --- a/examples/LidDrivenCavity2D.jl +++ b/examples/LidDrivenCavity2D.jl @@ -23,7 +23,7 @@ using GLMakie #!md using IncompressibleNavierStokes # Case name for saving results -name = "LidDrivenCavity2D" +output = "output/LidDrivenCavity2D" # The code allows for using different floating point number types, including single # precision (`Float32`) and double precision (`Float64`). On the CPU, the speed @@ -92,12 +92,11 @@ setup = Setup(x, y; boundary_conditions, Re, ArrayType); # - [`SpectralPressureSolver`](@ref) (only for periodic boundary conditions and # uniform grids) -pressure_solver = CGPressureSolverManual(setup); +pressure_solver = DirectPressureSolver(setup); # The initial conditions are provided in function. The value `dim()` determines # the velocity component. u₀, p₀ = create_initial_conditions(setup, (dim, x, y) -> zero(x); pressure_solver); -u, p = u₀, p₀ # ## Solve problems # @@ -115,56 +114,46 @@ u, p = u₀, p₀ # later returned by `solve_unsteady`. processors = ( - rtp = realtimeplotter(; - setup, - plot = fieldplot, - ## plot = energy_history_plot, - ## plot = energy_spectrum_plot, - nupdate = 50, - ), - ## anim = animator(; setup, path = "vorticity.mkv", nupdate = 20), - ## vtk = vtk_writer(; setup, nupdate = 10, dir = "output/$name", filename = "solution"), + ## rtp = realtimeplotter(; setup, plot = fieldplot, nupdate = 50), + ## ehist = realtimeplotter(; setup, plot = energy_history_plot, nupdate = 10), + ## espec = realtimeplotter(; setup, plot = energy_spectrum_plot, nupdate = 10), + ## anim = animator(; setup, path = "$output/solution.mkv", nupdate = 20), + vtk = vtk_writer(; setup, nupdate = 100, dir = output, filename = "solution"), ## field = fieldsaver(; setup, nupdate = 10), log = timelogger(; nupdate = 1000), ); # By default, a standard fourth order Runge-Kutta method is used. If we don't # provide the time step explicitly, an adaptive time step is used. +tlims = (T(0), T(10.0)) u, p, outputs = - solve_unsteady(setup, u, p, (T(0), T(0.1)); Δt = T(0.001), pressure_solver, processors); + solve_unsteady(setup, u₀, p₀, tlims; Δt = T(1e-3), pressure_solver, processors); # ## Post-process # # We may visualize or export the computed fields `(V, p)` +state = (; u, p, t = tlims[end]) + # Export fields to VTK. The file `output/solution.vti` may be opened for # visualization in [ParaView](https://www.paraview.org/). This is particularly # useful for inspecting results from 3D simulations. -save_vtk(setup, u, p, "output/solution") +save_vtk(setup, u, p, "$output/solution") # Plot pressure -plot_pressure(setup, p) +fieldplot(state; setup, fieldname = :pressure) # Plot velocity. Note the time stamp used for computing boundary conditions, if # any. -plot_velocity(setup, u) - -# Plot vorticity (with custom levels) -levels = [-7, -5, -4, -3, -2, -1, -0.5, 0, 0.5, 1, 2, 3, 7] -plot_vorticity(setup, u; levels) +fieldplot(state; setup, fieldname = :velocity) -# Plot streamfunction. Note the time stamp used for computing boundary -# conditions, if any -plot_streamfunction(setup, u) +# Plot vorticity +fieldplot(state; setup, fieldname = :vorticity) # In addition, the tuple `outputs` contains quantities from our processors. -# -# The [`field_plotter`](@ref) returns the field plot figure. -outputs[1] - # The [`vtk_writer`](@ref) returns the file name of the ParaView collection # file. This allows for visualizing the solution time series in ParaView. -outputs[2] +outputs.vtk # The logger returns nothing. -outputs[3] +outputs.log diff --git a/examples/LidDrivenCavity3D.jl b/examples/LidDrivenCavity3D.jl index a150f4247..178adabb7 100644 --- a/examples/LidDrivenCavity3D.jl +++ b/examples/LidDrivenCavity3D.jl @@ -11,18 +11,12 @@ end #src # solution should reach at steady state equilibrium after a certain time. The same steady # state should be obtained when solving a steady state problem. -# We start by loading packages. -# A [Makie](https://github.com/JuliaPlots/Makie.jl) plotting backend is needed -# for plotting. `GLMakie` creates an interactive window (useful for real-time -# plotting), but does not work when building this example on GitHub. -# `CairoMakie` makes high-quality static vector-graphics plots. - #md using CairoMakie using GLMakie #!md using IncompressibleNavierStokes # Case name for saving results -name = "LidDrivenCavity3D" +output = "output/LidDrivenCavity3D" # Floating point type T = Float64 @@ -64,29 +58,21 @@ setup = Setup(x, y, z; Re, boundary_conditions, ArrayType); # Initial conditions u₀, p₀ = create_initial_conditions(setup, (dim, x, y, z) -> zero(x)) -# Solve steady state problem -## u, p = solve_steady_state(setup, u₀, p₀; npicard = 5, maxiter = 15); -nothing - # Solve unsteady problem u, p, outputs = solve_unsteady( setup, u₀, p₀, (T(0), T(0.2)); - Δt = T(0.001), + Δt = T(1e-3), processors = ( - rtp = realtimeplotter(; - setup, - plot = fieldplot, - ## plot = energy_history_plot, - ## plot = energy_spectrum_plot, - nupdate = 1, - ), - ## anim = animator(; setup, path = "vorticity.mkv", nupdate = 20), - ## vtk = vtk_writer(; setup, nupdate = 10, dir = "output/$name", filename = "solution"), + ## rtp = realtimeplotter(; setup, plot = fieldplot, nupdate = 50), + ehist = realtimeplotter(; setup, plot = energy_history_plot, nupdate = 10), + ## espec = realtimeplotter(; setup, plot = energy_spectrum_plot, nupdate = 10), + ## anim = animator(; setup, path = "$output/solution.mkv", nupdate = 20), + # vtk = vtk_writer(; setup, nupdate = 100, dir = output, filename = "solution"), ## field = fieldsaver(; setup, nupdate = 10), - log = timelogger(; nupdate = 1), + log = timelogger(; nupdate = 20), ), ); @@ -95,17 +81,7 @@ u, p, outputs = solve_unsteady( # We may visualize or export the computed fields `(V, p)` # Export to VTK -save_vtk(setup, u, p, "output/solution") - -# Plot pressure -plot_pressure(setup, p) - -# Plot velocity -plot_velocity(setup, u) - -# Plot vorticity -plot_vorticity(setup, u) +save_vtk(setup, u, p, "$output/solution") -# Plot streamfunction -## plot_streamfunction(setup, u) -nothing +# Energy history +outputs.ehist diff --git a/examples/PlaneJets2D.jl b/examples/PlaneJets2D.jl index a3f512721..d99b29873 100644 --- a/examples/PlaneJets2D.jl +++ b/examples/PlaneJets2D.jl @@ -85,8 +85,8 @@ U(x, y) = (1 + T(0.1) * (rand(T) - T(0.5))) * U(y) n = 64 ## n = 128 ## n = 256 -x = LinRange(T(0), T(16), 4n) -y = LinRange(-T(10), T(10), 5n) +x = LinRange(T(0), T(16), 4n + 1) +y = LinRange(-T(10), T(10), 5n + 1) plot_grid(x, y) # Build setup and assemble operators diff --git a/examples/TaylorGreenVortex2D.jl b/examples/TaylorGreenVortex2D.jl index d2b15f882..01c405109 100644 --- a/examples/TaylorGreenVortex2D.jl +++ b/examples/TaylorGreenVortex2D.jl @@ -5,7 +5,7 @@ if isdefined(@__MODULE__, :LanguageServer) #src using .IncompressibleNavierStokes #src end #src -# # Taylor-Green vortex - 2D +# # Convergence study: Taylor-Green vortex (2D) # # In this example we consider the Taylor-Green vortex. # In 2D, it has an analytical solution, given by @@ -24,6 +24,9 @@ using GLMakie #!md using IncompressibleNavierStokes using LinearAlgebra +""" +Compare numerical solution with analytical solution at final time. +""" function compute_convergence(; D, nlist, lims, Re, tlims, Δt, uref, ArrayType = Array) T = typeof(lims[1]) e = zeros(T, length(nlist)) @@ -43,6 +46,7 @@ function compute_convergence(; D, nlist, lims, Re, tlims, Δt, uref, ArrayType = (dim, x...) -> uref(dim, x..., tlims[2]), tlims[2]; pressure_solver, + project = false, ) u, p, outputs = solve_unsteady(setup, u₀, p₀, tlims; Δt, pressure_solver) (; Ip) = setup.grid @@ -56,9 +60,11 @@ function compute_convergence(; D, nlist, lims, Re, tlims, Δt, uref, ArrayType = e end +# Analytical solution for 2D Taylor-Green vortex solution(Re) = (dim, x, y, t) -> (dim() == 1 ? -sin(x) * cos(y) : cos(x) * sin(y)) * exp(-2t / Re) +# Compute error for different resolutions Re = 2.0e3 nlist = [2, 4, 8, 16, 32, 64, 128, 256] e = compute_convergence(; @@ -72,22 +78,16 @@ e = compute_convergence(; ) # Plot convergence -with_theme(; -# linewidth = 5, -# markersize = 20, -# fontsize = 20, -) do - fig = Figure() - ax = Axis( - fig[1, 1]; - xscale = log10, - yscale = log10, - xticks = nlist, - xlabel = "n", - title = "Relative error", - ) - scatterlines!(nlist, e; label = "Data") - lines!(collect(extrema(nlist)), n -> n^-2.0; linestyle = :dash, label = "n^-2") - axislegend() - fig -end +fig = Figure() +ax = Axis( + fig[1, 1]; + xscale = log10, + yscale = log10, + xticks = nlist, + xlabel = "n", + title = "Relative error", +) +scatterlines!(ax, nlist, e; label = "Data") +lines!(ax, collect(extrema(nlist)), n -> n^-2.0; linestyle = :dash, label = "n^-2") +axislegend(ax) +fig diff --git a/examples/TaylorGreenVortex3D.jl b/examples/TaylorGreenVortex3D.jl index a4b0ce569..311b63772 100644 --- a/examples/TaylorGreenVortex3D.jl +++ b/examples/TaylorGreenVortex3D.jl @@ -9,18 +9,12 @@ end #src # # In this example we consider the Taylor-Green vortex. -# We start by loading packages. -# A [Makie](https://github.com/JuliaPlots/Makie.jl) plotting backend is needed -# for plotting. `GLMakie` creates an interactive window (useful for real-time -# plotting), but does not work when building this example on GitHub. -# `CairoMakie` makes high-quality static vector-graphics plots. - #md using CairoMakie using GLMakie #!md using IncompressibleNavierStokes -# Case name for saving results -name = "TaylorGreenVortex3D" +# Output directory +output = "output/TaylorGreenVortex3D" # Floating point precision T = Float64 @@ -33,7 +27,7 @@ ArrayType = Array ## using Metal; ArrayType = MtlArray # Reynolds number -Re = T(10_000) +Re = T(6_000) # A 3D grid is a Cartesian product of three vectors n = 32 @@ -41,7 +35,6 @@ lims = T(0), T(1) x = LinRange(lims..., n + 1) y = LinRange(lims..., n + 1) z = LinRange(lims..., n + 1) -plot_grid(x, y, z) # Build setup and assemble operators setup = Setup(x, y, z; Re, ArrayType); @@ -58,34 +51,27 @@ u₀, p₀ = create_initial_conditions( dim() == 2 ? -cospi(2x) * sinpi(2y) * sinpi(2z) / 2 : zero(x); pressure_solver, ); -u, p = u₀, p₀ - -GC.gc() -CUDA.reclaim() - -# Solve steady state problem -## u, p = solve_steady_state(setup, u₀, p₀; npicard = 6) -nothing # Solve unsteady problem u, p, outputs = solve_unsteady( setup, u₀, p₀, - (T(0), T(5)); - Δt = T(0.01), + (T(0), T(1.0)); + Δt = T(1e-3), processors = ( - rtp = realtimeplotter(; + ## rtp = realtimeplotter(; setup, plot = fieldplot, nupdate = 10), + ehist = realtimeplotter(; setup, - plot = fieldplot, - ## plot = energy_history_plot, - ## plot = energy_spectrum_plot, + plot = energy_history_plot, nupdate = 1, + displayfig = false, ), - ## anim = animator(; setup, path = "vorticity.mkv", nupdate = 20), - ## vtk = vtk_writer(; setup, nupdate = 10, dir = "output/$name", filename = "solution"), + espec = realtimeplotter(; setup, plot = energy_spectrum_plot, nupdate = 10), + ## anim = animator(; setup, path = "$output/solution.mkv", nupdate = 20), + ## vtk = vtk_writer(; setup, nupdate = 10, dir = output, filename = "solution"), ## field = fieldsaver(; setup, nupdate = 10), - log = timelogger(; nupdate = 1), + log = timelogger(; nupdate = 100), ), pressure_solver, ); @@ -94,24 +80,11 @@ u, p, outputs = solve_unsteady( # # We may visualize or export the computed fields `(u, p)` -# Export to VTK -save_vtk(setup, u, p, "output/solution") - -# Plot pressure -nothing #md -plot_pressure(setup, p; levels = 3, alpha = 0.05) #!md - -# Plot velocity -nothing #md -plot_velocity(setup, u; levels = 3, alpha = 0.05) #!md +# Energy history +outputs.ehist -# Plot vorticity -nothing #md -plot_vorticity(setup, u; levels = 5, alpha = 0.05) #!md +# Energy spectrum +outputs.espec -# Plot streamfunction -## plot_streamfunction(setup, u) -nothing - -# Field plot -outputs[1] +# Export to VTK +save_vtk(setup, u, p, "$output/solution") diff --git a/src/create_initial_conditions.jl b/src/create_initial_conditions.jl index f0d6ea055..866c7ee67 100644 --- a/src/create_initial_conditions.jl +++ b/src/create_initial_conditions.jl @@ -15,7 +15,7 @@ function create_initial_conditions( setup, initial_velocity, t = convert(eltype(setup.grid.x[1]), 0); - pressure_solver = CGPressureSolverManual(setup), + pressure_solver = DirectPressureSolver(setup), project = true, ) (; grid) = setup diff --git a/src/operators.jl b/src/operators.jl index c6e98d9c6..1fba69068 100644 --- a/src/operators.jl +++ b/src/operators.jl @@ -337,9 +337,6 @@ function laplacian_mat(setup) I0 = Ia - oneunit(Ia) for α = 1:D a, b = boundary_conditions[α] - # i = Ip[ntuple(β -> α == β ? (Ia[α]+1:Ib[α]-1) : (:), D)...][:] - # ia = Ip[ntuple(β -> α == β ? (Ia[α]:Ia[α]) : (:), D)...][:] - # ib = Ip[ntuple(β -> α == β ? (Ib[α]:Ib[α]) : (:), D)...][:] i = Ip[ntuple(β -> α == β ? (2:Np[α]-1) : (:), D)...][:] ia = Ip[ntuple(β -> α == β ? (1:1) : (:), D)...][:] ib = Ip[ntuple(β -> α == β ? (Np[α]:Np[α]) : (:), D)...][:] @@ -347,26 +344,30 @@ function laplacian_mat(setup) ja = if isnothing(aa) j .- δ(α) elseif aa isa PressureBC - error() + # The weight of the "left" BC is zero, but still needs a J inside Ip, so + # just set it to ia + ia elseif aa isa PeriodicBC ib elseif aa isa SymmetricBC ia elseif aa isa DirichletBC - # The weight of the "left" is zero, but still needs a J in Ip, so + # The weight of the "left" BC is zero, but still needs a J inside Ip, so # just set it to ia ia end jb = if isnothing(bb) j .+ δ(α) elseif bb isa PressureBC - error() + # The weight of the "right" BC is zero, but still needs a J inside Ip, so + # just set it to ib + ib elseif bb isa PeriodicBC ia elseif bb isa SymmetricBC ib elseif bb isa DirichletBC - # The weight of the "right" bc is zero, but still needs a J in Ip, so + # The weight of the "right" BC is zero, but still needs a J inside Ip, so # just set it to ib ib end @@ -406,7 +407,8 @@ function laplacian_mat(setup) # JJ = copyto!(KernelAbstractions.zeros(backend, Int, length(J)), J) # sparse(II, JJ, val) - Ω isa CuArray ? cu(L) : L + L + # Ω isa CuArray ? cu(L) : L end """ @@ -608,6 +610,61 @@ Qfield(u, setup) = Qfield!( setup, ) +""" + eig2field!(λ, u, setup) + +Compute the second eigenvalue of ``S^2 + \\Omega^2``. +""" +function eig2field!(λ, u, setup) + (; boundary_conditions, grid) = setup + (; dimension, Np, Ip, Δ, Δu) = grid + D = dimension() + δ = Offset{D}() + @assert D == 3 "eig2 only implemented in 3D" + @kernel function _eig2field!(λ, u, I0) + I = @index(Global, Cartesian) + I = I + I0 + ∂x(uα, I, ::Val{α}, ::Val{β}, Δβ, Δuβ) where {α,β} = + α == β ? (uα[I] - uα[I-δ(β)]) / Δβ[I[β]] : + ( + (uα[I+δ(β)] - uα[I]) / Δuβ[I[β]] + + (uα[I-δ(α)+δ(β)] - uα[I-δ(α)]) / Δuβ[I[β]] + + (uα[I] - uα[I-δ(β)]) / Δuβ[I[β]-1] + + (uα[I-δ(α)] - uα[I-δ(α)-δ(β)]) / Δuβ[I[β]-1] + ) / 4 + ∇u = SMatrix{3,3,eltype(λ),9}( + ∂x(u[1], I, Val(1), Val(1), Δ[1], Δu[1]), + ∂x(u[2], I, Val(2), Val(1), Δ[1], Δu[1]), + ∂x(u[3], I, Val(3), Val(1), Δ[1], Δu[1]), + ∂x(u[1], I, Val(1), Val(2), Δ[2], Δu[2]), + ∂x(u[2], I, Val(2), Val(2), Δ[2], Δu[2]), + ∂x(u[3], I, Val(3), Val(2), Δ[2], Δu[2]), + ∂x(u[1], I, Val(1), Val(3), Δ[3], Δu[3]), + ∂x(u[2], I, Val(2), Val(3), Δ[3], Δu[3]), + ∂x(u[3], I, Val(3), Val(3), Δ[3], Δu[3]), + ) + S = @. (∇u + ∇u') / 2 + Ω = @. (∇u - ∇u') / 2 + λ[I] = eigvals(S^2 + Ω^2)[2] + end + I0 = first(Ip) + I0 -= oneunit(I0) + _eig2field!(get_backend(u[1]), WORKGROUP)(λ, u, I0; ndrange = Np) + # synchronize(get_backend(u[1])) + λ +end + +""" + eig2field(u, setup) + +Compute the second eigenvalue of ``S^2 + \\Omega^2``. +""" +eig2field(u, setup) = eig2field!( + KernelAbstractions.zeros(get_backend(u[1]), eltype(u[1]), setup.grid.N), + u, + setup, +) + """ kinetic_energy(setup, u) diff --git a/src/processors/processors.jl b/src/processors/processors.jl index 94fa8fff2..1dd802868 100644 --- a/src/processors/processors.jl +++ b/src/processors/processors.jl @@ -80,24 +80,39 @@ vtk_writer(; D = dimension() ispath(dir) || mkpath(dir) pvd = paraview_collection(joinpath(dir, filename)) - xp = Array.(xp) + xparr = Array.(xp) + (; u, p) = state[] + if :velocity ∈ fields + up = interpolate_u_p(u, setup) + uparr = Array.(up) + # ParaView prefers 3D vectors. Add zero z-component. + D == 2 && (uparr = (uparr..., zero(up[1]))) + end + if :pressure ∈ fields + parr = Array(p) + end + if :vorticity ∈ fields + ω = vorticity(u, setup) + ωp = interpolate_ω_p(ω, setup) + ωparr = D == 2 ? Array(ωp) : Array.(ωp) + end on(state) do (; u, p, t, n) n % nupdate == 0 || return tformat = replace(string(t), "." => "p") - vtk_grid("$(dir)/$(filename)_t=$tformat", xp...) do vtk + vtk_grid("$(dir)/$(filename)_t=$tformat", xparr...) do vtk if :velocity ∈ fields - up = interpolate_u_p(u, setup) - if D == 2 - # ParaView prefers 3D vectors. Add zero z-component. - up = (up..., zero(up[1])) + interpolate_u_p!(up, u, setup) + for α = 1:D + copyto!(uparr[α], up[α]) end - vtk["velocity"] = Array.(up) + vtk["velocity"] = uparr end - :pressure ∈ fields && (vtk["pressure"] = Array(p)) + :pressure ∈ fields && (vtk["pressure"] = copyto!(parr, p)) if :vorticity ∈ fields - ω = interpolate_ω_p(vorticity(u, setup), setup) - ω = D == 2 ? Array(ω) : Array.(ω) - vtk["vorticity"] = ω + vorticity!(ω, u, setup) + interpolate_ω_p!(ωp, ω, setup) + D == 2 ? copyto!(ωparr, ωp) : copyto!.(ωparr, ωp) + vtk["vorticity"] = ωparr end pvd[t] = vtk end diff --git a/src/processors/real_time_plot.jl b/src/processors/real_time_plot.jl index 601023525..b477624d9 100644 --- a/src/processors/real_time_plot.jl +++ b/src/processors/real_time_plot.jl @@ -97,6 +97,7 @@ function fieldplot( (; u, p, t) = state[] _f = if fieldname == :velocity up = interpolate_u_p(u, setup) + upnorm = zero(p) elseif fieldname == :vorticity ω = vorticity(u, setup) ωp = interpolate_ω_p(ω, setup) @@ -111,6 +112,7 @@ function fieldplot( f = if fieldname == :velocity interpolate_u_p!(up, u, setup) map((u, v) -> √sum(u^2 + v^2), up...) + @. upnorm = sqrt(up[1]^2 + up[2]^2) elseif fieldname == :vorticity apply_bc_u!(u, t, setup) vorticity!(ω, u, setup) @@ -175,7 +177,7 @@ function fieldplot( ::Dimension{3}, state; setup, - fieldname = :Dfield, + fieldname = :eig2field, alpha = convert(eltype(setup.grid.x[1]), 0.1), isorange = convert(eltype(setup.grid.x[1]), 0.5), equal_axis = true, @@ -202,6 +204,9 @@ function fieldplot( elseif fieldname == :Qfield u = state[].u Q = KernelAbstractions.zeros(get_backend(u[1]), eltype(u[1]), setup.grid.N) + elseif fieldname == :eig2field + u = state[].u + λ = KernelAbstractions.zeros(get_backend(u[1]), eltype(u[1]), setup.grid.N) else error("Unknown fieldname") end @@ -224,6 +229,9 @@ function fieldplot( elseif fieldname == :Qfield Qfield!(Q, u, setup) Q + elseif fieldname == :eig2field + eig2field!(λ, u, setup) + λ end Array(f)[Ip] end @@ -275,9 +283,10 @@ function energy_history_plot(state; setup) end """ - energy_spectrum_plot(state; setup) + energy_spectrum_plot(state; setup, naverage = 5^setup.grid.dimension()) Create energy spectrum plot. +The energy modes are averaged over the `naverage` nearest modes. """ function energy_spectrum_plot(state; setup, naverage = 5^setup.grid.dimension()) state isa Observable || (state = Observable(state)) @@ -315,6 +324,15 @@ function energy_spectrum_plot(state; setup, naverage = 5^setup.grid.dimension()) e = sum(up -> up[Ip] .^ 2, up) Array(A * reshape(abs.(fft(e)[ntuple(α -> kx[α] .+ 1, D)...]) ./ size(e, 1), :)) end + + # Buid inertial slope above energy + krange = LinRange(extrema(k)..., 100) + slope, slopelabel = D == 2 ? (-T(3), L"$k^{-3}") : (-T(5 / 3), L"$k^{-5/3}") + inertia = @lift begin + slopeconst = maximum($ehat ./ k .^ slope) + 2 .* slopeconst .* krange .^ slope + end + fig = Figure() ax = Axis( fig[1, 1]; @@ -325,10 +343,7 @@ function energy_spectrum_plot(state; setup, naverage = 5^setup.grid.dimension()) limits = (extrema(k)..., T(1e-8), T(1)), ) lines!(ax, k, ehat; label = "Kinetic energy") - krange = LinRange(extrema(k)..., 100) - D == 2 && lines!(ax, krange, 1e4 * krange .^ (-3); label = "k⁻³", color = :red) - D == 3 && - lines!(ax, krange, 1e2 * krange .^ (-5 / 3); label = L"$k^{-5/3}$", color = :red) + lines!(ax, krange, inertia; label = slopelabel, color = :red) axislegend(ax) # autolimits!(ax) on(e -> autolimits!(ax), ehat) diff --git a/src/solvers/pressure/pressure_poisson.jl b/src/solvers/pressure/pressure_poisson.jl index feaf4e06b..e19353555 100644 --- a/src/solvers/pressure/pressure_poisson.jl +++ b/src/solvers/pressure/pressure_poisson.jl @@ -27,12 +27,17 @@ function pressure_poisson! end function pressure_poisson!(solver::DirectPressureSolver, p, f) (; setup, fact) = solver (; Ip) = setup.grid - solver.f .= view(view(f, Ip), :) + T = eltype(p) + # solver.f .= view(view(f, Ip), :) + # copyto!(solver.f, view(view(f, Ip), :)) + copyto!(view(solver.f, 1:length(solver.f)-1), Array(view(view(f, Ip), :))) # @infiltrate solver.p .= fact \ solver.f - # ldiv!(pp, fact, ff) + # ldiv!(solver.p, fact, solver.f) pp = view(view(p, Ip), :) - pp .= solver.p + # pp .= solver.p + # copyto!(pp, solver.p) + copyto!(pp, T.(view(solver.p, 1:length(solver.p)-1))) p end diff --git a/src/solvers/pressure/pressure_solvers.jl b/src/solvers/pressure/pressure_solvers.jl index 1e0d46167..39467d050 100644 --- a/src/solvers/pressure/pressure_solvers.jl +++ b/src/solvers/pressure/pressure_solvers.jl @@ -17,12 +17,18 @@ struct DirectPressureSolver{T,S,F,A} <: AbstractPressureSolver{T} p::A function DirectPressureSolver(setup) (; x, Np) = setup.grid - T = eltype(x[1]) + # T = eltype(x[1]) + T = Float64 backend = get_backend(x[1]) - f = KernelAbstractions.zeros(backend, T, prod(Np)) - p = KernelAbstractions.zeros(backend, T, prod(Np)) + # f = KernelAbstractions.zeros(backend, T, prod(Np)) + # p = KernelAbstractions.zeros(backend, T, prod(Np)) + f = zeros(T, prod(Np) + 1) + p = zeros(T, prod(Np) + 1) L = laplacian_mat(setup) - fact = lu(L) + e = ones(T, size(L, 2)) + L = [L e; e' 0] + # fact = lu(L) + fact = factorize(L) new{T,typeof(setup),typeof(fact),typeof(f)}(setup, fact, f, p) end end diff --git a/src/solvers/solve_unsteady.jl b/src/solvers/solve_unsteady.jl index e313d2e3d..b9a68dbbe 100644 --- a/src/solvers/solve_unsteady.jl +++ b/src/solvers/solve_unsteady.jl @@ -1,12 +1,16 @@ """ - function solve_unsteady( - setup, u₀, p₀, tlims; - method = RK44(; T = eltype(u₀)), + solve_unsteady( + setup, + u₀, + p₀, + tlims; + method = RK44(; T = eltype(u₀[1])), pressure_solver = DirectPressureSolver(setup), - Δt = zero(eltype(u₀)), + Δt = zero(eltype(u₀[1])), cfl = 1, n_adapt_Δt = 1, inplace = true, + docopy = true, processors = (;), ) @@ -31,7 +35,7 @@ function solve_unsteady( p₀, tlims; method = RK44(; T = eltype(u₀[1])), - pressure_solver = CGPressureSolverManual(setup), + pressure_solver = DirectPressureSolver(setup), Δt = zero(eltype(u₀[1])), cfl = 1, n_adapt_Δt = 1, From 2b5df9a0d56461b1297b61cd7abf75d6f0b15493 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Sun, 26 Nov 2023 22:23:04 +0100 Subject: [PATCH 142/379] Fix docs --- docs/make.jl | 47 +++++++------- docs/src/api/api.md | 29 ++------- docs/src/features/bc.md | 8 ++- docs/src/features/gpu.md | 6 +- docs/src/features/les.md | 4 ++ docs/src/features/operators.md | 2 + docs/src/features/pressure.md | 4 ++ docs/src/generated/LidDrivenCavity2D.md | 1 - examples/Actuator2D.jl | 2 +- examples/Actuator3D.jl | 2 +- examples/BackwardFacingStep2D.jl | 2 +- examples/BackwardFacingStep3D.jl | 2 +- examples/LidDrivenCavity2D.jl | 24 ++++--- examples/LidDrivenCavity3D.jl | 2 +- examples/PlanarMixing2D.jl | 2 +- examples/PlaneJets2D.jl | 2 +- examples/ShearLayer2D.jl | 2 +- src/IncompressibleNavierStokes.jl | 34 ++-------- src/models/convection_models.jl | 34 ---------- src/postprocess/plot_force.jl | 53 ---------------- src/postprocess/plot_pressure.jl | 55 ---------------- src/postprocess/plot_streamfunction.jl | 61 ------------------ src/postprocess/plot_velocity.jl | 57 ----------------- src/postprocess/plot_vorticity.jl | 63 ------------------- src/processors/processors.jl | 2 +- src/setup.jl | 3 - .../plot_grid.jl => utils/plotgrid.jl} | 10 ++- src/{postprocess => utils}/save_vtk.jl | 0 test/postprocess2D.jl | 2 +- test/postprocess3D.jl | 2 +- 30 files changed, 80 insertions(+), 437 deletions(-) delete mode 100644 docs/src/generated/LidDrivenCavity2D.md delete mode 100644 src/models/convection_models.jl delete mode 100644 src/postprocess/plot_force.jl delete mode 100644 src/postprocess/plot_pressure.jl delete mode 100644 src/postprocess/plot_streamfunction.jl delete mode 100644 src/postprocess/plot_velocity.jl delete mode 100644 src/postprocess/plot_vorticity.jl rename src/{postprocess/plot_grid.jl => utils/plotgrid.jl} (90%) rename src/{postprocess => utils}/save_vtk.jl (100%) diff --git a/docs/make.jl b/docs/make.jl index 75175db88..bb034f57e 100644 --- a/docs/make.jl +++ b/docs/make.jl @@ -19,33 +19,30 @@ DocMeta.setdocmeta!( bib = CitationBibliography(joinpath(@__DIR__, "references.bib")) -# # Generate examples -# examples = [ -# "Tutorial: Lid-Driven Cavity (2D)" => "LidDrivenCavity2D", -# "Convergence: Taylor-Green Vortex (2D)" => "TaylorGreenVortex2D", -# "Unsteady inflow: Actuator (2D)" => "Actuator2D", -# # "Actuator (3D)" => "Actuator3D", -# "Walls: Backward Facing Step (2D)" => "BackwardFacingStep2D", -# # "Backward Facing Step (3D)" => "BackwardFacingStep3D", -# "Decaying Turbulunce (2D)" => "DecayingTurbulence2D", -# # "Decaying Turbulunce (3D)" => "DecayingTurbulence3D", -# # "Lid-Driven Cavity (3D)" => "LidDrivenCavity3D", -# "Planar Mixing (2D)" => "PlanarMixing2D", -# # "Shear Layer (2D)" => "ShearLayer2D", -# # "Taylor-Green Vortex (3D)" => "TaylorGreenVortex3D", -# ] -# -# output = "generated" -# for e ∈ examples -# e = joinpath(@__DIR__, "..", "examples", "$(e.second).jl") -# o = joinpath(@__DIR__, "src", output) -# Literate.markdown(e, o) -# # Literate.notebook(e, o) -# # Literate.script(e, o) -# end +# Generate examples +examples = [ + "Tutorial: Lid-Driven Cavity (2D)" => "LidDrivenCavity2D", + "Convergence: Taylor-Green Vortex (2D)" => "TaylorGreenVortex2D", + # "Unsteady inflow: Actuator (2D)" => "Actuator2D", + # # "Actuator (3D)" => "Actuator3D", + # "Walls: Backward Facing Step (2D)" => "BackwardFacingStep2D", + # # "Backward Facing Step (3D)" => "BackwardFacingStep3D", + # "Decaying Turbulunce (2D)" => "DecayingTurbulence2D", + # # "Decaying Turbulunce (3D)" => "DecayingTurbulence3D", + # # "Lid-Driven Cavity (3D)" => "LidDrivenCavity3D", + # "Planar Mixing (2D)" => "PlanarMixing2D", + # # "Shear Layer (2D)" => "ShearLayer2D", + # # "Taylor-Green Vortex (3D)" => "TaylorGreenVortex3D", +] output = "generated" -examples = ["Tutorial: Lid-Driven Cavity (2D)" => "LidDrivenCavity2D"] +for e ∈ examples + e = joinpath(@__DIR__, "..", "examples", "$(e.second).jl") + o = joinpath(@__DIR__, "src", output) + Literate.markdown(e, o) + # Literate.notebook(e, o) + # Literate.script(e, o) +end makedocs(; modules = [IncompressibleNavierStokes], diff --git a/docs/src/api/api.md b/docs/src/api/api.md index c91e9cbf2..1f51ef873 100644 --- a/docs/src/api/api.md +++ b/docs/src/api/api.md @@ -20,28 +20,6 @@ max_size stretched_grid ``` -## Convection Models - -```@docs -AbstractConvectionModel -NoRegConvectionModel -C2ConvectionModel -C4ConvectionModel -LerayConvectionModel -``` - -## Postprocess - -```@docs -plot_force -plot_grid -plot_pressure -plot_streamfunction -plot_velocity -plot_vorticity -save_vtk -``` - ## Preprocess ```@docs @@ -52,13 +30,14 @@ random_field ## Processors ```@docs +processor timelogger vtk_writer fieldsaver realtimeplotter fieldplot -energy_history_plotter -energy_spectrum_plotter +energy_history_plot +energy_spectrum_plot animator ``` @@ -73,6 +52,8 @@ solve_steady_state ## Utils ```@docs +save_vtk +plotgrid get_lims plotmat ``` diff --git a/docs/src/features/bc.md b/docs/src/features/bc.md index 15e65b990..fcb244076 100644 --- a/docs/src/features/bc.md +++ b/docs/src/features/bc.md @@ -1,3 +1,7 @@ +```@meta +CurrentModule = IncompressibleNavierStokes +``` + # Boundary conditions Each boundary has exactly one type of boundary conditions. For periodic @@ -12,6 +16,6 @@ PressureBC ``` ```@docs -IncompressibleNavierStokes.offset_p -IncompressibleNavierStokes.offset_u +offset_p +offset_u ``` diff --git a/docs/src/features/gpu.md b/docs/src/features/gpu.md index 5b0dc0ed9..ce7e98850 100644 --- a/docs/src/features/gpu.md +++ b/docs/src/features/gpu.md @@ -8,12 +8,10 @@ Even if a GPU is not available, the operators are multithreaded if Julia is sta Limitations: -- [`DirectPressureSolver`](@ref) is currently not supported on the GPU. Use [`CGPressureSolver`](@ref) instead. -- Unsteady boundary conditions are currently not supported on the GPU. +- [`DirectPressureSolver`](@ref) is currently used on the CPU with double precision. [`CGPressureSolver`](@ref) works on the GPU. - This has not been tested with other GPU interfaces, such as - [AMDGPU.jl](https://github.com/JuliaGPU/AMDGPU.jl) - [Metal.jl](https://github.com/JuliaGPU/Metal.jl) - [oneAPI.jl](https://github.com/JuliaGPU/oneAPI.jl) If they start supporting sparse matrices and fast Fourier transforms they - could also be used. Alternatively, IncompressibleNavierStokes may also be - refactored to apply the operators without assembling any sparse arrays. + could also be used. diff --git a/docs/src/features/les.md b/docs/src/features/les.md index 5495ef225..dd67c2c80 100644 --- a/docs/src/features/les.md +++ b/docs/src/features/les.md @@ -1,3 +1,7 @@ +```@meta +CurrentModule = IncompressibleNavierStokes +``` + # Large eddy simulation Depending on the problem specification, a given grid resolution may not be diff --git a/docs/src/features/operators.md b/docs/src/features/operators.md index efc841fdd..379cef343 100644 --- a/docs/src/features/operators.md +++ b/docs/src/features/operators.md @@ -37,5 +37,7 @@ Dfield! Dfield Qfield! Qfield +eig2field! +eig2field kinetic_energy ``` diff --git a/docs/src/features/pressure.md b/docs/src/features/pressure.md index 44b1e5bdb..ebbc7161b 100644 --- a/docs/src/features/pressure.md +++ b/docs/src/features/pressure.md @@ -1,3 +1,7 @@ +```@meta +CurrentModule = IncompressibleNavierStokes +``` + # Pressure solvers The discrete pressure Poisson equation diff --git a/docs/src/generated/LidDrivenCavity2D.md b/docs/src/generated/LidDrivenCavity2D.md deleted file mode 100644 index 5042c9155..000000000 --- a/docs/src/generated/LidDrivenCavity2D.md +++ /dev/null @@ -1 +0,0 @@ -# LidDrivenCavity2D diff --git a/examples/Actuator2D.jl b/examples/Actuator2D.jl index 35b026354..9516001de 100644 --- a/examples/Actuator2D.jl +++ b/examples/Actuator2D.jl @@ -28,7 +28,7 @@ name = "Actuator2D" n = 40 x = LinRange(0.0, 10.0, 5n + 1) y = LinRange(-2.0, 2.0, 2n + 1) -plot_grid(x, y) +plotgrid(x, y) # Boundary conditions boundary_conditions = ( diff --git a/examples/Actuator3D.jl b/examples/Actuator3D.jl index 7525d6218..f5f9a807a 100644 --- a/examples/Actuator3D.jl +++ b/examples/Actuator3D.jl @@ -41,7 +41,7 @@ Re = T(100) x = LinRange(0.0, 6.0, 31) y = LinRange(-2.0, 2.0, 41) z = LinRange(-2.0, 2.0, 41) -plot_grid(x, y, z) +plotgrid(x, y, z) # Boundary conditions: Unsteady BC requires time derivatives boundary_conditions = ( diff --git a/examples/BackwardFacingStep2D.jl b/examples/BackwardFacingStep2D.jl index 4a80d10c1..440006bbc 100644 --- a/examples/BackwardFacingStep2D.jl +++ b/examples/BackwardFacingStep2D.jl @@ -54,7 +54,7 @@ boundary_conditions = ( # the walls. x = LinRange(T(0), T(10), 301) y = cosine_grid(-T(0.5), T(0.5), 51) -plot_grid(x, y) +plotgrid(x, y) # Build setup and assemble operators setup = Setup(x, y; Re, boundary_conditions, ArrayType); diff --git a/examples/BackwardFacingStep3D.jl b/examples/BackwardFacingStep3D.jl index 136ec9e8a..2a9a0b46e 100644 --- a/examples/BackwardFacingStep3D.jl +++ b/examples/BackwardFacingStep3D.jl @@ -42,7 +42,7 @@ Re = T(3000) x = LinRange(T(0), T(10), 129) y = LinRange(-T(0.5), T(0.5), 17) z = LinRange(-T(0.25), T(0.25), 9) -plot_grid(x, y, z) +plotgrid(x, y, z) # Boundary conditions: steady inflow on the top half U(dim, x, y, z, t) = dim() == 1 && y ≥ 0 ? 24y * (one(x) / 2 - y) : zero(x) diff --git a/examples/LidDrivenCavity2D.jl b/examples/LidDrivenCavity2D.jl index 92447e804..85ba1cc76 100644 --- a/examples/LidDrivenCavity2D.jl +++ b/examples/LidDrivenCavity2D.jl @@ -33,13 +33,13 @@ output = "output/LidDrivenCavity2D" # scaled judiciously to avoid vanishing digits when applying differential # operators of the form "right minus left divided by small distance". +# Note how floating point type hygiene is enforced in the following using `T` +# to avoid mixing different precisions. + T = Float64 ## T = Float32 ## T = Float16 -# Note how floating point type hygiene is enforced in the following using `T` -# to avoid mixing different precisions. - # We can also choose to do the computations on a different device. By default, # the computations are performed on the host (CPU). An optional `ArrayType` # allows for moving arrays to a different device such as a GPU. @@ -78,7 +78,7 @@ n = 32 lims = T(0), T(1) x = cosine_grid(lims..., n) y = cosine_grid(lims..., n) -plot_grid(x, y) +plotgrid(x, y) # We can now build the setup and assemble operators. # A 3D setup is built if we also provide a vector of z-coordinates. @@ -105,6 +105,7 @@ u₀, p₀ = create_initial_conditions(setup, (dim, x, y) -> zero(x); pressure_s # The [`solve_steady_state`](@ref) function is for computing a state where the right hand side of the # momentum equation is zero. ## u, p = solve_steady_state(setup, u₀, p₀) +nothing # For this test case, the same steady state may be obtained by solving an # unsteady problem for a sufficiently long time. @@ -118,7 +119,7 @@ processors = ( ## ehist = realtimeplotter(; setup, plot = energy_history_plot, nupdate = 10), ## espec = realtimeplotter(; setup, plot = energy_spectrum_plot, nupdate = 10), ## anim = animator(; setup, path = "$output/solution.mkv", nupdate = 20), - vtk = vtk_writer(; setup, nupdate = 100, dir = output, filename = "solution"), + ## vtk = vtk_writer(; setup, nupdate = 100, dir = output, filename = "solution"), ## field = fieldsaver(; setup, nupdate = 10), log = timelogger(; nupdate = 1000), ); @@ -150,10 +151,13 @@ fieldplot(state; setup, fieldname = :velocity) # Plot vorticity fieldplot(state; setup, fieldname = :vorticity) -# In addition, the tuple `outputs` contains quantities from our processors. -# The [`vtk_writer`](@ref) returns the file name of the ParaView collection -# file. This allows for visualizing the solution time series in ParaView. -outputs.vtk - +# In addition, the named tuple `outputs` contains quantities from our +# processors. # The logger returns nothing. +## outputs.rtp +## outputs.ehist +## outputs.espec +## outputs.anim +## outputs.vtk +## outputs.field outputs.log diff --git a/examples/LidDrivenCavity3D.jl b/examples/LidDrivenCavity3D.jl index 178adabb7..d87a193a1 100644 --- a/examples/LidDrivenCavity3D.jl +++ b/examples/LidDrivenCavity3D.jl @@ -36,7 +36,7 @@ Re = T(1_000) x = cosine_grid(T(0), T(1), 25) y = cosine_grid(T(0), T(1), 25) z = LinRange(-T(0.2), T(0.2), 11) -plot_grid(x, y, z) +plotgrid(x, y, z) # Boundary conditions: horizontal movement of the top lid U(dim, x, y, z, t) = dim() == 1 ? one(x) : dim() == 2 ? zero(x) : one(x) / 5 diff --git a/examples/PlanarMixing2D.jl b/examples/PlanarMixing2D.jl index d9e613e60..ccf4b1a24 100644 --- a/examples/PlanarMixing2D.jl +++ b/examples/PlanarMixing2D.jl @@ -50,7 +50,7 @@ n = 64 ## n = 256 x = LinRange(0.0, 256.0, 4n) y = LinRange(-32.0, 32.0, n) -plot_grid(x, y) +plotgrid(x, y) # Build setup and assemble operators setup = Setup(x, y; Re, boundary_conditions); diff --git a/examples/PlaneJets2D.jl b/examples/PlaneJets2D.jl index d99b29873..43b26bb3d 100644 --- a/examples/PlaneJets2D.jl +++ b/examples/PlaneJets2D.jl @@ -87,7 +87,7 @@ n = 64 ## n = 256 x = LinRange(T(0), T(16), 4n + 1) y = LinRange(-T(10), T(10), 5n + 1) -plot_grid(x, y) +plotgrid(x, y) # Build setup and assemble operators setup = Setup(x, y; Re, ArrayType); diff --git a/examples/ShearLayer2D.jl b/examples/ShearLayer2D.jl index 5f4079fd5..81d3c2037 100644 --- a/examples/ShearLayer2D.jl +++ b/examples/ShearLayer2D.jl @@ -40,7 +40,7 @@ n = 128 lims = T(0), T(2π) x = LinRange(lims..., n + 1) y = LinRange(lims..., n + 1) -plot_grid(x, y) +plotgrid(x, y) # Build setup and assemble operators setup = Setup(x, y; Re, ArrayType); diff --git a/src/IncompressibleNavierStokes.jl b/src/IncompressibleNavierStokes.jl index 1b8e5486c..353e31162 100644 --- a/src/IncompressibleNavierStokes.jl +++ b/src/IncompressibleNavierStokes.jl @@ -31,9 +31,6 @@ using CUDA # Let this be constant for now const WORKGROUP = 64 -# Convenience notation -const ⊗ = kron - # # Easily retrieve value from Val # (::Val{x})() where {x} = x @@ -49,7 +46,6 @@ include("grid/max_size.jl") # Models include("models/viscosity_models.jl") -include("models/convection_models.jl") # Setup include("setup.jl") @@ -86,18 +82,11 @@ include("solvers/solve_steady_state.jl") include("solvers/solve_unsteady.jl") # Utils +include("utils/plotgrid.jl") +include("utils/save_vtk.jl") include("utils/get_lims.jl") include("utils/plotmat.jl") -# Postprocess -include("postprocess/plot_force.jl") -include("postprocess/plot_grid.jl") -include("postprocess/plot_pressure.jl") -include("postprocess/plot_velocity.jl") -include("postprocess/plot_vorticity.jl") -include("postprocess/plot_streamfunction.jl") -include("postprocess/save_vtk.jl") - # Closure models include("closures/closure.jl") include("closures/cnn.jl") @@ -109,8 +98,7 @@ include("closures/create_les_data.jl") export PeriodicBC, DirichletBC, SymmetricBC, PressureBC # Models -export AbstractViscosityModel, LaminarModel, MixingLengthModel, SmagorinskyModel, QRModel -export NoRegConvectionModel, C2ConvectionModel, C4ConvectionModel, LerayConvectionModel +export LaminarModel, MixingLengthModel, SmagorinskyModel, QRModel # Processors export processor, timelogger, vtk_writer, fieldsaver, realtimeplotter @@ -124,21 +112,14 @@ export Setup export stretched_grid, cosine_grid # Pressure solvers -export AbstractPressureSolver, - DirectPressureSolver, CGPressureSolver, CGPressureSolverManual, SpectralPressureSolver -export pressure_poisson, - pressure_poisson!, pressure_additional_solve, pressure_additional_solve! - -# Operators -export momentum, divergence, pressuregradient, Dfield!, Qfield! +export DirectPressureSolver, CGPressureSolver, SpectralPressureSolver -# Problems +# Solvers export solve_unsteady, solve_steady_state export create_initial_conditions, random_field -export plot_force, - plot_grid, plot_pressure, plot_streamfunction, plot_velocity, plot_vorticity, save_vtk +export plotgrid, save_vtk export plotmat # Closure models @@ -152,9 +133,6 @@ export create_neural_closure export AdamsBashforthCrankNicolsonMethod, OneLegMethod -# Runge Kutta methods -export ExplicitRungeKuttaMethod, ImplicitRungeKuttaMethod, runge_kutta_method - # Explicit Methods export FE11, SSP22, SSP42, SSP33, SSP43, SSP104, rSSPs2, rSSPs3, Wray3, RK56, DOPRI6 diff --git a/src/models/convection_models.jl b/src/models/convection_models.jl deleted file mode 100644 index 10e47f340..000000000 --- a/src/models/convection_models.jl +++ /dev/null @@ -1,34 +0,0 @@ -""" - AbstractConvectionModel - -Abstract convection model. -""" -abstract type AbstractConvectionModel end - -""" - NoRegConvection() - -Unregularized convection model. -""" -struct NoRegConvectionModel <: AbstractConvectionModel end - -""" - C2ConvectionModel() - -C2 regularization convection model. -""" -struct C2ConvectionModel <: AbstractConvectionModel end - -""" - C4ConvectionModel() - -C4 regularization convection model. -""" -struct C4ConvectionModel <: AbstractConvectionModel end - -""" - LerayConvectionModel() - -Leray regularization convection model. -""" -struct LerayConvectionModel <: AbstractConvectionModel end diff --git a/src/postprocess/plot_force.jl b/src/postprocess/plot_force.jl deleted file mode 100644 index 17fa505c1..000000000 --- a/src/postprocess/plot_force.jl +++ /dev/null @@ -1,53 +0,0 @@ -""" - plot_force(setup, t; kwargs...) - -Plot body force. -""" -function plot_force end - -plot_force(setup, t; kwargs...) = plot_force(setup.grid.dimension, setup, t; kwargs...) - -# 2D version -function plot_force(::Dimension{2}, setup, t; kwargs...) - (; grid, force) = setup - (; xp, yp, xlims, ylims) = grid - (; xu, yu, indu, xlims, ylims) = grid - F = force - - Fp = reshape(F[indu], size(xu)) - # TODO: norm of F instead of Fu - xp, yp = xu[:, 1], yu[1, :] - - # Levels - μ, σ = mean(Fp), std(Fp) - ≈(μ + σ, μ; rtol = 1e-8, atol = 1e-8) && (σ = 1e-4) - levels = LinRange(μ - 1.5σ, μ + 1.5σ, 10) - - fig = Figure() - ax = Axis(fig[1, 1]; aspect = DataAspect(), title = "Force", xlabel = "x", ylabel = "y") - limits!(ax, xlims[1], xlims[2], ylims[1], ylims[2]) - cf = contourf!(ax, xp, yp, Fp; extendlow = :auto, extendhigh = :auto, levels, kwargs...) - Colorbar(fig[1, 2], cf) - fig -end - -# 3D version -function plot_force(::Dimension{3}, setup, t; kwargs...) - (; grid, force) = setup - (; xu, yu, zu, indu) = grid - F = force - - # Get force at pressure points - # up, vp, wp = get_velocity(setup, V, t) - # qp = map((u, v, w) -> √sum(u^2 + v^2 + w^2), up, vp, wp) - Fp = reshape(F[indu], size(xu)) - # TODO: norm of F instead of Fu - xp, yp, zp = xu[:, 1, 1], yu[1, :, 1], zu[1, 1, :] - - # Levels - μ, σ = mean(Fp), std(Fp) - levels = LinRange(μ - 3σ, μ + 3σ, 10) - - # contour(xp, yp, zp, Fp; levels, kwargs...) - contour(xp, yp, zp, Fp) -end diff --git a/src/postprocess/plot_pressure.jl b/src/postprocess/plot_pressure.jl deleted file mode 100644 index f098a6618..000000000 --- a/src/postprocess/plot_pressure.jl +++ /dev/null @@ -1,55 +0,0 @@ -""" - plot_pressure(setup, p; kwargs...) - -Plot pressure. -""" -function plot_pressure end - -plot_pressure(setup, p; kwargs...) = - plot_pressure(setup.grid.dimension, setup, p; kwargs...) - -# 2D version -function plot_pressure(::Dimension{2}, setup, p; kwargs...) - (; xp, xlims) = setup.grid - - xp = Array.(xp) - p = Array(p) - - T = eltype(xp[1]) - - # Levels - μ, σ = mean(p), std(p) - # ≈(μ + σ, μ; rtol = sqrt(eps(T)), atol = sqrt(eps(T))) && (σ = sqrt(sqrt(eps(T)))) - levels = LinRange(μ - T(1.5) * σ, μ + T(1.5) * σ, 10) - - # Plot pressure - fig = Figure() - ax = Axis( - fig[1, 1]; - aspect = DataAspect(), - title = "Pressure", - xlabel = "x", - ylabel = "y", - ) - limits!(ax, xlims[1]..., xlims[2]...) - cf = contourf!(ax, xp..., p; extendlow = :auto, extendhigh = :auto, levels, kwargs...) - Colorbar(fig[1, 2], cf) - - # save("output/pressure.png", fig, pt_per_unit = 2) - - fig -end - -# 3D version -function plot_pressure(::Dimension{3}, setup, p; kwargs...) - (; xp) = setup.grid - - # Levels - μ, σ = mean(p), std(p) - levels = LinRange(μ - 5σ, μ + 5σ, 10) - - p = Array(p) - contour(xp..., p; levels, kwargs...) - - # save("output/pressure.png", fig, pt_per_unit = 2) -end diff --git a/src/postprocess/plot_streamfunction.jl b/src/postprocess/plot_streamfunction.jl deleted file mode 100644 index 8bb1dd2f5..000000000 --- a/src/postprocess/plot_streamfunction.jl +++ /dev/null @@ -1,61 +0,0 @@ -""" - plot_streamfunction(setup, V, t; kwargs...) - -Plot streamfunction. -""" -function plot_streamfunction end - -plot_streamfunction(setup, V, t; kwargs...) = - plot_streamfunction(setup.grid.dimension, setup, V, t; kwargs...) - -# 2D version -function plot_streamfunction(::Dimension{2}, setup, V, t; kwargs...) - (; grid, boundary_conditions) = setup - (; x, y, xlims, ylims) = grid - - if all(==(:periodic), (boundary_conditions.u.x[1], boundary_conditions.v.y[1])) - xψ = x - yψ = y - else - xψ = x[2:(end-1)] - yψ = y[2:(end-1)] - end - - # Get fields - ψ = get_streamfunction(setup, V, t) - - # Levels - μ, σ = mean(ψ), std(ψ) - ≈(μ + σ, μ; rtol = 1e-8, atol = 1e-8) && (σ = 1e-4) - levels = LinRange(μ - 1.5σ, μ + 1.5σ, 10) - - # Plot stream function - fig = Figure() - ax = Axis( - fig[1, 1], - aspect = DataAspect(), - title = "Stream function", - xlabel = "x", - ylabel = "y", - ) - limits!(ax, xlims[1], xlims[2], ylims[1], ylims[2]) - cf = contourf!( - ax, - xψ, - yψ, - ψ; - extendlow = :auto, - extendhigh = :auto, - # levels, - kwargs..., - ) - Colorbar(fig[1, 2], cf) - # save("output/streamfunction.png", fig, pt_per_unit = 2) - - fig -end - -# 3D version -function plot_streamfunction(::Dimension{3}, setup, V, t; kwargs...) - error("Not implemented (3D)") -end diff --git a/src/postprocess/plot_velocity.jl b/src/postprocess/plot_velocity.jl deleted file mode 100644 index 260d72959..000000000 --- a/src/postprocess/plot_velocity.jl +++ /dev/null @@ -1,57 +0,0 @@ -""" - plot_velocity(setup, V, t; kwargs...) - -Plot velocity. -""" -function plot_velocity end - -plot_velocity(setup, u; kwargs...) = - plot_velocity(setup.grid.dimension, setup, u; kwargs...) - -# 2D version -function plot_velocity(::Dimension{2}, setup, u; kwargs...) - (; xp, xlims) = setup.grid - T = eltype(xp[1]) - - # Get velocity at pressure points - up = interpolate_u_p(u, setup) - # qp = map((u, v) -> √(u^2 + v^2), up, vp) - qp = sqrt.(up[1] .^ 2 .+ up[2] .^ 2) - - # Levels - μ, σ = mean(qp), std(qp) - ≈(μ + σ, μ; rtol = sqrt(eps(T)), atol = sqrt(eps(T))) && (σ = sqrt(sqrt(eps(T)))) - levels = LinRange(μ - T(1.5) * σ, μ + T(1.5) * σ, 10) - - fig = Figure() - ax = Axis( - fig[1, 1]; - aspect = DataAspect(), - title = "Velocity", - xlabel = "x", - ylabel = "y", - ) - limits!(ax, xlims[1]..., xlims[2]...) - cf = contourf!(ax, xp..., qp; extendlow = :auto, extendhigh = :auto, levels, kwargs...) - Colorbar(fig[1, 2], cf) - # Colorbar(fig[2,1], cf; vertical = false) - fig -end - -# 3D version -function plot_velocity(::Dimension{3}, setup, u; kwargs...) - (; xp) = setup.grid - - # Get velocity at pressure points - up = interpolate_u_p(u, setup) - qp = map((u, v, w) -> √sum(u^2 + v^2 + w^2), up...) - - # Levels - μ, σ = mean(qp), std(qp) - levels = LinRange(μ - 3σ, μ + 3σ, 10) - - xp = Array(xp) - qp = Array(qp) - contour(xp..., qp; levels, kwargs...) - # contour(xp..., qp; kwargs...) -end diff --git a/src/postprocess/plot_vorticity.jl b/src/postprocess/plot_vorticity.jl deleted file mode 100644 index 1f7cda9b8..000000000 --- a/src/postprocess/plot_vorticity.jl +++ /dev/null @@ -1,63 +0,0 @@ -""" - plot_vorticity(setup, V, t; kwargs...) - -Plot vorticity field. -""" -function plot_vorticity end - -plot_vorticity(setup, u; kwargs...) = - plot_vorticity(setup.grid.dimension, setup, u; kwargs...) - -# 2D version -function plot_vorticity(::Dimension{2}, setup, u; kwargs...) - (; grid) = setup - (; xp, xlims, Ip) = grid - T = eltype(xp[1]) - - xf = Array.(getindex.(xp, Ip.indices)) - - # Get fields - ω = vorticity(u, setup) - ωp = interpolate_ω_p(ω, setup) - ωp = Array(ωp)[Ip] - - # Levels - μ, σ = mean(ωp), std(ωp) - # ≈(μ + σ, μ; rtol = 1e-8, atol = 1e-8) && (σ = 1e-4) - levels = LinRange(μ - T(1.5) * σ, μ + T(1.5) * σ, 10) - - # Plot vorticity - fig = Figure() - ax = Makie.Axis( - fig[1, 1]; - aspect = DataAspect(), - title = "Vorticity", - xlabel = "x", - ylabel = "y", - ) - limits!(ax, xlims[1]..., xlims[2]...) - # cf = contourf!(ax, xp..., ω; extendlow = :auto, extendhigh = :auto, levels, kwargs...) - cf = heatmap!(ax, xf..., ωp; kwargs...) - Colorbar(fig[1, 2], cf) - - # save("output/vorticity.png", fig, pt_per_unit = 2) - - fig -end - -# 3D version -function plot_vorticity(::Dimension{3}, setup, u; kwargs...) - (; grid) = setup - (; xp) = grid - - ωp = interpolate_ω_p(vorticity(u, setup), setup) - qp = map((u, v, w) -> √sum(u^2 + v^2 + w^2), ωp...) - - # Levels - μ, σ = mean(qp), std(qp) - levels = LinRange(μ - 3σ, μ + 3σ, 10) - - xp = Array.(xp) - qp = Array(qp) - contour(xp..., qp; levels, kwargs...) -end diff --git a/src/processors/processors.jl b/src/processors/processors.jl index 1dd802868..cd69498c5 100644 --- a/src/processors/processors.jl +++ b/src/processors/processors.jl @@ -125,7 +125,7 @@ vtk_writer(; Create processor that stores the solution every `nupdate` time step to the vector of vectors `V` and `p`. The solution times are stored in the vector `t`. """ -field_saver(; setup, nupdate = 1) = +fieldsaver(; setup, nupdate = 1) = processor() do state T = eltype(setup.grid.x[1]) (; u, p) = state[] diff --git a/src/setup.jl b/src/setup.jl index ed1ec6fc0..59048d24d 100644 --- a/src/setup.jl +++ b/src/setup.jl @@ -4,7 +4,6 @@ boundary_conditions = ntuple(d -> (PeriodicBC(), PeriodicBC()), length(x)), Re = convert(eltype(x[1]), 1_000), viscosity_model = LaminarModel(), - convection_model = NoRegConvectionModel(), bodyforce = nothing, closure_model = nothing, ArrayType = Array, @@ -17,7 +16,6 @@ Setup( boundary_conditions = ntuple(d -> (PeriodicBC(), PeriodicBC()), length(x)), Re = convert(eltype(x[1]), 1_000), viscosity_model = LaminarModel(), - convection_model = NoRegConvectionModel(), bodyforce = nothing, closure_model = nothing, ArrayType = Array, @@ -26,7 +24,6 @@ Setup( boundary_conditions, Re, viscosity_model, - convection_model, bodyforce, closure_model, ArrayType, diff --git a/src/postprocess/plot_grid.jl b/src/utils/plotgrid.jl similarity index 90% rename from src/postprocess/plot_grid.jl rename to src/utils/plotgrid.jl index 8d6717db9..cc03cdf4c 100644 --- a/src/postprocess/plot_grid.jl +++ b/src/utils/plotgrid.jl @@ -1,20 +1,18 @@ """ - plot_grid(x, y) - plot_grid(x, y, z) - plot_grid(grid) + plotgrid(x...) Plot nonuniform Cartesian grid. """ -function plot_grid end +function plotgrid end -plot_grid(x, y) = wireframe( +plotgrid(x, y) = wireframe( x, y, zeros(eltype(x), length(x), length(y)); axis = (; aspect = DataAspect(), xlabel = "x", ylabel = "y"), ) -function plot_grid(x, y, z) +function plotgrid(x, y, z) nx, ny, nz = length(x), length(y), length(z) T = eltype(x) diff --git a/src/postprocess/save_vtk.jl b/src/utils/save_vtk.jl similarity index 100% rename from src/postprocess/save_vtk.jl rename to src/utils/save_vtk.jl diff --git a/test/postprocess2D.jl b/test/postprocess2D.jl index e33a8c501..dd85c5d32 100644 --- a/test/postprocess2D.jl +++ b/test/postprocess2D.jl @@ -7,7 +7,7 @@ setup = Setup(x, y) - @test plot_grid(x, y) isa Makie.FigureAxisPlot + @test plotgrid(x, y) isa Makie.FigureAxisPlot pressure_solver = SpectralPressureSolver(setup) diff --git a/test/postprocess3D.jl b/test/postprocess3D.jl index ec1de19ed..857102070 100644 --- a/test/postprocess3D.jl +++ b/test/postprocess3D.jl @@ -10,7 +10,7 @@ z = LinRange(lims..., n) setup = Setup(x, y, z) - @test plot_grid(x, y, z) isa Makie.Figure + @test plotgrid(x, y, z) isa Makie.Figure pressure_solver = SpectralPressureSolver(setup) From 2fd779aa99bf27272c9364441ff6d729c22ebb53 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Sun, 26 Nov 2023 22:25:23 +0100 Subject: [PATCH 143/379] Add bound --- Project.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/Project.toml b/Project.toml index 7505d1f76..47c359661 100644 --- a/Project.toml +++ b/Project.toml @@ -42,6 +42,7 @@ Printf = "1" Random = "1" SparseArrays = "1" Statistics = "1" +StaticArrays = "1" Tullio = "0.3" WriteVTK = "1" Zygote = "0.6" From 9cb485352aeba18c057c7b64fa2a63d5c953ced1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Sun, 26 Nov 2023 22:26:57 +0100 Subject: [PATCH 144/379] fix: typo --- src/processors/real_time_plot.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/processors/real_time_plot.jl b/src/processors/real_time_plot.jl index b477624d9..3555322c0 100644 --- a/src/processors/real_time_plot.jl +++ b/src/processors/real_time_plot.jl @@ -325,7 +325,7 @@ function energy_spectrum_plot(state; setup, naverage = 5^setup.grid.dimension()) Array(A * reshape(abs.(fft(e)[ntuple(α -> kx[α] .+ 1, D)...]) ./ size(e, 1), :)) end - # Buid inertial slope above energy + # Build inertial slope above energy krange = LinRange(extrema(k)..., 100) slope, slopelabel = D == 2 ? (-T(3), L"$k^{-3}") : (-T(5 / 3), L"$k^{-5/3}") inertia = @lift begin From 9c31c13f4e6b7ff449db4d7f3ac14ca5b9698ffc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Sun, 26 Nov 2023 22:49:39 +0100 Subject: [PATCH 145/379] Add bounds --- Project.toml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/Project.toml b/Project.toml index 47c359661..9504135cc 100644 --- a/Project.toml +++ b/Project.toml @@ -27,9 +27,12 @@ Zygote = "e88e6eb3-aa80-5325-afca-941959d7151f" [compat] Adapt = "3" +Aqua = "0.8" CUDA = "5" +CairoMakie = "0.11" ComponentArrays = "0.15" FFTW = "1" +GLMakie = "0.9" IterativeSolvers = "0.9" KernelAbstractions = "0.9" LinearAlgebra = "1" @@ -41,8 +44,9 @@ Optimisers = "0.3" Printf = "1" Random = "1" SparseArrays = "1" -Statistics = "1" StaticArrays = "1" +Statistics = "1" +Test = "1" Tullio = "0.3" WriteVTK = "1" Zygote = "0.6" From 1cf9c44a01d139fba8c1f00169f1ca9d3244f38c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Sun, 26 Nov 2023 22:59:09 +0100 Subject: [PATCH 146/379] fix broadcast --- examples/LidDrivenCavity2D.jl | 3 ++- src/operators.jl | 12 ++++++++---- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/examples/LidDrivenCavity2D.jl b/examples/LidDrivenCavity2D.jl index 85ba1cc76..73e347905 100644 --- a/examples/LidDrivenCavity2D.jl +++ b/examples/LidDrivenCavity2D.jl @@ -132,7 +132,7 @@ u, p, outputs = # ## Post-process # -# We may visualize or export the computed fields `(V, p)` +# We may visualize or export the computed fields `(u, p)` state = (; u, p, t = tlims[end]) @@ -154,6 +154,7 @@ fieldplot(state; setup, fieldname = :vorticity) # In addition, the named tuple `outputs` contains quantities from our # processors. # The logger returns nothing. + ## outputs.rtp ## outputs.ehist ## outputs.espec diff --git a/src/operators.jl b/src/operators.jl index 1fba69068..bb9f53f60 100644 --- a/src/operators.jl +++ b/src/operators.jl @@ -342,7 +342,8 @@ function laplacian_mat(setup) ib = Ip[ntuple(β -> α == β ? (Np[α]:Np[α]) : (:), D)...][:] for (aa, bb, j) in [(a, nothing, ia), (nothing, nothing, i), (nothing, b, ib)] ja = if isnothing(aa) - j .- δ(α) + # j .- δ(α) + CartesianIndex.(Tuple.(j) .- Tuple(δ(α))) elseif aa isa PressureBC # The weight of the "left" BC is zero, but still needs a J inside Ip, so # just set it to ia @@ -357,7 +358,8 @@ function laplacian_mat(setup) ia end jb = if isnothing(bb) - j .+ δ(α) + # j .+ δ(α) + CartesianIndex.(Tuple.(j) .+ Tuple(δ(α))) elseif bb isa PressureBC # The weight of the "right" BC is zero, but still needs a J inside Ip, so # just set it to ib @@ -394,8 +396,10 @@ function laplacian_mat(setup) # ERROR: CUDA error: an illegal memory access was encountered (code 700, ERROR_ILLEGAL_ADDRESS) I = Array(I) J = Array(J) - I = I .- I0 - J = J .- I0 + # I = I .- I0 + # J = J .- I0 + I = CartesianIndex.(Tuple.(I) .- Tuple(I0)) + J = CartesianIndex.(Tuple.(J) .- Tuple(I0)) # linear = copyto!(KernelAbstractions.zeros(backend, Int, Np), collect(LinearIndices(Ip))) linear = LinearIndices(Ip) I = linear[I] From c7db79124f69f99d687e13401c358ae1e6df5772 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Sun, 26 Nov 2023 23:50:32 +0100 Subject: [PATCH 147/379] Fix broadcast --- examples/LidDrivenCavity2D.jl | 3 ++- src/operators.jl | 10 ++++------ 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/examples/LidDrivenCavity2D.jl b/examples/LidDrivenCavity2D.jl index 73e347905..c469a3a77 100644 --- a/examples/LidDrivenCavity2D.jl +++ b/examples/LidDrivenCavity2D.jl @@ -14,6 +14,7 @@ end #src # We start by loading packages. # A [Makie](https://github.com/JuliaPlots/Makie.jl) plotting backend is needed +# # for plotting. `GLMakie` creates an interactive window (useful for real-time # plotting), but does not work when building this example on GitHub. # `CairoMakie` makes high-quality static vector-graphics plots. @@ -126,7 +127,7 @@ processors = ( # By default, a standard fourth order Runge-Kutta method is used. If we don't # provide the time step explicitly, an adaptive time step is used. -tlims = (T(0), T(10.0)) +tlims = (T(0), T(10)) u, p, outputs = solve_unsteady(setup, u₀, p₀, tlims; Δt = T(1e-3), pressure_solver, processors); diff --git a/src/operators.jl b/src/operators.jl index bb9f53f60..bd53f3b44 100644 --- a/src/operators.jl +++ b/src/operators.jl @@ -342,8 +342,7 @@ function laplacian_mat(setup) ib = Ip[ntuple(β -> α == β ? (Np[α]:Np[α]) : (:), D)...][:] for (aa, bb, j) in [(a, nothing, ia), (nothing, nothing, i), (nothing, b, ib)] ja = if isnothing(aa) - # j .- δ(α) - CartesianIndex.(Tuple.(j) .- Tuple(δ(α))) + j .- [δ(α)] elseif aa isa PressureBC # The weight of the "left" BC is zero, but still needs a J inside Ip, so # just set it to ia @@ -358,8 +357,7 @@ function laplacian_mat(setup) ia end jb = if isnothing(bb) - # j .+ δ(α) - CartesianIndex.(Tuple.(j) .+ Tuple(δ(α))) + j .+ [δ(α)] elseif bb isa PressureBC # The weight of the "right" BC is zero, but still needs a J inside Ip, so # just set it to ib @@ -398,8 +396,8 @@ function laplacian_mat(setup) J = Array(J) # I = I .- I0 # J = J .- I0 - I = CartesianIndex.(Tuple.(I) .- Tuple(I0)) - J = CartesianIndex.(Tuple.(J) .- Tuple(I0)) + I = I .- [I0] + J = J .- [I0] # linear = copyto!(KernelAbstractions.zeros(backend, Int, Np), collect(LinearIndices(Ip))) linear = LinearIndices(Ip) I = linear[I] From 5bbbeb807cdd4e3d42281f0283f3bdfe150b9437 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Mon, 27 Nov 2023 16:05:37 +0100 Subject: [PATCH 148/379] Put loops inside kernels --- src/operators.jl | 119 ++++++++++++++++++++++++----------------------- 1 file changed, 62 insertions(+), 57 deletions(-) diff --git a/src/operators.jl b/src/operators.jl index bd53f3b44..4b0807e71 100644 --- a/src/operators.jl +++ b/src/operators.jl @@ -22,10 +22,14 @@ function divergence!(M, u, setup) (; Δ, N, Ip) = grid D = length(u) δ = Offset{D}() - @kernel function _divergence!(M, u, ::Val{α}, I0) where {α} + @kernel function _divergence!(M, u, I0) I = @index(Global, Cartesian) I = I + I0 - M[I] += (u[α][I] - u[α][I-δ(α)]) / Δ[α][I[α]] + m = zero(eltype(M)) + for α = 1:D + m += (u[α][I] - u[α][I-δ(α)]) / Δ[α][I[α]] + end + M[I] = m end # All volumes have a right velocity # All volumes have a left velocity except the first one @@ -35,10 +39,7 @@ function divergence!(M, u, setup) # ndrange = Np # I0 = first(Ip) I0 -= oneunit(I0) - M .= 0 - for α = 1:D - _divergence!(get_backend(M), WORKGROUP)(M, u, Val(α), I0; ndrange) - end + _divergence!(get_backend(M), WORKGROUP)(M, u, I0; ndrange) M end @@ -98,21 +99,21 @@ function vorticity!(::Dimension{3}, ω, u, setup) (; dimension, Δu, N) = grid D = dimension() δ = Offset{D}() - @kernel function _vorticity!(ω, u, ::Val{α}, I0) where {α} + @kernel function _vorticity!(ω, u, I0) T = eltype(ω) I = @index(Global, Cartesian) I = I + I0 - α₊ = mod1(α + 1, D) - α₋ = mod1(α - 1, D) - ω[α][I] = - (u[α₋][I+δ(α₊)] - u[α₋][I]) / Δu[α₊][I[α₊]] - - (u[α₊][I+δ(α₋)] - u[α₊][I]) / Δu[α₋][I[α₋]] + for (α, α₊, α₋) in ((1, 2, 3), (2, 3, 1), (3, 1, 2)) + # α₊ = mod1(α + 1, D) + # α₋ = mod1(α - 1, D) + ω[α][I] = + (u[α₋][I+δ(α₊)] - u[α₋][I]) / Δu[α₊][I[α₊]] - + (u[α₊][I+δ(α₋)] - u[α₊][I]) / Δu[α₋][I[α₋]] + end end I0 = CartesianIndex(ntuple(Returns(1), D)) I0 -= oneunit(I0) - for α = 1:D - _vorticity!(get_backend(ω[1]), WORKGROUP)(ω, u, Val(α), I0; ndrange = N .- 1) - end + _vorticity!(get_backend(ω[1]), WORKGROUP)(ω, u, I0; ndrange = N .- 1) ω end @@ -126,29 +127,31 @@ function convection!(F, u, setup) (; dimension, Δ, Δu, Nu, Iu, A) = grid D = dimension() δ = Offset{D}() - @kernel function _convection!(F, u, ::Val{α}, ::Val{β}, I0) where {α,β} + @kernel function _convection!(F, u, ::Val{α}, ::Val{βrange}, I0) where {α,βrange} I = @index(Global, Cartesian) I = I + I0 - Δuαβ = α == β ? Δu[β] : Δ[β] - uαβ1 = A[α][β][2][I[β]-1] * u[α][I-δ(β)] + A[α][β][1][I[β]] * u[α][I] - uαβ2 = A[α][β][2][I[β]] * u[α][I] + A[α][β][1][I[β]+1] * u[α][I+δ(β)] - uβα1 = A[β][α][2][I[α]-1] * u[β][I-δ(β)] + A[β][α][1][I[α]+1] * u[β][I-δ(β)+δ(α)] - uβα2 = A[β][α][2][I[α]] * u[β][I] + A[β][α][1][I[α]] * u[β][I+δ(α)] - F[α][I] -= (uαβ2 * uβα2 - uαβ1 * uβα1) / Δuαβ[I[β]] + # for β = 1:D + KernelAbstractions.Extras.LoopInfo.@unroll for β in βrange + Δuαβ = α == β ? Δu[β] : Δ[β] + uαβ1 = A[α][β][2][I[β]-1] * u[α][I-δ(β)] + A[α][β][1][I[β]] * u[α][I] + uαβ2 = A[α][β][2][I[β]] * u[α][I] + A[α][β][1][I[β]+1] * u[α][I+δ(β)] + uβα1 = + A[β][α][2][I[α]-1] * u[β][I-δ(β)] + A[β][α][1][I[α]+1] * u[β][I-δ(β)+δ(α)] + uβα2 = A[β][α][2][I[α]] * u[β][I] + A[β][α][1][I[α]] * u[β][I+δ(α)] + F[α][I] -= (uαβ2 * uβα2 - uαβ1 * uβα1) / Δuαβ[I[β]] + end end for α = 1:D I0 = first(Iu[α]) I0 -= oneunit(I0) - for β = 1:D - _convection!(get_backend(F[1]), WORKGROUP)( - F, - u, - Val(α), - Val(β), - I0; - ndrange = Nu[α], - ) - end + _convection!(get_backend(F[1]), WORKGROUP)( + F, + u, + Val(α), + Val(1:D), + I0; + ndrange = Nu[α], + ) end F end @@ -164,29 +167,30 @@ function diffusion!(F, u, setup) D = dimension() δ = Offset{D}() ν = 1 / Re - @kernel function _diffusion!(F, u, ::Val{α}, ::Val{β}, I0) where {α,β} + @kernel function _diffusion!(F, u, ::Val{α}, ::Val{βrange}, I0) where {α,βrange} I = @index(Global, Cartesian) I = I + I0 - Δuαβ = (α == β ? Δu[β] : Δ[β]) - F[α][I] += - ν * ( - (u[α][I+δ(β)] - u[α][I]) / (β == α ? Δ[β][I[β]+1] : Δu[β][I[β]]) - - (u[α][I] - u[α][I-δ(β)]) / (β == α ? Δ[β][I[β]] : Δu[β][I[β]-1]) - ) / Δuαβ[I[β]] + # for β = 1:D + KernelAbstractions.Extras.LoopInfo.@unroll for β in βrange + Δuαβ = (α == β ? Δu[β] : Δ[β]) + F[α][I] += + ν * ( + (u[α][I+δ(β)] - u[α][I]) / (β == α ? Δ[β][I[β]+1] : Δu[β][I[β]]) - + (u[α][I] - u[α][I-δ(β)]) / (β == α ? Δ[β][I[β]] : Δu[β][I[β]-1]) + ) / Δuαβ[I[β]] + end end for α = 1:D I0 = first(Iu[α]) I0 -= oneunit(I0) - for β = 1:D - _diffusion!(get_backend(F[1]), WORKGROUP)( - F, - u, - Val(α), - Val(β), - I0; - ndrange = Nu[α], - ) - end + _diffusion!(get_backend(F[1]), WORKGROUP)( + F, + u, + Val(α), + Val(1:D), + I0; + ndrange = Nu[α], + ) end F end @@ -302,23 +306,24 @@ function laplacian!(L, p, setup) (; dimension, Δ, Δu, N, Np, Ip, Ω) = grid D = dimension() δ = Offset{D}() - @kernel function _laplacian!(L, p, ::Val{α}, I0) where {α} + @kernel function _laplacian!(L, p, I0) I = @index(Global, Cartesian) I = I + I0 - L[I] += - Ω[I] / Δ[α][I[α]] * - ((p[I+δ(α)] - p[I]) / Δu[α][I[α]] - (p[I] - p[I-δ(α)]) / Δu[α][I[α]-1]) + lap = zero(eltype(p)) + for α = 1:D + lap += + Ω[I] / Δ[α][I[α]] * + ((p[I+δ(α)] - p[I]) / Δu[α][I[α]] - (p[I] - p[I-δ(α)]) / Δu[α][I[α]-1]) + end + L[I] = lap end - L .= 0 # All volumes have a right velocity # All volumes have a left velocity except the first one # Start at second volume ndrange = Np I0 = first(Ip) I0 -= oneunit(I0) - for α = 1:D - _laplacian!(get_backend(L), WORKGROUP)(L, p, Val(α), I0; ndrange) - end + _laplacian!(get_backend(L), WORKGROUP)(L, p, I0; ndrange) L end From b26a6064b8e4311c12b744e626fa6f71488dd9d5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Mon, 27 Nov 2023 16:24:30 +0100 Subject: [PATCH 149/379] Rename kernels --- src/operators.jl | 91 +++++++++++++++++++----------------------------- 1 file changed, 35 insertions(+), 56 deletions(-) diff --git a/src/operators.jl b/src/operators.jl index 4b0807e71..542ed0cf8 100644 --- a/src/operators.jl +++ b/src/operators.jl @@ -13,23 +13,23 @@ struct Offset{D} end (::Offset{D})(α) where {D} = CartesianIndex(ntuple(β -> β == α ? 1 : 0, D)) """ - divergence!(M, u, setup) + divergence!(div, u, setup) Compute divergence of velocity field (in-place version). """ -function divergence!(M, u, setup) +function divergence!(div, u, setup) (; grid) = setup (; Δ, N, Ip) = grid D = length(u) δ = Offset{D}() - @kernel function _divergence!(M, u, I0) + @kernel function div!(div, u, I0) I = @index(Global, Cartesian) I = I + I0 - m = zero(eltype(M)) + d = zero(eltype(div)) for α = 1:D - m += (u[α][I] - u[α][I-δ(α)]) / Δ[α][I[α]] + d += (u[α][I] - u[α][I-δ(α)]) / Δ[α][I[α]] end - M[I] = m + div[I] = d end # All volumes have a right velocity # All volumes have a left velocity except the first one @@ -39,8 +39,8 @@ function divergence!(M, u, setup) # ndrange = Np # I0 = first(Ip) I0 -= oneunit(I0) - _divergence!(get_backend(M), WORKGROUP)(M, u, I0; ndrange) - M + div!(get_backend(div), WORKGROUP)(div, u, I0; ndrange) + div end """ @@ -82,7 +82,7 @@ function vorticity!(::Dimension{2}, ω, u, setup) (; dimension, Δu, N) = grid D = dimension() δ = Offset{D}() - @kernel function _vorticity!(ω, u, I0) + @kernel function ω!(ω, u, I0) I = @index(Global, Cartesian) I = I + I0 ω[I] = @@ -90,7 +90,7 @@ function vorticity!(::Dimension{2}, ω, u, setup) end I0 = CartesianIndex(ntuple(Returns(1), D)) I0 -= oneunit(I0) - _vorticity!(get_backend(ω), WORKGROUP)(ω, u, I0; ndrange = N .- 1) + ω!(get_backend(ω), WORKGROUP)(ω, u, I0; ndrange = N .- 1) ω end @@ -99,7 +99,7 @@ function vorticity!(::Dimension{3}, ω, u, setup) (; dimension, Δu, N) = grid D = dimension() δ = Offset{D}() - @kernel function _vorticity!(ω, u, I0) + @kernel function ω!(ω, u, I0) T = eltype(ω) I = @index(Global, Cartesian) I = I + I0 @@ -113,7 +113,7 @@ function vorticity!(::Dimension{3}, ω, u, setup) end I0 = CartesianIndex(ntuple(Returns(1), D)) I0 -= oneunit(I0) - _vorticity!(get_backend(ω[1]), WORKGROUP)(ω, u, I0; ndrange = N .- 1) + ω!(get_backend(ω[1]), WORKGROUP)(ω, u, I0; ndrange = N .- 1) ω end @@ -127,7 +127,7 @@ function convection!(F, u, setup) (; dimension, Δ, Δu, Nu, Iu, A) = grid D = dimension() δ = Offset{D}() - @kernel function _convection!(F, u, ::Val{α}, ::Val{βrange}, I0) where {α,βrange} + @kernel function conv!(F, u, ::Val{α}, ::Val{βrange}, I0) where {α,βrange} I = @index(Global, Cartesian) I = I + I0 # for β = 1:D @@ -144,14 +144,7 @@ function convection!(F, u, setup) for α = 1:D I0 = first(Iu[α]) I0 -= oneunit(I0) - _convection!(get_backend(F[1]), WORKGROUP)( - F, - u, - Val(α), - Val(1:D), - I0; - ndrange = Nu[α], - ) + conv!(get_backend(F[1]), WORKGROUP)(F, u, Val(α), Val(1:D), I0; ndrange = Nu[α]) end F end @@ -167,7 +160,7 @@ function diffusion!(F, u, setup) D = dimension() δ = Offset{D}() ν = 1 / Re - @kernel function _diffusion!(F, u, ::Val{α}, ::Val{βrange}, I0) where {α,βrange} + @kernel function diff!(F, u, ::Val{α}, ::Val{βrange}, I0) where {α,βrange} I = @index(Global, Cartesian) I = I + I0 # for β = 1:D @@ -183,14 +176,7 @@ function diffusion!(F, u, setup) for α = 1:D I0 = first(Iu[α]) I0 -= oneunit(I0) - _diffusion!(get_backend(F[1]), WORKGROUP)( - F, - u, - Val(α), - Val(1:D), - I0; - ndrange = Nu[α], - ) + diff!(get_backend(F[1]), WORKGROUP)(F, u, Val(α), Val(1:D), I0; ndrange = Nu[α]) end F end @@ -203,9 +189,10 @@ Compute body force. function bodyforce!(F, u, t, setup) (; grid, bodyforce) = setup (; dimension, Δ, Δu, Nu, Iu, x, xp) = grid + isnothing(bodyforce) && return F D = dimension() δ = Offset{D}() - @kernel function _bodyforce!(F, force, ::Val{α}, t, I0) where {α} + @kernel function f!(F, force, ::Val{α}, t, I0) where {α} I = @index(Global, Cartesian) I = I + I0 F[α][I] += @@ -214,14 +201,7 @@ function bodyforce!(F, u, t, setup) for α = 1:D I0 = first(Iu[α]) I0 -= oneunit(I0) - isnothing(bodyforce) || _bodyforce!(get_backend(F[1]), WORKGROUP)( - F, - bodyforce, - Val(α), - t, - I0; - ndrange = Nu[α], - ) + f!(get_backend(F[1]), WORKGROUP)(F, bodyforce, Val(α), t, I0; ndrange = Nu[α]) end F end @@ -268,7 +248,7 @@ function pressuregradient!(G, p, setup) (; dimension, Δu, Nu, Iu) = grid D = dimension() δ = Offset{D}() - @kernel function _pressuregradient!(G, p, ::Val{α}, I0) where {α} + @kernel function G!(G, p, ::Val{α}, I0) where {α} I = @index(Global, Cartesian) I = I0 + I G[α][I] = (p[I+δ(α)] - p[I]) / Δu[α][I[α]] @@ -277,7 +257,7 @@ function pressuregradient!(G, p, setup) for α = 1:D I0 = first(Iu[α]) I0 -= oneunit(I0) - _pressuregradient!(get_backend(G[1]), WORKGROUP)(G, p, Val(α), I0; ndrange = Nu[α]) + G!(get_backend(G[1]), WORKGROUP)(G, p, Val(α), I0; ndrange = Nu[α]) end G end @@ -306,7 +286,7 @@ function laplacian!(L, p, setup) (; dimension, Δ, Δu, N, Np, Ip, Ω) = grid D = dimension() δ = Offset{D}() - @kernel function _laplacian!(L, p, I0) + @kernel function lap!(L, p, I0) I = @index(Global, Cartesian) I = I + I0 lap = zero(eltype(p)) @@ -323,7 +303,7 @@ function laplacian!(L, p, setup) ndrange = Np I0 = first(Ip) I0 -= oneunit(I0) - _laplacian!(get_backend(L), WORKGROUP)(L, p, I0; ndrange) + lap!(get_backend(L), WORKGROUP)(L, p, I0; ndrange) L end @@ -442,7 +422,7 @@ function interpolate_u_p!(up, u, setup) (; dimension, Np, Ip) = grid D = dimension() δ = Offset{D}() - @kernel function _interpolate_u_p!(up, u, ::Val{α}, I0) where {α} + @kernel function int!(up, u, ::Val{α}, I0) where {α} I = @index(Global, Cartesian) I = I + I0 up[α][I] = (u[α][I-δ(α)] + u[α][I]) / 2 @@ -450,7 +430,7 @@ function interpolate_u_p!(up, u, setup) for α = 1:D I0 = first(Ip) I0 -= oneunit(I0) - _interpolate_u_p!(get_backend(up[1]), WORKGROUP)(up, u, Val(α), I0; ndrange = Np) + int!(get_backend(up[1]), WORKGROUP)(up, u, Val(α), I0; ndrange = Np) end up end @@ -483,14 +463,14 @@ function interpolate_ω_p!(::Dimension{2}, ωp, ω, setup) (; dimension, Np, Ip) = grid D = dimension() δ = Offset{D}() - @kernel function _interpolate_ω_p!(ωp, ω, I0) + @kernel function int!(ωp, ω, I0) I = @index(Global, Cartesian) I = I + I0 ωp[I] = (ω[I-δ(1)-δ(2)] + ω[I]) / 2 end I0 = first(Ip) I0 -= oneunit(I0) - _interpolate_ω_p!(get_backend(ωp), WORKGROUP)(ωp, ω, I0; ndrange = Np) + int!(get_backend(ωp), WORKGROUP)(ωp, ω, I0; ndrange = Np) ωp end @@ -499,7 +479,7 @@ function interpolate_ω_p!(::Dimension{3}, ωp, ω, setup) (; dimension, Np, Ip) = grid D = dimension() δ = Offset{D}() - @kernel function _interpolate_ω_p!(ωp, ω, ::Val{α}, I0) where {α} + @kernel function int!(ωp, ω, ::Val{α}, I0) where {α} I = @index(Global, Cartesian) I = I + I0 α₊ = mod1(α + 1, D) @@ -509,7 +489,7 @@ function interpolate_ω_p!(::Dimension{3}, ωp, ω, setup) I0 = first(Ip) I0 -= oneunit(I0) for α = 1:D - _interpolate_ω_p!(get_backend(ωp[1]), WORKGROUP)(ωp, ω, Val(α), I0; ndrange = Np) + int!(get_backend(ωp[1]), WORKGROUP)(ωp, ω, Val(α), I0; ndrange = Np) end ωp end @@ -529,7 +509,7 @@ function Dfield!(d, G, p, setup; ϵ = eps(eltype(p))) T = eltype(p) D = dimension() δ = Offset{D}() - @kernel function _Dfield!(d, G, p, I0) + @kernel function D!(d, G, p, I0) I = @index(Global, Cartesian) I = I + I0 g = zero(eltype(p)) @@ -555,7 +535,7 @@ function Dfield!(d, G, p, setup; ϵ = eps(eltype(p))) pressuregradient!(G, p, setup) I0 = first(Ip) I0 -= oneunit(I0) - _Dfield!(get_backend(p), WORKGROUP)(d, G, p, I0; ndrange = Np) + D!(get_backend(p), WORKGROUP)(d, G, p, I0; ndrange = Np) d end @@ -589,7 +569,7 @@ function Qfield!(Q, u, setup; ϵ = eps(eltype(Q))) (; dimension, Np, Ip, Δ) = grid D = dimension() δ = Offset{D}() - @kernel function _Qfield!(Q, u, I0) + @kernel function Q!(Q, u, I0) I = @index(Global, Cartesian) I = I + I0 q = zero(eltype(Q)) @@ -602,7 +582,7 @@ function Qfield!(Q, u, setup; ϵ = eps(eltype(Q))) end I0 = first(Ip) I0 -= oneunit(I0) - _Qfield!(get_backend(u[1]), WORKGROUP)(Q, u, I0; ndrange = Np) + Q!(get_backend(u[1]), WORKGROUP)(Q, u, I0; ndrange = Np) Q end @@ -628,7 +608,7 @@ function eig2field!(λ, u, setup) D = dimension() δ = Offset{D}() @assert D == 3 "eig2 only implemented in 3D" - @kernel function _eig2field!(λ, u, I0) + @kernel function λ!(λ, u, I0) I = @index(Global, Cartesian) I = I + I0 ∂x(uα, I, ::Val{α}, ::Val{β}, Δβ, Δuβ) where {α,β} = @@ -656,8 +636,7 @@ function eig2field!(λ, u, setup) end I0 = first(Ip) I0 -= oneunit(I0) - _eig2field!(get_backend(u[1]), WORKGROUP)(λ, u, I0; ndrange = Np) - # synchronize(get_backend(u[1])) + λ!(get_backend(u[1]), WORKGROUP)(λ, u, I0; ndrange = Np) λ end From dd594e16a8aa6ba4e66bf3d53b3dff0a208a315b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Mon, 27 Nov 2023 16:36:34 +0100 Subject: [PATCH 150/379] Siplify zeros --- src/operators.jl | 54 ++++++++++-------------------------------------- 1 file changed, 11 insertions(+), 43 deletions(-) diff --git a/src/operators.jl b/src/operators.jl index 542ed0cf8..dd90f066b 100644 --- a/src/operators.jl +++ b/src/operators.jl @@ -48,11 +48,7 @@ end Compute divergence of velocity field. """ -divergence(u, setup) = divergence!( - KernelAbstractions.zeros(get_backend(u[1]), eltype(u[1]), setup.grid.N...), - u, - setup, -) +divergence(u, setup) = divergence!(similar(u[1], setup.grid.N), u, setup) """ vorticity(u, setup) @@ -60,12 +56,8 @@ divergence(u, setup) = divergence!( Compute vorticity field. """ vorticity(u, setup) = vorticity!( - setup.grid.dimension() == 2 ? - KernelAbstractions.zeros(get_backend(u[1]), eltype(u[1]), setup.grid.N) : - ntuple( - α -> KernelAbstractions.zeros(get_backend(u[1]), eltype(u[1]), setup.grid.N), - setup.grid.dimension(), - ), + length(u) == 2 ? similar(u[1], setup.grid.N) : + ntuple(α -> similar(u[1], setup.grid.N), length(u)), u, setup, ) @@ -268,10 +260,7 @@ end Compute pressure gradient. """ pressuregradient(p, setup) = pressuregradient!( - ntuple( - α -> KernelAbstractions.zeros(get_backend(p), eltype(p), setup.grid.N), - setup.grid.dimension(), - ), + ntuple(α -> similar(p, setup.grid.N), setup.grid.dimension()), p, setup, ) @@ -403,14 +392,8 @@ end Interpolate velocity to pressure points. """ -interpolate_u_p(u, setup) = interpolate_u_p!( - ntuple( - α -> KernelAbstractions.zeros(get_backend(u[1]), eltype(u[1]), setup.grid.N), - setup.grid.dimension(), - ), - u, - setup, -) +interpolate_u_p(u, setup) = + interpolate_u_p!(ntuple(α -> similar(u[1], setup.grid.N), length(u)), u, setup) """ interpolate_u_p!(up, u, setup) @@ -441,12 +424,8 @@ end Interpolate vorticity to pressure points. """ interpolate_ω_p(ω, setup) = interpolate_ω_p!( - setup.grid.dimension() == 2 ? - KernelAbstractions.zeros(get_backend(ω), eltype(ω), setup.grid.N) : - ntuple( - α -> KernelAbstractions.zeros(get_backend(ω[1]), eltype(ω[1]), setup.grid.N), - setup.grid.dimension(), - ), + setup.grid.dimension() == 2 ? similar(ω, setup.grid.N) : + ntuple(α -> similar(ω[1], setup.grid.N), length(ω)), ω, setup, ) @@ -546,10 +525,7 @@ Compute the ``D``-field. """ Dfield(p, setup) = Dfield!( zero(p), - ntuple( - α -> KernelAbstractions.zeros(get_backend(p), eltype(p), setup.grid.N), - setup.grid.dimension(), - ), + ntuple(α -> similar(p, setup.grid.N), setup.grid.dimension()), p, setup, ) @@ -591,11 +567,7 @@ end Compute the ``Q``-field. """ -Qfield(u, setup) = Qfield!( - KernelAbstractions.zeros(get_backend(u[1]), eltype(u[1]), setup.grid.N), - u, - setup, -) +Qfield(u, setup) = Qfield!(similar(u[1], setup.grid.N), u, setup) """ eig2field!(λ, u, setup) @@ -645,11 +617,7 @@ end Compute the second eigenvalue of ``S^2 + \\Omega^2``. """ -eig2field(u, setup) = eig2field!( - KernelAbstractions.zeros(get_backend(u[1]), eltype(u[1]), setup.grid.N), - u, - setup, -) +eig2field(u, setup) = eig2field!(similar(u[1], setup.grid.N), u, setup) """ kinetic_energy(setup, u) From 1ce8839fefd9538ce667a5e09618f9acd42158ba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Mon, 27 Nov 2023 16:36:54 +0100 Subject: [PATCH 151/379] Remove sqrt from kinetic energy --- src/operators.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/operators.jl b/src/operators.jl index dd90f066b..e257b01e1 100644 --- a/src/operators.jl +++ b/src/operators.jl @@ -634,5 +634,5 @@ function kinetic_energy(u, setup) # E += sum(I -> Ω[I] * up[α][I]^2, Ip) E += sum(Ω[Ip] .* up[α][Ip] .^ 2) end - sqrt(E) + E end From 2201e4fca01989af2e0235b96594dce073bfb18d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Mon, 27 Nov 2023 16:41:06 +0100 Subject: [PATCH 152/379] Project by default --- src/create_initial_conditions.jl | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/src/create_initial_conditions.jl b/src/create_initial_conditions.jl index 866c7ee67..3798c6599 100644 --- a/src/create_initial_conditions.jl +++ b/src/create_initial_conditions.jl @@ -39,16 +39,8 @@ function create_initial_conditions( apply_bc_u!(u, t, setup) - # Kinetic energy and momentum of initial velocity field - # Iteration 1 corresponds to t₀ = 0 (for unsteady simulations) - maxdiv = maximum(abs, divergence(u, setup)) - - # TODO: Maybe eps(T)^(3//4) - if project && maxdiv > 1e-12 - @warn "Initial velocity field not (discretely) divergence free: $maxdiv.\n" * - "Performing additional projection." - - # Make velocity field divergence free + # Make velocity field divergence free + if project f = divergence(u, setup) @. f *= Ω Δp = pressure_poisson(pressure_solver, f) From 27a22e2e56b089033ff3836d37ccc574112b8191 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Mon, 27 Nov 2023 16:49:37 +0100 Subject: [PATCH 153/379] Rename function --- docs/src/features/pressure.md | 2 +- src/IncompressibleNavierStokes.jl | 2 +- src/closures/create_les_data.jl | 2 +- src/create_initial_conditions.jl | 2 +- .../pressure/pressure_additional_solve.jl | 2 +- src/solvers/pressure/pressure_poisson.jl | 22 +++++++++---------- src/time_steppers/step_ab_cn.jl | 2 +- .../step_explicit_runge_kutta.jl | 2 +- .../step_implicit_runge_kutta.jl | 2 +- src/time_steppers/step_one_leg.jl | 2 +- test/pressure_solvers.jl | 2 +- 11 files changed, 21 insertions(+), 21 deletions(-) diff --git a/docs/src/features/pressure.md b/docs/src/features/pressure.md index ebbc7161b..a3f26ebfa 100644 --- a/docs/src/features/pressure.md +++ b/docs/src/features/pressure.md @@ -19,6 +19,6 @@ CGPressureSolverManual SpectralPressureSolver pressure_additional_solve pressure_additional_solve! -pressure_poisson +poisson pressure_poisson! ``` diff --git a/src/IncompressibleNavierStokes.jl b/src/IncompressibleNavierStokes.jl index 353e31162..f08ef558f 100644 --- a/src/IncompressibleNavierStokes.jl +++ b/src/IncompressibleNavierStokes.jl @@ -52,7 +52,7 @@ include("setup.jl") # Pressure solvers include("solvers/pressure/pressure_solvers.jl") -include("solvers/pressure/pressure_poisson.jl") +include("solvers/pressure/poisson.jl") include("solvers/pressure/pressure_additional_solve.jl") # Time steppers diff --git a/src/closures/create_les_data.jl b/src/closures/create_les_data.jl index 4efc9f7f0..49dc85df3 100644 --- a/src/closures/create_les_data.jl +++ b/src/closures/create_les_data.jl @@ -63,7 +63,7 @@ _filter_saver(dns, les, comp, pressure_solver; nupdate = 1) = divergence!(M, FΦ, les) @. M *= les.grid.Ω - pressure_poisson!(pressure_solver, q, M) + poisson!(pressure_solver, q, M) apply_bc_p!(q, t, les) pressuregradient!(GΦ, q, les) for α = 1:D diff --git a/src/create_initial_conditions.jl b/src/create_initial_conditions.jl index 3798c6599..13e024993 100644 --- a/src/create_initial_conditions.jl +++ b/src/create_initial_conditions.jl @@ -43,7 +43,7 @@ function create_initial_conditions( if project f = divergence(u, setup) @. f *= Ω - Δp = pressure_poisson(pressure_solver, f) + Δp = poisson(pressure_solver, f) p .= Δp apply_bc_p!(p, t, setup) G = pressuregradient(p, setup) diff --git a/src/solvers/pressure/pressure_additional_solve.jl b/src/solvers/pressure/pressure_additional_solve.jl index 046de7e29..12b745e87 100644 --- a/src/solvers/pressure/pressure_additional_solve.jl +++ b/src/solvers/pressure/pressure_additional_solve.jl @@ -22,7 +22,7 @@ function pressure_additional_solve!(pressure_solver, u, p, t, setup, F, G, M) divergence!(M, F, setup) @. M *= Ω - pressure_poisson!(pressure_solver, p, M) + poisson!(pressure_solver, p, M) # dp = pressure_poisson(pressure_solver, M) # p .+= dp apply_bc_p!(p, t, setup) diff --git a/src/solvers/pressure/pressure_poisson.jl b/src/solvers/pressure/pressure_poisson.jl index e19353555..b2a844d06 100644 --- a/src/solvers/pressure/pressure_poisson.jl +++ b/src/solvers/pressure/pressure_poisson.jl @@ -1,30 +1,30 @@ """ - pressure_poisson(solver, f) + poisson(solver, f) Solve the Poisson equation for the pressure with right hand side `f` at time `t`. For periodic and no-slip BC, the sum of `f` should be zero. Non-mutating/allocating/out-of-place version. -See also [`pressure_poisson!`](@ref). +See also [`poisson!`](@ref). """ -function pressure_poisson end +function poisson end -pressure_poisson(solver, f) = pressure_poisson!(solver, zero(f), f) +poisson(solver, f) = poisson!(solver, zero(f), f) """ - pressure_poisson!(solver, p, f) + poisson!(solver, p, f) Solve the Poisson equation for the pressure with right hand side `f` at time `t`. For periodic and no-slip BC, the sum of `f` should be zero. Mutating/non-allocating/in-place version. -See also [`pressure_poisson`](@ref). +See also [`poisson`](@ref). """ -function pressure_poisson! end +function poisson! end -function pressure_poisson!(solver::DirectPressureSolver, p, f) +function poisson!(solver::DirectPressureSolver, p, f) (; setup, fact) = solver (; Ip) = setup.grid T = eltype(p) @@ -41,7 +41,7 @@ function pressure_poisson!(solver::DirectPressureSolver, p, f) p end -function pressure_poisson!(solver::CGPressureSolver, p, f) +function poisson!(solver::CGPressureSolver, p, f) (; A, abstol, reltol, maxiter) = solver f = view(f, :) p = view(p, :) @@ -58,7 +58,7 @@ end # # instead. This way, the matrix is still positive definite. # For initial guess, we already know the average is zero. -function pressure_poisson!(solver::CGPressureSolverManual, p, f) +function poisson!(solver::CGPressureSolverManual, p, f) (; setup, abstol, reltol, maxiter, r, L, q, preconditioner) = solver (; Np, Ip, Ω) = setup.grid T = typeof(reltol) @@ -131,7 +131,7 @@ function pressure_poisson!(solver::CGPressureSolverManual, p, f) p end -function pressure_poisson!(solver::SpectralPressureSolver, p, f) +function poisson!(solver::SpectralPressureSolver, p, f) (; setup, plan, Ahat, fhat, phat) = solver (; Ip) = setup.grid diff --git a/src/time_steppers/step_ab_cn.jl b/src/time_steppers/step_ab_cn.jl index f7c23b616..26d6651c0 100644 --- a/src/time_steppers/step_ab_cn.jl +++ b/src/time_steppers/step_ab_cn.jl @@ -129,7 +129,7 @@ function timestep(method::AdamsBashforthCrankNicolsonMethod, stepper, Δt) f = (M * V + yM) / Δt - M * y_Δp # Solve the Poisson equation for the pressure - Δp = pressure_poisson(pressure_solver, f) + Δp = poisson(pressure_solver, f) # Update velocity field V -= Δt ./ Ω .* (G * Δp .+ y_Δp) diff --git a/src/time_steppers/step_explicit_runge_kutta.jl b/src/time_steppers/step_explicit_runge_kutta.jl index a35f72988..60495f790 100644 --- a/src/time_steppers/step_explicit_runge_kutta.jl +++ b/src/time_steppers/step_explicit_runge_kutta.jl @@ -55,7 +55,7 @@ function timestep!(method::ExplicitRungeKuttaMethod, stepper, Δt; cache) @. M *= Ω / (c[i] * Δt) # Solve the Poisson equation - pressure_poisson!(pressure_solver, p, M) + poisson!(pressure_solver, p, M) # Compute pressure correction term apply_bc_p!(p, t, setup) diff --git a/src/time_steppers/step_implicit_runge_kutta.jl b/src/time_steppers/step_implicit_runge_kutta.jl index fe5d0478c..718a293a6 100644 --- a/src/time_steppers/step_implicit_runge_kutta.jl +++ b/src/time_steppers/step_implicit_runge_kutta.jl @@ -142,7 +142,7 @@ function timestep(method::ImplicitRungeKuttaMethod, stepper, Δt) (; yM) = bc_vectors f = 1 / Δtₙ .* (M * V .+ yM) - p = pressure_poisson!(pressure_solver, f) + p = poisson!(pressure_solver, f) mul!(Gp, G, p) V = @. V - Δtₙ / Ω * Gp diff --git a/src/time_steppers/step_one_leg.jl b/src/time_steppers/step_one_leg.jl index 2fb305f15..9c0e1a9e9 100644 --- a/src/time_steppers/step_one_leg.jl +++ b/src/time_steppers/step_one_leg.jl @@ -86,7 +86,7 @@ function timestep(method::OneLegMethod, stepper, Δt) f = (M * V + yM) / Δtᵦ # Solve the Poisson equation for the pressure - Δp = pressure_poisson(pressure_solver, f) + Δp = poisson(pressure_solver, f) GΔp = G * Δp # Update velocity field diff --git a/test/pressure_solvers.jl b/test/pressure_solvers.jl index 88988caf9..4ed21f08c 100644 --- a/test/pressure_solvers.jl +++ b/test/pressure_solvers.jl @@ -15,7 +15,7 @@ p_exact = reshape(initial_pressure.(setup.grid.xpp, setup.grid.ypp), :) f = A * p_exact - p_direct = pressure_poisson(direct, f) + p_direct = poisson(direct, f) p_cg = pressure_poisson(cg, f) p_spectral = pressure_poisson(spectral, f) From 80752cbf7f5370ab63b2b7b00f8f57e01fe7c7ca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Mon, 27 Nov 2023 16:53:08 +0100 Subject: [PATCH 154/379] Rename files --- src/IncompressibleNavierStokes.jl | 4 ++-- src/solvers/pressure/{pressure_poisson.jl => poisson.jl} | 0 .../pressure/{pressure_additional_solve.jl => pressure.jl} | 0 src/solvers/pressure/{pressure_solvers.jl => solvers.jl} | 0 4 files changed, 2 insertions(+), 2 deletions(-) rename src/solvers/pressure/{pressure_poisson.jl => poisson.jl} (100%) rename src/solvers/pressure/{pressure_additional_solve.jl => pressure.jl} (100%) rename src/solvers/pressure/{pressure_solvers.jl => solvers.jl} (100%) diff --git a/src/IncompressibleNavierStokes.jl b/src/IncompressibleNavierStokes.jl index f08ef558f..46733dc54 100644 --- a/src/IncompressibleNavierStokes.jl +++ b/src/IncompressibleNavierStokes.jl @@ -51,9 +51,9 @@ include("models/viscosity_models.jl") include("setup.jl") # Pressure solvers -include("solvers/pressure/pressure_solvers.jl") +include("solvers/pressure/solvers.jl") include("solvers/pressure/poisson.jl") -include("solvers/pressure/pressure_additional_solve.jl") +include("solvers/pressure/pressure.jl") # Time steppers include("time_steppers/methods.jl") diff --git a/src/solvers/pressure/pressure_poisson.jl b/src/solvers/pressure/poisson.jl similarity index 100% rename from src/solvers/pressure/pressure_poisson.jl rename to src/solvers/pressure/poisson.jl diff --git a/src/solvers/pressure/pressure_additional_solve.jl b/src/solvers/pressure/pressure.jl similarity index 100% rename from src/solvers/pressure/pressure_additional_solve.jl rename to src/solvers/pressure/pressure.jl diff --git a/src/solvers/pressure/pressure_solvers.jl b/src/solvers/pressure/solvers.jl similarity index 100% rename from src/solvers/pressure/pressure_solvers.jl rename to src/solvers/pressure/solvers.jl From c7cf2627199d2249b7fdaf9626bfdb38e0e3181a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Mon, 27 Nov 2023 16:57:50 +0100 Subject: [PATCH 155/379] Rename function --- docs/src/features/pressure.md | 6 +++--- scratch/multigrid.jl | 2 +- scratch/train_model.jl | 2 +- src/closures/create_les_data.jl | 2 +- src/create_initial_conditions.jl | 6 +++--- src/solvers/pressure/pressure.jl | 14 +++++++------- src/time_steppers/step_ab_cn.jl | 6 +++--- src/time_steppers/step_explicit_runge_kutta.jl | 2 +- src/time_steppers/step_implicit_runge_kutta.jl | 10 +++++----- src/time_steppers/step_one_leg.jl | 6 +++--- test/pressure_solvers.jl | 2 +- 11 files changed, 29 insertions(+), 29 deletions(-) diff --git a/docs/src/features/pressure.md b/docs/src/features/pressure.md index a3f26ebfa..f3cb0bc61 100644 --- a/docs/src/features/pressure.md +++ b/docs/src/features/pressure.md @@ -17,8 +17,8 @@ DirectPressureSolver CGPressureSolver CGPressureSolverManual SpectralPressureSolver -pressure_additional_solve -pressure_additional_solve! +pressure +pressure! poisson -pressure_poisson! +poisson! ``` diff --git a/scratch/multigrid.jl b/scratch/multigrid.jl index cfc956c68..960ccd339 100644 --- a/scratch/multigrid.jl +++ b/scratch/multigrid.jl @@ -103,7 +103,7 @@ for (i, n) in enumerate(ntest) pressure_solver = SpectralPressureSolver(setup) u = device.(data_test[i].u[1]) u₀ = device(data_test[i].u[1][1]) - p₀ = pressure_additional_solve(pressure_solver, u₀, T(0), setup) + p₀ = pressure(pressure_solver, u₀, T(0), setup) tlims = (T(0), params.tsim) (; Δt) = data_test[i] processors = (; relerr = relerr_track(u, setup)) diff --git a/scratch/train_model.jl b/scratch/train_model.jl index c36b90b84..eab273bd6 100644 --- a/scratch/train_model.jl +++ b/scratch/train_model.jl @@ -197,7 +197,7 @@ relerr_track(uref, setup) = u, u₀, p₀ = nothing, nothing, nothing u = device.(data_test.u[1]) u₀ = device(data_test.u[1][1]) -p₀ = pressure_additional_solve(pressure_solver, u₀, T(0), setup) +p₀ = pressure(pressure_solver, u₀, T(0), setup) length(u) u_nm, p_nm, outputs = solve_unsteady( diff --git a/src/closures/create_les_data.jl b/src/closures/create_les_data.jl index 49dc85df3..69b5d54bc 100644 --- a/src/closures/create_les_data.jl +++ b/src/closures/create_les_data.jl @@ -40,7 +40,7 @@ _filter_saver(dns, les, comp, pressure_solver; nupdate = 1) = F = zero.(state[].u) G = zero.(state[].u) Φu = zero.(face_average(state[].u, les, comp)) - q = zero(pressure_additional_solve(pressure_solver, Φu, state[].t, les)) + q = zero(pressure(pressure_solver, Φu, state[].t, les)) M = zero(q) ΦF = zero.(Φu) FΦ = zero.(Φu) diff --git a/src/create_initial_conditions.jl b/src/create_initial_conditions.jl index 13e024993..8de19a533 100644 --- a/src/create_initial_conditions.jl +++ b/src/create_initial_conditions.jl @@ -52,7 +52,7 @@ function create_initial_conditions( end end - p = pressure_additional_solve(pressure_solver, u, t, setup) + p = pressure(pressure_solver, u, t, setup) apply_bc_p!(p, t, setup) # Initial conditions, including initial boundary condititions @@ -121,14 +121,14 @@ function random_field( p = zero(M) # Make velocity field divergence free - pressure_poisson!(pressure_solver, p, M) + poisson!(pressure_solver, p, M) apply_bc_p!(p, t, setup) G = pressuregradient(p, setup) for α = 1:D @. u[α] -= G[α] end apply_bc_u!(u, t, setup) - p = pressure_additional_solve(pressure_solver, u, t, setup) + p = pressure(pressure_solver, u, t, setup) apply_bc_p!(p, t, setup) u, p diff --git a/src/solvers/pressure/pressure.jl b/src/solvers/pressure/pressure.jl index 12b745e87..c011947db 100644 --- a/src/solvers/pressure/pressure.jl +++ b/src/solvers/pressure/pressure.jl @@ -1,10 +1,10 @@ """ - pressure_additional_solve!(pressure_solver, u, p, t, setup, F, f, M) + pressure!(pressure_solver, u, p, t, setup, F, f, M) -Do additional pressure solve. This makes the pressure compatible with the velocity +Compute pressure from velocity field. This makes the pressure compatible with the velocity field, resulting in same order pressure as velocity. """ -function pressure_additional_solve!(pressure_solver, u, p, t, setup, F, G, M) +function pressure!(pressure_solver, u, p, t, setup, F, G, M) (; grid) = setup (; dimension, Iu, Ip, Ω) = grid D = dimension() @@ -23,19 +23,19 @@ function pressure_additional_solve!(pressure_solver, u, p, t, setup, F, G, M) @. M *= Ω poisson!(pressure_solver, p, M) - # dp = pressure_poisson(pressure_solver, M) + # dp = poisson(pressure_solver, M) # p .+= dp apply_bc_p!(p, t, setup) p end """ - pressure_additional_solve(pressure_solver, u, t, setup) + pressure(pressure_solver, u, t, setup) Do additional pressure solve. This makes the pressure compatible with the velocity field, resulting in same order pressure as velocity. """ -function pressure_additional_solve(pressure_solver, u, t, setup) +function pressure(pressure_solver, u, t, setup) D = setup.grid.dimension() p = KernelAbstractions.zeros(get_backend(u[1]), eltype(u[1]), setup.grid.N) F = ntuple( @@ -47,5 +47,5 @@ function pressure_additional_solve(pressure_solver, u, t, setup) D, ) M = KernelAbstractions.zeros(get_backend(u[1]), eltype(u[1]), setup.grid.N) - pressure_additional_solve!(pressure_solver, u, p, t, setup, F, G, M) + pressure!(pressure_solver, u, p, t, setup, F, G, M) end diff --git a/src/time_steppers/step_ab_cn.jl b/src/time_steppers/step_ab_cn.jl index 26d6651c0..9d1e4e605 100644 --- a/src/time_steppers/step_ab_cn.jl +++ b/src/time_steppers/step_ab_cn.jl @@ -138,7 +138,7 @@ function timestep(method::AdamsBashforthCrankNicolsonMethod, stepper, Δt) p = pₙ .+ Δp if p_add_solve - p = pressure_additional_solve(pressure_solver, V, p, tₙ + Δt, setup; bc_vectors) + p = pressure(pressure_solver, V, p, tₙ + Δt, setup; bc_vectors) end t = tₙ + Δtₙ @@ -283,7 +283,7 @@ function timestep!( f = (M * V + yM) / Δt - M * y_Δp # Solve the Poisson equation for the pressure - pressure_poisson!(pressure_solver, Δp, f) + poisson!(pressure_solver, Δp, f) # Update velocity field V .-= Δt ./ Ω .* (G * Δp .+ y_Δp) @@ -292,7 +292,7 @@ function timestep!( p .= pₙ .+ Δp if p_add_solve - pressure_additional_solve!( + pressure!( pressure_solver, V, p, diff --git a/src/time_steppers/step_explicit_runge_kutta.jl b/src/time_steppers/step_explicit_runge_kutta.jl index 60495f790..fce21437d 100644 --- a/src/time_steppers/step_explicit_runge_kutta.jl +++ b/src/time_steppers/step_explicit_runge_kutta.jl @@ -73,7 +73,7 @@ function timestep!(method::ExplicitRungeKuttaMethod, stepper, Δt; cache) t = t₀ + Δt # Do additional pressure solve to avoid first order pressure - p_add_solve && pressure_additional_solve!(pressure_solver, u, p, t, setup, F, G, M) + p_add_solve && pressure!(pressure_solver, u, p, t, setup, F, G, M) create_stepper(method; setup, pressure_solver, u, p, t, n = n + 1) end diff --git a/src/time_steppers/step_implicit_runge_kutta.jl b/src/time_steppers/step_implicit_runge_kutta.jl index 718a293a6..a1c32231e 100644 --- a/src/time_steppers/step_implicit_runge_kutta.jl +++ b/src/time_steppers/step_implicit_runge_kutta.jl @@ -148,7 +148,7 @@ function timestep(method::ImplicitRungeKuttaMethod, stepper, Δt) V = @. V - Δtₙ / Ω * Gp if p_add_solve - p = pressure_additional_solve( + p = pressure( pressure_solver, V, p, @@ -163,7 +163,7 @@ function timestep(method::ImplicitRungeKuttaMethod, stepper, Δt) else # For steady BC we do an additional pressure solve # That saves a pressure solve for iter = 1 in the next time step - # pressure_additional_solve!(pressure_solver, V, p, tₙ + Δtₙ, setup, momentum_cache, F, f, Δp) + # pressure!(pressure_solver, V, p, tₙ + Δtₙ, setup, momentum_cache, F, f, Δp) # Standard method; take pressure of last stage p = pⱼ[(end-Np+1):end] @@ -372,13 +372,13 @@ function timestep!(method::ImplicitRungeKuttaMethod, stepper, Δt; cache, moment f .= yM mul!(f, M, V, 1 / Δtₙ, 1 / Δtₙ) # f .= 1 / Δtₙ .* (M * V .+ yM) - pressure_poisson!(pressure_solver, p, f) + poisson!(pressure_solver, p, f) mul!(Gp, G, p) @. V -= Δtₙ / Ω * Gp if p_add_solve - pressure_additional_solve!( + pressure!( pressure_solver, V, p, @@ -397,7 +397,7 @@ function timestep!(method::ImplicitRungeKuttaMethod, stepper, Δt; cache, moment else # For steady BC we do an additional pressure solve # That saves a pressure solve for iter = 1 in the next time step - # pressure_additional_solve!(pressure_solver, V, p, tₙ + Δtₙ, setup, momentum_cache, F, f, Δp) + # pressure!(pressure_solver, V, p, tₙ + Δtₙ, setup, momentum_cache, F, f, Δp) # Standard method; take pressure of last stage p .= pⱼ[(end-Np+1):end] diff --git a/src/time_steppers/step_one_leg.jl b/src/time_steppers/step_one_leg.jl index 9c0e1a9e9..81c313b76 100644 --- a/src/time_steppers/step_one_leg.jl +++ b/src/time_steppers/step_one_leg.jl @@ -97,7 +97,7 @@ function timestep(method::OneLegMethod, stepper, Δt) # Alternatively, do an additional Poisson solve if p_add_solve - p = pressure_additional_solve(pressure_solver, V, p, tₙ + Δtₙ, setup; bc_vectors) + p = pressure(pressure_solver, V, p, tₙ + Δtₙ, setup; bc_vectors) end t = tₙ + Δtₙ @@ -182,7 +182,7 @@ function timestep!(method::OneLegMethod, stepper, Δt; cache, momentum_cache) # f .= (M * V + yM) / Δtᵦ # Solve the Poisson equation for the pressure - pressure_poisson!(pressure_solver, Δp, f) + poisson!(pressure_solver, Δp, f) mul!(GΔp, G, Δp) # Update velocity field @@ -193,7 +193,7 @@ function timestep!(method::OneLegMethod, stepper, Δt; cache, momentum_cache) # Alternatively, do an additional Poisson solve if p_add_solve - pressure_additional_solve!( + pressure!( pressure_solver, V, p, diff --git a/test/pressure_solvers.jl b/test/pressure_solvers.jl index 4ed21f08c..3e1923ee5 100644 --- a/test/pressure_solvers.jl +++ b/test/pressure_solvers.jl @@ -16,7 +16,7 @@ f = A * p_exact p_direct = poisson(direct, f) - p_cg = pressure_poisson(cg, f) + p_cg = poisson(cg, f) p_spectral = pressure_poisson(spectral, f) # Test that in-place and out-of-place versions give same result From 17d2e274e4b109d562d412a589b9c3aa563d55c8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Mon, 27 Nov 2023 17:03:24 +0100 Subject: [PATCH 156/379] Update files --- .gitignore | 2 +- examples/Project.toml | 10 --------- scratch/censor_sparse.jl | 46 ---------------------------------------- scratch/multigrid.jl | 15 +++++++------ scratch/train_model.jl | 12 ++++++----- 5 files changed, 17 insertions(+), 68 deletions(-) delete mode 100644 examples/Project.toml delete mode 100644 scratch/censor_sparse.jl diff --git a/.gitignore b/.gitignore index f30000905..60eb6c8dd 100644 --- a/.gitignore +++ b/.gitignore @@ -5,7 +5,7 @@ /docs/build/ /docs/src/generated/ *output/ -# scratch/* +scratch/* test.jl toto.jl tata.jl diff --git a/examples/Project.toml b/examples/Project.toml deleted file mode 100644 index a16800d51..000000000 --- a/examples/Project.toml +++ /dev/null @@ -1,10 +0,0 @@ -[deps] -CairoMakie = "13f3f980-e62b-5c42-98c6-ff1f3baf88f0" -FFTW = "7a1cc6ca-52ef-59f5-83cd-3a7055c09341" -GLMakie = "e9467ef8-e4e7-5192-8a1a-b1aee30e663a" -IncompressibleNavierStokes = "5e318141-6589-402b-868d-77d7df8c442e" -JLD2 = "033835bb-8acc-5ee8-8aae-3f567f8a3819" -LaTeXStrings = "b964fa9f-0449-5b57-a5c2-d3ea65f4040f" -Observables = "510215fc-4207-5dde-b226-833fc4488ee2" -Optimisers = "3bd65402-5787-11e9-1adc-39752487f4e2" -Zygote = "e88e6eb3-aa80-5325-afca-941959d7151f" diff --git a/scratch/censor_sparse.jl b/scratch/censor_sparse.jl deleted file mode 100644 index f97a2f8f4..000000000 --- a/scratch/censor_sparse.jl +++ /dev/null @@ -1,46 +0,0 @@ -using SparseArrays -using LinearAlgebra - -# Hide long sparse arrays -function Base.show(io::IO, _S::SparseArrays.AbstractSparseMatrixCSCInclAdjointAndTranspose) - SparseArrays._checkbuffers(_S) - # can't use `findnz`, because that expects all values not to be #undef - S = _S isa Adjoint || _S isa Transpose ? _S.parent : _S - I = rowvals(S) - K = nonzeros(S) - m, n = size(S) - if _S isa Adjoint - print(io, "adjoint(") - elseif _S isa Transpose - print(io, "transpose(") - end - nn = nnz(S) - if nn > 20 - print(io, "sparse()") - if _S isa Adjoint || _S isa Transpose - print(io, ")") - end - return - end - print(io, "sparse(", I, ", ") - if length(I) == 0 - print(io, eltype(SparseArrays.getcolptr(S)), "[]") - else - print(io, "[") - il = nnz(S) - 1 - for col = 1:size(S, 2), - k = SparseArrays.getcolptr(S)[col]:(SparseArrays.getcolptr(S)[col+1]-1) - - print(io, col) - if il > 0 - print(io, ", ") - il -= 1 - end - end - print(io, "]") - end - print(io, ", ", K, ", ", m, ", ", n, ")") - if _S isa Adjoint || _S isa Transpose - print(io, ")") - end -end diff --git a/scratch/multigrid.jl b/scratch/multigrid.jl index 960ccd339..c3c9019e1 100644 --- a/scratch/multigrid.jl +++ b/scratch/multigrid.jl @@ -20,6 +20,8 @@ using Optimisers using Random using Zygote +set_theme!(; GLMakie = (; scalefactor = 1.5)) + # Floating point precision T = Float64 @@ -58,6 +60,7 @@ data_train = [create_les_data(T; get_params(nles)..., nsim = 5) for nles in [32, data_valid = [create_les_data(T; get_params(nles)..., nsim = 1) for nles in [128]]; ntest = [8, 16, 32, 64, 128, 256, 512] data_test = [create_les_data(T; get_params(nles)..., nsim = 1) for nles in ntest]; +data_train = data_test # Inspect data g = 4 @@ -122,9 +125,10 @@ CairoMakie.activate!() # Plot convergence with_theme(; -# linewidth = 5, -# markersize = 20, -# fontsize = 20, + # linewidth = 5, + # markersize = 20, + # fontsize = 20, + palette = (; color = ["#3366cc", "#cc0000", "#669900", "#ffcc00"]), ) do fig = Figure() ax = Axis( @@ -137,6 +141,7 @@ with_theme(; ) scatterlines!(ntest, e_nm; label = "No closure") scatterlines!(ntest, e_cnn; label = "CNN") + scatterlines!(ntest, e_cnn_share; label = "CNN (shared parameters)") scatterlines!(ntest, e_fno_spec; label = "FNO (retrained)") scatterlines!(ntest, e_fno_share; label = "FNO (shared parameters)") lines!(collect(extrema(ntest)), n -> 100n^-2.0; linestyle = :dash, label = "n^-2") @@ -173,9 +178,7 @@ closure.NN # closure.NN # Create input/output arrays -io_train = create_io_arrays(data_train, setup); -io_valid = create_io_arrays(data_valid, setup); -io_test = create_io_arrays(data_test, setup); +io_train = create_io_arrays(data_train[end], setup); size(io_train[1]) diff --git a/scratch/train_model.jl b/scratch/train_model.jl index eab273bd6..ac240fb41 100644 --- a/scratch/train_model.jl +++ b/scratch/train_model.jl @@ -20,6 +20,8 @@ using Optimisers using Random using Zygote +set_theme!(; GLMakie = (; scalefactor = 1.5)) + # Floating point precision T = Float64 @@ -43,20 +45,20 @@ params = (; D = 2, Re = T(6_000), lims = (T(0), T(1)), - nles = 128, + nles = 64, compression = 8, tburn = T(0.05), tsim = T(0.2), Δt = T(1e-4), ArrayType, # ic_params = (; A = T(20_000_000), σ = T(5.0), s = T(3)), - ic_params = (; A = T(10_000_000)), + # ic_params = (; A = T(10_000_000)), ) # Create LES data from DNS data_train = create_les_data(T; params..., nsim = 10); data_valid = create_les_data(T; params..., nsim = 1); -data_test = create_les_data(T; params..., nsim = 5); +data_test = create_les_data(T; params..., nsim = 1); # Inspect data isim = 1 @@ -87,9 +89,9 @@ pressure_solver = SpectralPressureSolver(setup); closure, θ₀ = cnn(; setup, radii = [2, 2, 2, 2], - channels = [32, 32, 32, params.D], + channels = [5, 5, 5, params.D], activations = [leakyrelu, leakyrelu, leakyrelu, identity], - bias = [true, true, true, false], + use_bias = [true, true, true, false], ); closure.NN From 7f328bf3e32e0e87903a5fa3fcb3b1a8ae51c8e9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Mon, 27 Nov 2023 17:23:27 +0100 Subject: [PATCH 157/379] Make manual solver default --- src/solvers/pressure/poisson.jl | 14 +++--- src/solvers/pressure/solvers.jl | 86 ++++++++++++++------------------- 2 files changed, 44 insertions(+), 56 deletions(-) diff --git a/src/solvers/pressure/poisson.jl b/src/solvers/pressure/poisson.jl index b2a844d06..72967df33 100644 --- a/src/solvers/pressure/poisson.jl +++ b/src/solvers/pressure/poisson.jl @@ -41,12 +41,12 @@ function poisson!(solver::DirectPressureSolver, p, f) p end -function poisson!(solver::CGPressureSolver, p, f) - (; A, abstol, reltol, maxiter) = solver - f = view(f, :) - p = view(p, :) - cg!(p, A, f; abstol, reltol, maxiter) -end +# function poisson!(solver::CGPressureSolver, p, f) +# (; A, abstol, reltol, maxiter) = solver +# f = view(f, :) +# p = view(p, :) +# cg!(p, A, f; abstol, reltol, maxiter) +# end # Solve Lp = f # where Lp = Ω * div(pressurgrad(p)) @@ -58,7 +58,7 @@ end # # instead. This way, the matrix is still positive definite. # For initial guess, we already know the average is zero. -function poisson!(solver::CGPressureSolverManual, p, f) +function poisson!(solver::CGPressureSolver, p, f) (; setup, abstol, reltol, maxiter, r, L, q, preconditioner) = solver (; Np, Ip, Ω) = setup.grid T = typeof(reltol) diff --git a/src/solvers/pressure/solvers.jl b/src/solvers/pressure/solvers.jl index 39467d050..cbc7403c4 100644 --- a/src/solvers/pressure/solvers.jl +++ b/src/solvers/pressure/solvers.jl @@ -41,43 +41,43 @@ Adapt.adapt_structure(to, s::DirectPressureSolver) = error( "`DirectPressureSolver` is not yet implemented for CUDA. Consider using `CGPressureSolver`.", ) -""" - CGPressureSolver(setup; [abstol], [reltol], [maxiter]) - -Conjugate gradients iterative pressure solver. -""" -struct CGPressureSolver{T,M<:AbstractMatrix{T}} <: AbstractPressureSolver{T} - A::M - abstol::T - reltol::T - maxiter::Int -end - -function CGPressureSolver( - setup; - abstol = 0, - reltol = √eps(eltype(setup.operators.A)), - maxiter = size(setup.operators.A, 2), -) - (; A) = setup.operators - CGPressureSolver{eltype(A),typeof(A)}(A, abstol, reltol, maxiter) -end - -# This moves all the inner arrays to the GPU when calling -# `cu(::SpectralPressureSolver)` from CUDA.jl -Adapt.adapt_structure(to, s::CGPressureSolver) = CGPressureSolver( - adapt(to, s.A), - adapt(to, s.abstol), - adapt(to, s.reltol), - adapt(to, s.maxiter), -) +# """ +# CGPressureSolver(setup; [abstol], [reltol], [maxiter]) +# +# Conjugate gradients iterative pressure solver. +# """ +# struct CGPressureSolver{T,M<:AbstractMatrix{T}} <: AbstractPressureSolver{T} +# A::M +# abstol::T +# reltol::T +# maxiter::Int +# end + +# function CGPressureSolver( +# setup; +# abstol = 0, +# reltol = √eps(eltype(setup.operators.A)), +# maxiter = size(setup.operators.A, 2), +# ) +# (; A) = setup.operators +# CGPressureSolver{eltype(A),typeof(A)}(A, abstol, reltol, maxiter) +# end + +# # This moves all the inner arrays to the GPU when calling +# # `cu(::SpectralPressureSolver)` from CUDA.jl +# Adapt.adapt_structure(to, s::CGPressureSolver) = CGPressureSolver( +# adapt(to, s.A), +# adapt(to, s.abstol), +# adapt(to, s.reltol), +# adapt(to, s.maxiter), +# ) """ - CGPressureSolverManual(setup; [abstol], [reltol], [maxiter]) + CGPressureSolver(setup; [abstol], [reltol], [maxiter]) Conjugate gradients iterative pressure solver. """ -struct CGPressureSolverManual{T,S,A,F} <: AbstractPressureSolver{T} +struct CGPressureSolver{T,S,A,F} <: AbstractPressureSolver{T} setup::S abstol::T reltol::T @@ -111,33 +111,21 @@ function create_laplace_diag(setup) end end -CGPressureSolverManual( +CGPressureSolver( setup; abstol = zero(eltype(setup.grid.x[1])), reltol = sqrt(eps(eltype(setup.grid.x[1]))), maxiter = prod(setup.grid.Np), # preconditioner = copy!, preconditioner = create_laplace_diag(setup), -) = CGPressureSolverManual( +) = CGPressureSolver( setup, abstol, reltol, maxiter, - KernelAbstractions.zeros( - get_backend(setup.grid.x[1]), - eltype(setup.grid.x[1]), - setup.grid.N, - ), - KernelAbstractions.zeros( - get_backend(setup.grid.x[1]), - eltype(setup.grid.x[1]), - setup.grid.N, - ), - KernelAbstractions.zeros( - get_backend(setup.grid.x[1]), - eltype(setup.grid.x[1]), - setup.grid.N, - ), + similar(setup.grid.x[1], setup.grid.N), + similar(setup.grid.x[1], setup.grid.N), + similar(setup.grid.x[1], setup.grid.N), preconditioner, ) From 4f5db30f9a030255ea39023daefe2e158d65f9f7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Mon, 27 Nov 2023 17:28:36 +0100 Subject: [PATCH 158/379] Add operator --- src/operators.jl | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/operators.jl b/src/operators.jl index e257b01e1..a7dd87465 100644 --- a/src/operators.jl +++ b/src/operators.jl @@ -296,6 +296,13 @@ function laplacian!(L, p, setup) L end +""" + laplacian(p, setup) + +Compute Laplacian of pressure field. +""" +laplacian(p, setup) = laplacian!(similar(p), p, setup) + function laplacian_mat(setup) (; grid, boundary_conditions) = setup (; dimension, x, N, Np, Ip, Δ, Δu, Ω) = grid From 513b548637e38db99053830bf2f74e521ec0a1b3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Mon, 27 Nov 2023 17:41:53 +0100 Subject: [PATCH 159/379] Fix tests --- docs/src/features/operators.md | 1 + src/boundary_conditions.jl | 11 ++++++- test/pressure_solvers.jl | 58 ++++++++++++++++++++++++++-------- test/runtests.jl | 4 +-- 4 files changed, 57 insertions(+), 17 deletions(-) diff --git a/docs/src/features/operators.md b/docs/src/features/operators.md index 379cef343..e79e8b890 100644 --- a/docs/src/features/operators.md +++ b/docs/src/features/operators.md @@ -27,6 +27,7 @@ bodyforce! momentum! momentum laplacian! +laplacian pressuregradient! pressuregradient interpolate_u_p diff --git a/src/boundary_conditions.jl b/src/boundary_conditions.jl index 2da7151f1..391ea539a 100644 --- a/src/boundary_conditions.jl +++ b/src/boundary_conditions.jl @@ -112,6 +112,7 @@ function apply_bc_u!(u, t, setup; kwargs...) apply_bc_u!(boundary_conditions[β][1], u, β, t, setup; atend = false, kwargs...) apply_bc_u!(boundary_conditions[β][2], u, β, t, setup; atend = true, kwargs...) end + u end function apply_bc_p!(p, t, setup; kwargs...) @@ -122,6 +123,7 @@ function apply_bc_p!(p, t, setup; kwargs...) apply_bc_p!(boundary_conditions[β][1], p, β, t, setup; atend = false) apply_bc_p!(boundary_conditions[β][2], p, β, t, setup; atend = true) end + p end function apply_bc_u!(::PeriodicBC, u, β, t, setup; atend, kwargs...) @@ -145,6 +147,7 @@ function apply_bc_u!(::PeriodicBC, u, β, t, setup; atend, kwargs...) _bc_a!(get_backend(u[1]), WORKGROUP)(u, Val(α), Val(β); ndrange) end end + u end function apply_bc_p!(::PeriodicBC, p, β, t, setup; atend, kwargs...) @@ -166,6 +169,7 @@ function apply_bc_p!(::PeriodicBC, p, β, t, setup; atend, kwargs...) else _bc_a(get_backend(p), WORKGROUP)(p, Val(β); ndrange) end + p end function apply_bc_u!(bc::DirichletBC, u, β, t, setup; atend, dudt = false, kwargs...) @@ -193,6 +197,7 @@ function apply_bc_u!(bc::DirichletBC, u, β, t, setup; atend, dudt = false, kwar ) u[α][I] .= bcfunc.((Dimension(α),), xI..., t) end + u end function apply_bc_p!(::DirichletBC, p, β, t, setup; atend, kwargs...) @@ -206,6 +211,7 @@ function apply_bc_p!(::DirichletBC, p, β, t, setup; atend, kwargs...) I = CartesianIndices(ntuple(γ -> γ == β ? (1:1) : (1:N[γ]), D)) p[I] .= p[I.+δ(β)] end + p end function apply_bc_u!(::SymmetricBC, u, β, t, setup; atend, kwargs...) @@ -223,6 +229,7 @@ function apply_bc_u!(::SymmetricBC, u, β, t, setup; atend, kwargs...) end end end + u end function apply_bc_p!(::SymmetricBC, p, β, t, setup; atend, kwargs...) @@ -236,6 +243,7 @@ function apply_bc_p!(::SymmetricBC, p, β, t, setup; atend, kwargs...) I = CartesianIndices(ntuple(γ -> γ == β ? (1:1) : (1:N[γ]), D)) p[I] .= p[I.+δ(β)] end + p end function apply_bc_u!(bc::PressureBC, u, β, t, setup; atend, kwargs...) @@ -265,9 +273,10 @@ function apply_bc_u!(bc::PressureBC, u, β, t, setup; atend, kwargs...) _bc_a!(get_backend(u[1]), WORKGROUP)(u, Val(α), Val(β), I0; ndrange) end end + u end function apply_bc_p!(bc::PressureBC, p, β, t, setup; atend, kwargs...) # p is already zero at boundary - nothing + p end diff --git a/test/pressure_solvers.jl b/test/pressure_solvers.jl index 3e1923ee5..3bb7caa2c 100644 --- a/test/pressure_solvers.jl +++ b/test/pressure_solvers.jl @@ -1,10 +1,12 @@ @testset "Pressure solvers" begin @info "Testing pressure solvers" - n = 20 - x = LinRange(0, 2π, n) - y = LinRange(0, 2π, n) - setup = Setup(x, y) - (; A) = setup.operators + n = 32 + x = LinRange(0, 2π, n + 1) + y = LinRange(0, 2π, n + 1) + Re = 1e3 + setup = Setup(x, y; Re) + (; xp) = setup.grid + D = 2 # Pressure solvers direct = DirectPressureSolver(setup) @@ -12,20 +14,48 @@ spectral = SpectralPressureSolver(setup) initial_pressure(x, y) = 1 / 4 * (cos(2x) + cos(2y)) - p_exact = reshape(initial_pressure.(setup.grid.xpp, setup.grid.ypp), :) - f = A * p_exact + p_exact = + initial_pressure.( + ntuple(α -> reshape(xp[α], ntuple(Returns(1), α - 1)..., :), D)..., + ) + IncompressibleNavierStokes.apply_bc_p!(p_exact, 0.0, setup) + lap = IncompressibleNavierStokes.laplacian(p_exact, setup) - p_direct = poisson(direct, f) - p_cg = poisson(cg, f) - p_spectral = pressure_poisson(spectral, f) + p_direct = IncompressibleNavierStokes.apply_bc_p!( + IncompressibleNavierStokes.poisson(direct, lap), + 0.0, + setup, + ) + p_cg = IncompressibleNavierStokes.apply_bc_p!( + IncompressibleNavierStokes.poisson(cg, lap), + 0.0, + setup, + ) + p_spectral = IncompressibleNavierStokes.apply_bc_p!( + IncompressibleNavierStokes.poisson(spectral, lap), + 0.0, + setup, + ) # Test that in-place and out-of-place versions give same result - @test p_direct ≈ pressure_poisson!(direct, zero(p_exact), f) - @test p_cg ≈ pressure_poisson!(cg, zero(p_exact), f) - @test p_spectral ≈ pressure_poisson!(spectral, zero(p_exact), f) + @test p_direct ≈ IncompressibleNavierStokes.apply_bc_p!( + IncompressibleNavierStokes.poisson!(direct, zero(p_exact), lap), + 0.0, + setup, + ) + @test p_cg ≈ IncompressibleNavierStokes.apply_bc_p!( + IncompressibleNavierStokes.poisson!(cg, zero(p_exact), lap), + 0.0, + setup, + ) + @test p_spectral ≈ IncompressibleNavierStokes.apply_bc_p!( + IncompressibleNavierStokes.poisson!(spectral, zero(p_exact), lap), + 0.0, + setup, + ) # Test that solvers compute the exact pressure - @test_broken p_direct ≈ p_exact # `A` is really badly conditioned + @test p_direct ≈ p_exact @test p_cg ≈ p_exact @test p_spectral ≈ p_exact end diff --git a/test/runtests.jl b/test/runtests.jl index db8378bd2..ff2da57f8 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -13,8 +13,8 @@ using Statistics using Test @testset "IncompressibleNavierStokes" begin - # include("grid.jl") - # include("pressure_solvers.jl") + include("grid.jl") + include("pressure_solvers.jl") # include("models.jl") # include("solvers.jl") # include("simulation2D.jl") From e67faafde87a1bf6edca12d410dbdd144e2582e4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Mon, 27 Nov 2023 17:46:01 +0100 Subject: [PATCH 160/379] Format files --- src/time_steppers/step_implicit_runge_kutta.jl | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/src/time_steppers/step_implicit_runge_kutta.jl b/src/time_steppers/step_implicit_runge_kutta.jl index a1c32231e..5f12f3a14 100644 --- a/src/time_steppers/step_implicit_runge_kutta.jl +++ b/src/time_steppers/step_implicit_runge_kutta.jl @@ -148,14 +148,7 @@ function timestep(method::ImplicitRungeKuttaMethod, stepper, Δt) V = @. V - Δtₙ / Ω * Gp if p_add_solve - p = pressure( - pressure_solver, - V, - p, - tₙ + Δtₙ, - setup; - bc_vectors, - ) + p = pressure(pressure_solver, V, p, tₙ + Δtₙ, setup; bc_vectors) else # Standard method; take last pressure p = pⱼ[(end-Np+1):end] From 6b995ab838fbe01e5fa9ce16f0807b41e4f56ea6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Mon, 27 Nov 2023 19:20:33 +0100 Subject: [PATCH 161/379] Remove wrong name --- docs/src/features/pressure.md | 1 - examples/BackwardFacingStep2D.jl | 2 +- src/create_initial_conditions.jl | 2 +- 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/docs/src/features/pressure.md b/docs/src/features/pressure.md index f3cb0bc61..ec3c628dc 100644 --- a/docs/src/features/pressure.md +++ b/docs/src/features/pressure.md @@ -15,7 +15,6 @@ system. AbstractPressureSolver DirectPressureSolver CGPressureSolver -CGPressureSolverManual SpectralPressureSolver pressure pressure! diff --git a/examples/BackwardFacingStep2D.jl b/examples/BackwardFacingStep2D.jl index 440006bbc..2a2cc2826 100644 --- a/examples/BackwardFacingStep2D.jl +++ b/examples/BackwardFacingStep2D.jl @@ -59,7 +59,7 @@ plotgrid(x, y) # Build setup and assemble operators setup = Setup(x, y; Re, boundary_conditions, ArrayType); -pressure_solver = CGPressureSolverManual(setup); +pressure_solver = CGPressureSolver(setup); # Initial conditions (extend inflow) u₀, p₀ = diff --git a/src/create_initial_conditions.jl b/src/create_initial_conditions.jl index 8de19a533..60a107c74 100644 --- a/src/create_initial_conditions.jl +++ b/src/create_initial_conditions.jl @@ -3,7 +3,7 @@ setup, initial_velocity, t = 0; - pressure_solver = CGPressureSolverManual(setup), + pressure_solver = DirectPressureSolver(setup), project = true, ) From a2d3d072380fb560ddb0bf53015a8cd61bf998a6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Mon, 27 Nov 2023 19:57:19 +0100 Subject: [PATCH 162/379] Merge loops --- src/operators.jl | 37 +++++++++++++++++++++++++++++++++++-- 1 file changed, 35 insertions(+), 2 deletions(-) diff --git a/src/operators.jl b/src/operators.jl index a7dd87465..04cbe252b 100644 --- a/src/operators.jl +++ b/src/operators.jl @@ -173,6 +173,38 @@ function diffusion!(F, u, setup) F end +function convectiondiffusion!(F, u, setup) + (; grid, Re) = setup + (; dimension, Δ, Δu, Nu, Iu, A) = grid + D = dimension() + δ = Offset{D}() + ν = 1 / Re + @kernel function cd!(F, u, ::Val{α}, ::Val{βrange}, I0) where {α,βrange} + I = @index(Global, Cartesian) + I = I + I0 + # for β = 1:D + KernelAbstractions.Extras.LoopInfo.@unroll for β in βrange + Δuαβ = α == β ? Δu[β] : Δ[β] + uαβ1 = A[α][β][2][I[β]-1] * u[α][I-δ(β)] + A[α][β][1][I[β]] * u[α][I] + uαβ2 = A[α][β][2][I[β]] * u[α][I] + A[α][β][1][I[β]+1] * u[α][I+δ(β)] + uβα1 = + A[β][α][2][I[α]-1] * u[β][I-δ(β)] + A[β][α][1][I[α]+1] * u[β][I-δ(β)+δ(α)] + uβα2 = A[β][α][2][I[α]] * u[β][I] + A[β][α][1][I[α]] * u[β][I+δ(α)] + uαuβ1 = uαβ1 * uβα1 + uαuβ2 = uαβ2 * uβα2 + ∂βuα1 = (u[α][I] - u[α][I-δ(β)]) / (β == α ? Δ[β][I[β]] : Δu[β][I[β]-1]) + ∂βuα2 = (u[α][I+δ(β)] - u[α][I]) / (β == α ? Δ[β][I[β]+1] : Δu[β][I[β]]) + F[α][I] += (ν * (∂βuα2 - ∂βuα1) - (uαuβ2 - uαuβ1)) / Δuαβ[I[β]] + end + end + for α = 1:D + I0 = first(Iu[α]) + I0 -= oneunit(I0) + cd!(get_backend(F[1]), WORKGROUP)(F, u, Val(α), Val(1:D), I0; ndrange = Nu[α]) + end + F +end + """ bodyforce!(F, u, setup) @@ -211,8 +243,9 @@ function momentum!(F, u, t, setup) for α = 1:D F[α] .= 0 end - diffusion!(F, u, setup) - convection!(F, u, setup) + # diffusion!(F, u, setup) + # convection!(F, u, setup) + convectiondiffusion!(F, u, setup) bodyforce!(F, u, t, setup) if !isnothing(closure_model) m = closure_model(u) From 5a92fa9353d6e8c99ee9d841f4b6e9d39683e0af Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Tue, 28 Nov 2023 13:50:22 +0100 Subject: [PATCH 163/379] Fix allocation --- src/create_initial_conditions.jl | 20 ++++++++-------- src/filter.jl | 10 +++----- src/operators.jl | 6 ++--- src/processors/real_time_plot.jl | 40 ++++++++++++-------------------- src/solvers/pressure/poisson.jl | 4 ++-- src/solvers/pressure/pressure.jl | 26 ++++----------------- 6 files changed, 37 insertions(+), 69 deletions(-) diff --git a/src/create_initial_conditions.jl b/src/create_initial_conditions.jl index 60a107c74..b87bf89a1 100644 --- a/src/create_initial_conditions.jl +++ b/src/create_initial_conditions.jl @@ -24,9 +24,8 @@ function create_initial_conditions( T = eltype(x[1]) D = dimension() - # Allocate velocity and pressure - u = ntuple(d -> KernelAbstractions.zeros(get_backend(x[1]), T, N...), D) - p = KernelAbstractions.zeros(get_backend(x[1]), T, N...) + # Allocate velocity + u = ntuple(d -> similar(x[1], N), D) # Initial velocities for α = 1:D @@ -43,14 +42,14 @@ function create_initial_conditions( if project f = divergence(u, setup) @. f *= Ω - Δp = poisson(pressure_solver, f) - p .= Δp + p = poisson(pressure_solver, f) apply_bc_p!(p, t, setup) G = pressuregradient(p, setup) for α = 1:D u[α] .-= G[α] end end + apply_bc_u!(u, t, setup) p = pressure(pressure_solver, u, t, setup) apply_bc_p!(p, t, setup) @@ -109,25 +108,26 @@ function random_field( pressure_solver = DirectPressureSolver(setup), ) (; dimension, x, N, Ip, Ω) = setup.grid - D = dimension() T = eltype(x[1]) backend = get_backend(x[1]) + # Create random velocity field u = ntuple(α -> real.(ifft(create_spectrum(N; A, σ, s, backend))), D) apply_bc_u!(u, t, setup) - M = divergence(u, setup) - @. M *= Ω - p = zero(M) # Make velocity field divergence free - poisson!(pressure_solver, p, M) + M = divergence(u, setup) + @. M *= Ω + p = poisson(pressure_solver, M) apply_bc_p!(p, t, setup) G = pressuregradient(p, setup) for α = 1:D @. u[α] -= G[α] end apply_bc_u!(u, t, setup) + + # Compute pressure p = pressure(pressure_solver, u, t, setup) apply_bc_p!(p, t, setup) diff --git a/src/filter.jl b/src/filter.jl index f93b0e4a2..e468a8a27 100644 --- a/src/filter.jl +++ b/src/filter.jl @@ -8,7 +8,7 @@ function face_average!(v, u, setup_les, comp) (; Nu, Iu) = grid D = length(u) δ = Offset{D}() - @kernel function _face_average!(v, u, ::Val{α}, face, I0) where {α} + @kernel function Φ!(v, u, ::Val{α}, face, I0) where {α} I = @index(Global, Cartesian) J = I0 + comp * (I - oneunit(I)) s = zero(eltype(v[α])) @@ -22,18 +22,14 @@ function face_average!(v, u, setup_les, comp) I0 = first(Iu[α]) I0 -= oneunit(I0) face = CartesianIndices(ntuple(β -> β == α ? (comp:comp) : (1:comp), D)) - _face_average!(get_backend(v[1]), WORKGROUP)(v, u, Val(α), face, I0; ndrange) + Φ!(get_backend(v[1]), WORKGROUP)(v, u, Val(α), face, I0; ndrange) end # synchronize(get_backend(u[1])) v end face_average(u, setup_les, comp) = face_average!( - ntuple( - α -> - KernelAbstractions.zeros(get_backend(u[1]), eltype(u[1]), setup_les.grid.N), - length(u), - ), + ntuple(α -> similar(u[1], setup_les.grid.N), length(u)), u, setup_les, comp, diff --git a/src/operators.jl b/src/operators.jl index 04cbe252b..554f6551f 100644 --- a/src/operators.jl +++ b/src/operators.jl @@ -243,9 +243,9 @@ function momentum!(F, u, t, setup) for α = 1:D F[α] .= 0 end - # diffusion!(F, u, setup) - # convection!(F, u, setup) - convectiondiffusion!(F, u, setup) + diffusion!(F, u, setup) + convection!(F, u, setup) + # convectiondiffusion!(F, u, setup) bodyforce!(F, u, t, setup) if !isnothing(closure_model) m = closure_model(u) diff --git a/src/processors/real_time_plot.jl b/src/processors/real_time_plot.jl index 3555322c0..685f3c1c3 100644 --- a/src/processors/real_time_plot.jl +++ b/src/processors/real_time_plot.jl @@ -107,8 +107,7 @@ function fieldplot( p end _f = Array(_f)[Ip] - field = @lift begin - (; u, p, t) = $state + field = lift(state) do (; u, p, t) f = if fieldname == :velocity interpolate_u_p!(up, u, setup) map((u, v) -> √sum(u^2 + v^2), up...) @@ -126,8 +125,7 @@ function fieldplot( copyto!(_f, view(f, Ip)) end - lims = @lift begin - f = $field + lims = lift(field) do f if type ∈ (heatmap, image) lims = get_lims(f) elseif type ∈ (contour, contourf) @@ -190,29 +188,23 @@ function fieldplot( (; xlims, x, xp, Ip) = grid xf = Array.(getindex.(setup.grid.xp, Ip.indices)) + (; u, p) = state[] if fieldname == :velocity elseif fieldname == :vorticity elseif fieldname == :streamfunction elseif fieldname == :pressure elseif fieldname == :Dfield - p = state[].p - d = KernelAbstractions.zeros(get_backend(p), eltype(p), setup.grid.N) - G = ntuple( - α -> KernelAbstractions.zeros(get_backend(p), eltype(p), setup.grid.N), - setup.grid.dimension(), - ) + G = similar.(state[].u) + d = similar(state[].p) elseif fieldname == :Qfield - u = state[].u - Q = KernelAbstractions.zeros(get_backend(u[1]), eltype(u[1]), setup.grid.N) + Q = similar(state[].p) elseif fieldname == :eig2field - u = state[].u - λ = KernelAbstractions.zeros(get_backend(u[1]), eltype(u[1]), setup.grid.N) + λ = similar(state[].p) else error("Unknown fieldname") end - field = @lift begin - (; u, p, t) = $state + field = lift(state) do (; u, p, t) f = if fieldname == :velocity up = interpolate_u_p(u, setup) map((u, v, w) -> √sum(u^2 + v^2 + w^2), up...) @@ -222,7 +214,7 @@ function fieldplot( elseif fieldname == :streamfunction get_streamfunction(setup, u, t) elseif fieldname == :pressure - reshape(copy(p), length(xp), length(yp), length(zp)) + p elseif fieldname == :Dfield Dfield!(d, G, p, setup) d @@ -237,7 +229,7 @@ function fieldplot( end # lims = @lift get_lims($field) - lims = isnothing(levels) ? @lift(get_lims($field)) : extrema(levels) + lims = isnothing(levels) ? lift(get_lims, field) : extrema(levels) isnothing(levels) && (levels = @lift(LinRange($(lims)..., 10))) @@ -272,8 +264,7 @@ Create energy history plot. function energy_history_plot(state; setup) @assert state isa Observable "Energy history requires observable state." _points = Point2f[] - points = @lift begin - (; u, p, t) = $state + points = lift(state) do (; u, p, t) E = kinetic_energy(u, setup) push!(_points, Point2f(t, E)) end @@ -296,7 +287,7 @@ function energy_spectrum_plot(state; setup, naverage = 5^setup.grid.dimension()) D = dimension() K = size(Ip) .÷ 2 kx = ntuple(α -> 1:K[α]-1, D) - k = KernelAbstractions.zeros(backend, T, length.(kx)...) + k = KernelAbstractions.zeros(backend, T, length.(kx)) for α = 1:D kα = reshape(kx[α], ntuple(Returns(1), α - 1)..., :, ntuple(Returns(1), D - α)...) k .+= kα .^ 2 @@ -318,8 +309,7 @@ function energy_spectrum_plot(state; setup, naverage = 5^setup.grid.dimension()) k = Array(A * k) up = interpolate_u_p(state[].u, setup) - ehat = @lift begin - (; u, p, t) = $state + ehat = lift(state) do (; u, p, t) interpolate_u_p!(up, u, setup) e = sum(up -> up[Ip] .^ 2, up) Array(A * reshape(abs.(fft(e)[ntuple(α -> kx[α] .+ 1, D)...]) ./ size(e, 1), :)) @@ -328,8 +318,8 @@ function energy_spectrum_plot(state; setup, naverage = 5^setup.grid.dimension()) # Build inertial slope above energy krange = LinRange(extrema(k)..., 100) slope, slopelabel = D == 2 ? (-T(3), L"$k^{-3}") : (-T(5 / 3), L"$k^{-5/3}") - inertia = @lift begin - slopeconst = maximum($ehat ./ k .^ slope) + inertia = lift(ehat) do ehat + slopeconst = maximum(ehat ./ k .^ slope) 2 .* slopeconst .* krange .^ slope end diff --git a/src/solvers/pressure/poisson.jl b/src/solvers/pressure/poisson.jl index 72967df33..8027673d7 100644 --- a/src/solvers/pressure/poisson.jl +++ b/src/solvers/pressure/poisson.jl @@ -112,9 +112,9 @@ function poisson!(solver::CGPressureSolver, p, f) apply_bc_p!(q, T(0), setup) laplacian!(L, q, setup) # α = ρ / sum(q[Ip] .* L[Ip]) - # α = ρ / dot(view(q, Ip), view(L, Ip)) + α = ρ / dot(view(q, Ip), view(L, Ip)) # α = ρ / innerdot(q, L) - α = ρ / dot(q, L) + # α = ρ / dot(q, L) p .+= α .* q r .-= α .* L diff --git a/src/solvers/pressure/pressure.jl b/src/solvers/pressure/pressure.jl index c011947db..f76293384 100644 --- a/src/solvers/pressure/pressure.jl +++ b/src/solvers/pressure/pressure.jl @@ -8,23 +8,11 @@ function pressure!(pressure_solver, u, p, t, setup, F, G, M) (; grid) = setup (; dimension, Iu, Ip, Ω) = grid D = dimension() - momentum!(F, u, t, setup) - apply_bc_u!(F, t, setup; dudt = true) - - apply_bc_p!(p, t, setup) - pressuregradient!(G, p, setup) - for α = 1:D - F[α] .-= G[α] - # F[α][Iu[α]] .-= G[α][Iu[α]] - end divergence!(M, F, setup) @. M *= Ω - poisson!(pressure_solver, p, M) - # dp = poisson(pressure_solver, M) - # p .+= dp apply_bc_p!(p, t, setup) p end @@ -37,15 +25,9 @@ field, resulting in same order pressure as velocity. """ function pressure(pressure_solver, u, t, setup) D = setup.grid.dimension() - p = KernelAbstractions.zeros(get_backend(u[1]), eltype(u[1]), setup.grid.N) - F = ntuple( - α -> KernelAbstractions.zeros(get_backend(u[1]), eltype(u[1]), setup.grid.N), - D, - ) - G = ntuple( - α -> KernelAbstractions.zeros(get_backend(u[1]), eltype(u[1]), setup.grid.N), - D, - ) - M = KernelAbstractions.zeros(get_backend(u[1]), eltype(u[1]), setup.grid.N) + p = similar(u[1], setup.grid.N) + F = similar.(u) + G = similar.(u) + M = similar(u[1], setup.grid.N) pressure!(pressure_solver, u, p, t, setup, F, G, M) end From 2e989c1fbd0659edca971e599436e556f065fee6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Tue, 28 Nov 2023 15:39:31 +0100 Subject: [PATCH 164/379] Update force --- src/closures/create_les_data.jl | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/src/closures/create_les_data.jl b/src/closures/create_les_data.jl index 69b5d54bc..ccc28d32c 100644 --- a/src/closures/create_les_data.jl +++ b/src/closures/create_les_data.jl @@ -154,8 +154,10 @@ function create_les_data( force_dns = zero.(u₀) force_les = face_average(force_dns, les, compression) - _dns = (; dns..., force = force_dns) - _les = (; les..., force = force_les) + _dns = dns + _les = les + # _dns = (; dns..., bodyforce = force_dns) + # _les = (; les..., bodyforce = force_les) # Solve burn-in DNS @info "Burn-in for simulation $isim of $nsim" @@ -178,16 +180,15 @@ function create_les_data( (T(0), tsim); Δt, processors = ( - _filter_saver(_dns, _les, compression, pressure_solver_les), + f = _filter_saver(_dns, _les, compression, pressure_solver_les), # step_logger(; nupdate = 10), ), pressure_solver, ) - f = outputs[1] # Store result for current IC - push!(filtered.u, f.u) - push!(filtered.c, f.c) + push!(filtered.u, outputs.f.u) + push!(filtered.c, outputs.f.c) end filtered From c07cb7fa61999ebd5ecfc17ae1814cc84768408c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Tue, 28 Nov 2023 16:36:36 +0100 Subject: [PATCH 165/379] Update syntax --- README.md | 6 ++-- examples/Actuator2D.jl | 52 ++++++++++---------------------- examples/Actuator3D.jl | 30 +++++------------- examples/BackwardFacingStep2D.jl | 23 ++++++-------- examples/BackwardFacingStep3D.jl | 25 ++++++--------- examples/DecayingTurbulence2D.jl | 6 ++-- examples/DecayingTurbulence3D.jl | 2 +- examples/LidDrivenCavity2D.jl | 8 ++--- examples/LidDrivenCavity3D.jl | 2 +- examples/PlanarMixing2D.jl | 26 +++++----------- examples/PlaneJets2D.jl | 29 ++++++++---------- examples/ShearLayer2D.jl | 24 ++++++--------- examples/TaylorGreenVortex2D.jl | 2 +- examples/TaylorGreenVortex3D.jl | 4 +-- src/closures/create_les_data.jl | 4 +-- src/solvers/solve_unsteady.jl | 18 ++++++----- test/models.jl | 2 +- test/postprocess2D.jl | 2 +- test/postprocess3D.jl | 15 ++------- test/simulation2D.jl | 6 ++-- test/simulation3D.jl | 12 ++++---- test/solvers.jl | 34 ++++++++++----------- 22 files changed, 128 insertions(+), 204 deletions(-) diff --git a/README.md b/README.md index c4f25222a..b1855229f 100644 --- a/README.md +++ b/README.md @@ -95,12 +95,12 @@ setup = Setup(x, y; Re = 100.0, boundary_conditions, bodyforce); u₀, p₀ = create_initial_conditions(setup, (dim, x, y) -> dim() == 1 ? 1.0 : 0.0); # Solve unsteady Navier-Stokes equations -u, p, outputs = solve_unsteady( +solve_unsteady( setup, u₀, p₀, (0.0, 12.0); Δt = 0.05, processors = ( - animator(setup, "vorticity.mp4"; nupdate = 4), - timelogger(), + anim = animator(; setup, path ="vorticity.mp4", nupdate = 4), + log = timelogger(), ), ) ``` diff --git a/examples/Actuator2D.jl b/examples/Actuator2D.jl index 9516001de..4250b0b30 100644 --- a/examples/Actuator2D.jl +++ b/examples/Actuator2D.jl @@ -21,8 +21,8 @@ end #src using GLMakie #!md using IncompressibleNavierStokes -# Case name for saving results -name = "Actuator2D" +# Output directory +output = "output/Actuator2D" # A 2D grid is a Cartesian product of two vectors n = 40 @@ -66,24 +66,19 @@ u₀, p₀ = create_initial_conditions(setup, (dim, x, y) -> dim() == 1 ? 1.0 : u, p = u₀, p₀ # Solve unsteady problem -u, p, outputs = solve_unsteady( +state, outputs = solve_unsteady( setup, - copy.(u₀), - copy(p₀), - # u, p, + u₀, + p₀, (0.0, 12.0); method = RK44P2(), Δt = 0.05, processors = ( - rtp = realtimeplotter(; - setup, - plot = fieldplot, - ## plot = energy_history_plot, - ## plot = energy_spectrum_plot, - nupdate = 1, - ), - ## anim = animator(; setup, path = "vorticity.mkv", nupdate = 20), - ## vtk = vtk_writer(; setup, nupdate = 10, dir = "output/$name", filename = "solution"), + rtp = realtimeplotter(; setup, plot = fieldplot, nupdate = 1), + # ehist = realtimeplotter(; setup, plot = energy_history_plot, nupdate = 1), + # espec = realtimeplotter(; setup, plot = energy_spectrum_plot, nupdate = 1), + ## anim = animator(; setup, path = "$output/vorticity.mkv", nupdate = 20), + ## vtk = vtk_writer(; setup, nupdate = 10, dir = "$output", filename = "solution"), ## field = fieldsaver(; setup, nupdate = 10), log = timelogger(; nupdate = 1), ), @@ -93,41 +88,26 @@ u, p, outputs = solve_unsteady( # # We may visualize or export the computed fields `(u, p)`. +# Export to VTK +save_vtk(setup, state.u, state.p, "$output/solution") + # We create a box to visualize the actuator. box = ( [xc - δ / 2, xc - δ / 2, xc + δ / 2, xc + δ / 2, xc - δ / 2], [yc + D / 2, yc - D / 2, yc - D / 2, yc + D / 2, yc + D / 2], ) -# Export to VTK -save_vtk(setup, u, p, "output/solution") - -# Field plot -fig = outputs[1] -lines!(box...; color = :red) -fig - # Plot pressure -fig = plot_pressure(setup, p) +fig = fieldplot(state; setup, fieldname = :pressure) lines!(box...; color = :red) fig # Plot velocity -fig = plot_velocity(setup, u) +fig = fieldplot(state; setup, fieldname = :velocity) lines!(box...; color = :red) fig # Plot vorticity -fig = plot_vorticity(setup, u) -lines!(box...; color = :red) -fig - -# Plot streamfunction -fig = plot_streamfunction(setup, u) -lines!(box...; color = :red) -fig - -# Plot force -fig = plot_force(setup) +fig = fieldplot(state; setup, fieldname = :vorticity) lines!(box...; color = :red) fig diff --git a/examples/Actuator3D.jl b/examples/Actuator3D.jl index f5f9a807a..fb902a284 100644 --- a/examples/Actuator3D.jl +++ b/examples/Actuator3D.jl @@ -21,8 +21,8 @@ end #src using GLMakie #!md using IncompressibleNavierStokes -# Case name for saving results -name = "Actuator3D" +# Output directory +output = "output/Actuator3D" # Floating point type T = Float64 @@ -82,7 +82,7 @@ setup = Setup(x, y, z; Re, boundary_conditions, bodyforce, ArrayType); u₀, p₀ = create_initial_conditions(setup, (dim, x, y, z) -> dim() == 1 ? one(x) : zero(x)); # Solve unsteady problem -u, p, outputs = solve_unsteady( +(; u, p, t), outputs = solve_unsteady( setup, u₀, p₀, @@ -97,8 +97,8 @@ u, p, outputs = solve_unsteady( ## plot = energy_spectrum_plot, nupdate = 1, ), - ## anim = animator(; setup, path = "vorticity.mkv", nupdate = 20), - ## vtk = vtk_writer(; setup, nupdate = 10, dir = "output/$name", filename = "solution"), + ## anim = animator(; setup, path = "$output/vorticity.mkv", nupdate = 20), + ## vtk = vtk_writer(; setup, nupdate = 10, dir = "$output", filename = "solution"), ## field = fieldsaver(; setup, nupdate = 10), log = timelogger(; nupdate = 1), ), @@ -109,23 +109,7 @@ u, p, outputs = solve_unsteady( # We may visualize or export the computed fields `(V, p)` # Field plot -outputs[1] +outputs.rtp # Export to VTK -save_vtk(setup, u, p, "output/solution") - -# Plot pressure -plot_pressure(setup, p) - -# Plot velocity -plot_velocity(setup, u) - -# Plot vorticity -plot_vorticity(setup, u) - -# Plot streamfunction -## plot_streamfunction(setup, u) -nothing - -# Plot force -plot_force(setup) +save_vtk(setup, u, p, "$output/solution") diff --git a/examples/BackwardFacingStep2D.jl b/examples/BackwardFacingStep2D.jl index 2a2cc2826..452aeda2a 100644 --- a/examples/BackwardFacingStep2D.jl +++ b/examples/BackwardFacingStep2D.jl @@ -22,8 +22,8 @@ end #src using GLMakie #!md using IncompressibleNavierStokes -# Case name for saving results -name = "BackwardFacingStep2D" +# Output directory +output = "output/BackwardFacingStep2D" # Floating point type T = Float64 @@ -64,7 +64,6 @@ pressure_solver = CGPressureSolver(setup); # Initial conditions (extend inflow) u₀, p₀ = create_initial_conditions(setup, (dim, x, y) -> U(dim, x, y, zero(x)); pressure_solver); -u, p = copy.(u₀), copy(p₀) # Solve steady state problem ## u, p = solve_steady_state(setup, u₀, p₀); @@ -74,7 +73,6 @@ u, p, outputs = solve_unsteady( setup, u₀, p₀, - # u, p, (T(0), T(7)); Δt = T(0.002), pressure_solver, @@ -86,8 +84,8 @@ u, p, outputs = solve_unsteady( ## plot = energy_spectrum_plot, nupdate = 1, ), - ## anim = animator(; setup, path = "vorticity.mkv", nupdate = 20), - ## vtk = vtk_writer(; setup, nupdate = 10, dir = "output/$name", filename = "solution"), + ## anim = animator(; setup, path = "$output/vorticity.mkv", nupdate = 20), + ## vtk = vtk_writer(; setup, nupdate = 10, dir = output, filename = "solution"), ## field = fieldsaver(; setup, nupdate = 10), log = timelogger(; nupdate = 1), ), @@ -95,19 +93,16 @@ u, p, outputs = solve_unsteady( # ## Post-process # -# We may visualize or export the computed fields `(V, p)` +# We may visualize or export the computed fields # Export to VTK -save_vtk(setup, u, p, "output/solution") +save_vtk(setup, state.u, state.p, "$output/solution") # Plot pressure -plot_pressure(setup, p) +fieldplot(state; setup, fieldname = :pressure) # Plot velocity -plot_velocity(setup, u) +fieldplot(state; setup, fieldname = :velocity) # Plot vorticity -plot_vorticity(setup, u) - -# Plot streamfunction -plot_streamfunction(setup, u) +fieldplot(state; setup, fieldname = :vorticity) diff --git a/examples/BackwardFacingStep3D.jl b/examples/BackwardFacingStep3D.jl index 2a9a0b46e..4899a49b6 100644 --- a/examples/BackwardFacingStep3D.jl +++ b/examples/BackwardFacingStep3D.jl @@ -22,8 +22,8 @@ end #src using GLMakie #!md using IncompressibleNavierStokes -# Case name for saving results -name = "BackwardFacingStep3D" +# Output directory +output = "output/BackwardFacingStep3D" # Floating point type T = Float64 @@ -69,7 +69,7 @@ u₀, p₀ = create_initial_conditions(setup, (dim, x, y, z) -> U(dim, x, y, z, nothing # Solve unsteady problem -u, p, outputs = solve_unsteady( +state, outputs = solve_unsteady( setup, u₀, p₀, @@ -83,30 +83,25 @@ u, p, outputs = solve_unsteady( ## plot = energy_spectrum_plot, nupdate = 1, ), - ## anim = animator(; setup, path = "vorticity.mkv", nupdate = 20), - ## vtk = vtk_writer(; setup, nupdate = 10, dir = "output/$name", filename = "solution"), + ## anim = animator(; setup, path = "$output/vorticity.mkv", nupdate = 20), + ## vtk = vtk_writer(; setup, nupdate = 10, dir = output, filename = "solution"), ## field = fieldsaver(; setup, nupdate = 10), log = timelogger(; nupdate = 1), ), ) -#md current_figure() # ## Post-process # -# We may visualize or export the computed fields `(u, p)` +# We may visualize or export the computed fields # Export to VTK -save_vtk(setup, u, p, "output/solution") +save_vtk(setup, state.u, state.p, "$output/solution") # Plot pressure -plot_pressure(setup, p) +fieldplot(state; setup, fieldname = :pressure) # Plot velocity -plot_velocity(setup, u) +fieldplot(state; setup, fieldname = :velocity) # Plot vorticity -plot_vorticity(setup, u) - -# Plot streamfunction -## plot_streamfunction(setup, u) -nothing +fieldplot(state; setup, fieldname = :vorticity) diff --git a/examples/DecayingTurbulence2D.jl b/examples/DecayingTurbulence2D.jl index a6d046264..9a0e28d14 100644 --- a/examples/DecayingTurbulence2D.jl +++ b/examples/DecayingTurbulence2D.jl @@ -49,7 +49,7 @@ pressure_solver = SpectralPressureSolver(setup); u₀, p₀ = random_field(setup, T(0); pressure_solver); # Solve unsteady problem -u, p, outputs = solve_unsteady( +state, outputs = solve_unsteady( setup, u₀, p₀, @@ -76,8 +76,6 @@ u, p, outputs = solve_unsteady( # # We may visualize or export the computed fields `(u, p)` -state = (; u, p, t = T(1)); - # Energy history outputs.ehist @@ -85,7 +83,7 @@ outputs.ehist outputs.espec # Export to VTK -save_vtk(setup, u, p, "$output/solution") +save_vtk(setup, state.u, state.p, "$output/solution") # Plot field fieldplot(state; setup) diff --git a/examples/DecayingTurbulence3D.jl b/examples/DecayingTurbulence3D.jl index 11b98edd0..80e158dde 100644 --- a/examples/DecayingTurbulence3D.jl +++ b/examples/DecayingTurbulence3D.jl @@ -51,7 +51,7 @@ pressure_solver = SpectralPressureSolver(setup); u₀, p₀ = random_field(setup; pressure_solver); # Solve unsteady problem -u, p, outputs = solve_unsteady( +(; u, p, t), outputs = solve_unsteady( setup, u₀, p₀, diff --git a/examples/LidDrivenCavity2D.jl b/examples/LidDrivenCavity2D.jl index c469a3a77..d9cb30deb 100644 --- a/examples/LidDrivenCavity2D.jl +++ b/examples/LidDrivenCavity2D.jl @@ -128,19 +128,17 @@ processors = ( # By default, a standard fourth order Runge-Kutta method is used. If we don't # provide the time step explicitly, an adaptive time step is used. tlims = (T(0), T(10)) -u, p, outputs = +state, outputs = solve_unsteady(setup, u₀, p₀, tlims; Δt = T(1e-3), pressure_solver, processors); # ## Post-process # -# We may visualize or export the computed fields `(u, p)` - -state = (; u, p, t = tlims[end]) +# We may visualize or export the computed fields # Export fields to VTK. The file `output/solution.vti` may be opened for # visualization in [ParaView](https://www.paraview.org/). This is particularly # useful for inspecting results from 3D simulations. -save_vtk(setup, u, p, "$output/solution") +save_vtk(setup, state.u, state.p, "$output/solution") # Plot pressure fieldplot(state; setup, fieldname = :pressure) diff --git a/examples/LidDrivenCavity3D.jl b/examples/LidDrivenCavity3D.jl index d87a193a1..81227b338 100644 --- a/examples/LidDrivenCavity3D.jl +++ b/examples/LidDrivenCavity3D.jl @@ -59,7 +59,7 @@ setup = Setup(x, y, z; Re, boundary_conditions, ArrayType); u₀, p₀ = create_initial_conditions(setup, (dim, x, y, z) -> zero(x)) # Solve unsteady problem -u, p, outputs = solve_unsteady( +(; u, p, t), outputs = solve_unsteady( setup, u₀, p₀, diff --git a/examples/PlanarMixing2D.jl b/examples/PlanarMixing2D.jl index ccf4b1a24..5ac11884d 100644 --- a/examples/PlanarMixing2D.jl +++ b/examples/PlanarMixing2D.jl @@ -19,8 +19,8 @@ end #src using GLMakie #!md using IncompressibleNavierStokes -# Case name for saving results -name = "PlanarMixing2D" +# Output directory +output = "output/PlanarMixing2D" # Viscosity model Re = 500.0 @@ -59,7 +59,7 @@ setup = Setup(x, y; Re, boundary_conditions); u₀, p₀ = create_initial_conditions(setup, (dim, x, y) -> U(dim, x, y, 0.0)); # Solve unsteady problem -u, p, outputs = solve_unsteady( +state, outputs = solve_unsteady( setup, u₀, p₀, @@ -74,8 +74,8 @@ u, p, outputs = solve_unsteady( ## plot = energy_spectrum_plot, nupdate = 1, ), - ## anim = animator(; setup, path = "vorticity.mkv", nupdate = 20), - ## vtk = vtk_writer(; setup, nupdate = 10, dir = "output/$name", filename = "solution"), + ## anim = animator(; setup, path = "$output/vorticity.mkv", nupdate = 20), + ## vtk = vtk_writer(; setup, nupdate = 10, dir = output, filename = "solution"), ## field = fieldsaver(; setup, nupdate = 10), log = timelogger(; nupdate = 1), ), @@ -85,19 +85,7 @@ u, p, outputs = solve_unsteady( # # We may visualize or export the computed fields `(u, p)` -outputs[1] +outputs.rtp # Export to VTK -save_vtk(setup, u, p, "output/solution") - -# Plot pressure -plot_pressure(setup, p) - -# Plot velocity -plot_velocity(setup, u) - -# Plot vorticity -plot_vorticity(setup, u) - -# Plot streamfunction -plot_streamfunction(setup, u) +save_vtk(setup, state.u, state.p, "$output/solution") diff --git a/examples/PlaneJets2D.jl b/examples/PlaneJets2D.jl index 43b26bb3d..f8acba66b 100644 --- a/examples/PlaneJets2D.jl +++ b/examples/PlaneJets2D.jl @@ -22,8 +22,8 @@ using GLMakie #!md using IncompressibleNavierStokes using LaTeXStrings -# Case name for saving results -name = "PlaneJets2D" +# Output directory +output = "output/PlaneJets2D" # Floating point type T = Float64 @@ -163,7 +163,7 @@ function meanplot(; setup, state) end # Solve unsteady problem -toto, p, outputs = solve_unsteady( +state, outputs = solve_unsteady( setup, u₀, p₀, @@ -174,14 +174,14 @@ toto, p, outputs = solve_unsteady( processors = ( rtp = realtimeplotter(; setup, - ## plot = fieldplot, - ## plot = energy_history_plot, - ## plot = energy_spectrum_plot, + # plot = fieldplot, + # plot = energy_history_plot, + # plot = energy_spectrum_plot, plot = meanplot, nupdate = 1, ), - ## anim = animator(; setup, path = "vorticity.mkv", nupdate = 4), - ## vtk = vtk_writer(; setup, nupdate = 10, dir = "output/$name", filename = "solution"), + ## anim = animator(; setup, path = "$output/vorticity.mkv", nupdate = 4), + ## vtk = vtk_writer(; setup, nupdate = 10, dir = output, filename = "solution"), ## field = fieldsaver(; setup, nupdate = 10), log = timelogger(; nupdate = 1), ), @@ -194,17 +194,14 @@ toto, p, outputs = solve_unsteady( outputs.rtp # Export to VTK -save_vtk(setup, toto, p, "output/solution") +save_vtk(setup, state.u, state.p, "$output/solution") # Plot pressure -plot_pressure(setup, p) +fieldplot(state; setup, fieldname = :pressure) # Plot velocity -plot_velocity(setup, u₀) -plot_velocity(setup, toto) +fieldplot((; u = u₀, p = p₀, t = T(0)); setup, fieldname = :velocity) +fieldplot(state; setup, fieldname = :velocity) # Plot vorticity -plot_vorticity(setup, toto) - -# Plot stream function -plot_streamfunction(setup, toto) +fieldplot(state; setup, fieldname = :vorticity) diff --git a/examples/ShearLayer2D.jl b/examples/ShearLayer2D.jl index 81d3c2037..6a49a17db 100644 --- a/examples/ShearLayer2D.jl +++ b/examples/ShearLayer2D.jl @@ -19,8 +19,8 @@ end #src using GLMakie #!md using IncompressibleNavierStokes -# Case name for saving results -name = "ShearLayer2D" +# Output directory +output = "output/ShearLayer2D" # Floating point type T = Float64 @@ -60,7 +60,7 @@ u₀, p₀ = create_initial_conditions( ); # Solve unsteady problem -u, p, outputs = solve_unsteady( +state, outputs = solve_unsteady( setup, u₀, p₀, @@ -75,8 +75,8 @@ u, p, outputs = solve_unsteady( ## plot = energy_spectrum_plot, nupdate = 1, ), - ## anim = animator(; setup, path = "vorticity.mkv", nupdate = 20), - ## vtk = vtk_writer(; setup, nupdate = 10, dir = "output/$name", filename = "solution"), + ## anim = animator(; setup, path = "$output/vorticity.mkv", nupdate = 20), + ## vtk = vtk_writer(; setup, nupdate = 10, dir = output, filename = "solution"), ## field = fieldsaver(; setup, nupdate = 10), log = timelogger(; nupdate = 1), ), @@ -84,22 +84,18 @@ u, p, outputs = solve_unsteady( # ## Post-process # -# We may visualize or export the computed fields `(u, p)` +# We may visualize or export the computed fields outputs.rtp # Export to VTK -save_vtk(setup, u, p, "output/solution") +save_vtk(setup, state.u, state.p, "$output/solution") # Plot pressure -plot_pressure(setup, p) +fieldplot(state; setup, fieldname = :pressure) # Plot velocity -plot_velocity(setup, u) +fieldplot(state; setup, fieldname = :velocity) # Plot vorticity -plot_vorticity(setup, u) - -# Plot streamfunction -## plot_streamfunction(setup, u) -nothing +fieldplot(state; setup, fieldname = :vorticity) diff --git a/examples/TaylorGreenVortex2D.jl b/examples/TaylorGreenVortex2D.jl index 01c405109..949f04f5f 100644 --- a/examples/TaylorGreenVortex2D.jl +++ b/examples/TaylorGreenVortex2D.jl @@ -48,7 +48,7 @@ function compute_convergence(; D, nlist, lims, Re, tlims, Δt, uref, ArrayType = pressure_solver, project = false, ) - u, p, outputs = solve_unsteady(setup, u₀, p₀, tlims; Δt, pressure_solver) + (; u, p, t), outputs = solve_unsteady(setup, u₀, p₀, tlims; Δt, pressure_solver) (; Ip) = setup.grid a, b = T(0), T(0) for α = 1:D diff --git a/examples/TaylorGreenVortex3D.jl b/examples/TaylorGreenVortex3D.jl index 311b63772..d91cb49f6 100644 --- a/examples/TaylorGreenVortex3D.jl +++ b/examples/TaylorGreenVortex3D.jl @@ -53,7 +53,7 @@ u₀, p₀ = create_initial_conditions( ); # Solve unsteady problem -u, p, outputs = solve_unsteady( +(; u, p, t), outputs = solve_unsteady( setup, u₀, p₀, @@ -78,7 +78,7 @@ u, p, outputs = solve_unsteady( # ## Post-process # -# We may visualize or export the computed fields `(u, p)` +# We may visualize or export the computed fields # Energy history outputs.ehist diff --git a/src/closures/create_les_data.jl b/src/closures/create_les_data.jl index ccc28d32c..5419e446d 100644 --- a/src/closures/create_les_data.jl +++ b/src/closures/create_les_data.jl @@ -161,7 +161,7 @@ function create_les_data( # Solve burn-in DNS @info "Burn-in for simulation $isim of $nsim" - u, p, outputs = solve_unsteady( + (; u, p, t), outputs = solve_unsteady( _dns, u₀, p₀, @@ -173,7 +173,7 @@ function create_les_data( # Solve DNS and store filtered quantities @info "Solving DNS for simulation $isim of $nsim" - u, p, outputs = solve_unsteady( + (; u, p, t), outputs = solve_unsteady( _dns, u, p, diff --git a/src/solvers/solve_unsteady.jl b/src/solvers/solve_unsteady.jl index b9a68dbbe..4e10ff0e6 100644 --- a/src/solvers/solve_unsteady.jl +++ b/src/solvers/solve_unsteady.jl @@ -23,11 +23,12 @@ CFL-number `cfl` . The `processors` are called after every time step. -Return a named tuple with the outputs of `processors` with the same field names. - Note that the `state` observable passed to the `processor.initialize` function contains vector living on the device, and you may have to move them back to the host using `Array(u)` and `Array(p)` in the processor. + +Return `(; u, p, t), outputs`, where `outputs` is a named tuple with the +outputs of `processors` with the same field names. """ function solve_unsteady( setup, @@ -91,15 +92,16 @@ function solve_unsteady( state[] = get_state(stepper) end - finalized = (; + # Final state + (; u, p, t) = stepper + + # Processor outputs + outputs = (; (k => processors[k].finalize(initialized[k], state) for k in keys(processors))... ) - # Final state - (; u, p) = stepper - - # Move output arrays to host - u, p, finalized + # Return state and outputs + (; u, p, t), outputs end function get_state(stepper) diff --git a/test/models.jl b/test/models.jl index a122f8be4..c67aa81b9 100644 --- a/test/models.jl +++ b/test/models.jl @@ -60,7 +60,7 @@ broken = convection_model isa Union{C2ConvectionModel,C4ConvectionModel} @test sum(abs, V) / length(V) < lid_vel broken = broken - V, p, outputs = solve_unsteady(setup, V₀, p₀, tlims; Δt = 0.01) + (; u, p, t), outputs = solve_unsteady(setup, V₀, p₀, tlims; Δt = 0.01) # Check that the average velocity is smaller than the lid velocity broken = convection_model isa Union{C2ConvectionModel,C4ConvectionModel} diff --git a/test/postprocess2D.jl b/test/postprocess2D.jl index dd85c5d32..10a2e8eed 100644 --- a/test/postprocess2D.jl +++ b/test/postprocess2D.jl @@ -40,7 +40,7 @@ ) # Solve unsteady problem - V, p, outputs = + state, outputs = solve_unsteady(setup, V₀, p₀, tlims; Δt = 0.01, processors, pressure_solver) @testset "VTK files" begin diff --git a/test/postprocess3D.jl b/test/postprocess3D.jl index 857102070..f24b87ce0 100644 --- a/test/postprocess3D.jl +++ b/test/postprocess3D.jl @@ -44,23 +44,14 @@ ) # Solve unsteady problem - # FIXME: using too many processors results in endless compilation - V, p, outputs = solve_unsteady( - setup, - V₀, - p₀, - tlims; - Δt = T(0.01), - processors, - pressure_solver, - inplace = true, - ) + state, outputs = + solve_unsteady(setup, u₀, p₀, tlims; Δt = T(0.01), processors, pressure_solver) @testset "VTK files" begin @info "Testing 3D processors: VTK files" @test isfile("output/solution3D.pvd") @test isfile("output/solution3D_t=0p0.vti") - save_vtk(setup, V, p, t_end, "output/field3D") + save_vtk(setup, state.u, state.v, t_end, "output/field3D") @test isfile("output/field3D.vti") end diff --git a/test/simulation2D.jl b/test/simulation2D.jl index 8e2583a6d..98dd40faf 100644 --- a/test/simulation2D.jl +++ b/test/simulation2D.jl @@ -43,13 +43,13 @@ processors = (timelogger(),) @testset "Unsteady problem" begin - V, p, outputs = solve_unsteady(setup, V₀, p₀, tlims; Δt = 0.01, processors) + (; u, p, t), outputs = solve_unsteady(setup, V₀, p₀, tlims; Δt = 0.01, processors) # Check that solution did not explode - @test all(!isnan, V) + @test all(!isnan, u) @test all(!isnan, p) # Check that the average velocity is smaller than the lid velocity - @test sum(abs, V) / length(V) < lid_vel + @test sum(abs, u) / length(u) < lid_vel end end diff --git a/test/simulation3D.jl b/test/simulation3D.jl index 50842446e..511eb88a0 100644 --- a/test/simulation3D.jl +++ b/test/simulation3D.jl @@ -28,27 +28,27 @@ ) @testset "Steady state problem" begin - V, p = solve_steady_state(setup, V₀, p₀) + u, p = solve_steady_state(setup, V₀, p₀) # Check that solution did not explode - @test all(!isnan, V) + @test all(!isnan, u) @test all(!isnan, p) # Check that the average velocity is smaller than the lid velocity - @test sum(abs, V) / length(V) < norm(lid_vel) + @test sum(abs, u) / length(u) < norm(lid_vel) end # Iteration processors processors = (timelogger(),) @testset "Unsteady problem" begin - V, p, outputs = solve_unsteady(setup, V₀, p₀, tlims; Δt = 0.01, processors) + (; u, p, t), outputs = solve_unsteady(setup, V₀, p₀, tlims; Δt = 0.01, processors) # Check that solution did not explode - @test all(!isnan, V) + @test all(!isnan, u) @test all(!isnan, p) # Check that the average velocity is smaller than the lid velocity - @test sum(abs, V) / length(V) < norm(lid_vel) + @test sum(abs, u) / length(u) < norm(lid_vel) end end diff --git a/test/solvers.jl b/test/solvers.jl index 1bec7dfe2..4bc6ebe29 100644 --- a/test/solvers.jl +++ b/test/solvers.jl @@ -44,7 +44,7 @@ @testset "Unsteady solvers" begin @testset "Explicit Runge Kutta" begin @info "Testing explicit Runge-Kutta, out-of-place version" - V, p, outputs = solve_unsteady( + state, outputs = solve_unsteady( setup, V₀, p₀, @@ -53,9 +53,9 @@ pressure_solver, inplace = false, ) - @test norm(V - V_exact) / norm(V_exact) < 1e-4 + @test norm(state.u - u_exact) / norm(u_exact) < 1e-4 @info "Testing explicit Runge-Kutta, in-place version" - Vip, pip, outputsip = solve_unsteady( + stateip, outputsip = solve_unsteady( setup, V₀, p₀, @@ -64,8 +64,8 @@ pressure_solver, inplace = true, ) - @test Vip ≈ V - @test pip ≈ p + @test stateip.u ≈ state.u + @test stateip.p ≈ state.p end @testset "Implicit Runge Kutta" begin @@ -81,7 +81,7 @@ inplace = false, ) isa Tuple @info "Testing implicit Runge-Kutta, in-place version" - V, p, outputs = solve_unsteady( + (; u, p, t), outputs = solve_unsteady( setup, V₀, p₀, @@ -92,12 +92,12 @@ inplace = true, processors = (timelogger(),), ) - @test_broken norm(V - V_exact) / norm(V_exact) < 1e-3 + @test_broken norm(u - u_exact) / norm(u_exact) < 1e-3 end @testset "One-leg beta method" begin @info "Testing one-leg beta method, out-of-place version" - V, p, outputs = solve_unsteady( + state, outputs = solve_unsteady( setup, V₀, p₀, @@ -107,9 +107,9 @@ pressure_solver, inplace = false, ) - @test norm(V - V_exact) / norm(V_exact) < 1e-4 + @test norm(state.u - u_exact) / norm(u_exact) < 1e-4 @info "Testing one-leg beta method, in-place version" - Vip, pip, outputsip = solve_unsteady( + stateip, outputsip = solve_unsteady( setup, V₀, p₀, @@ -119,13 +119,13 @@ pressure_solver, inplace = true, ) - @test Vip ≈ V - @test pip ≈ p + @test stateip.u ≈ state.u + @test stateip.p ≈ state.p end @testset "Adams-Bashforth Crank-Nicolson" begin @info "Testing Adams-Bashforth Crank-Nicolson method, out-of-place version" - V, p, outputs = solve_unsteady( + state, outputs = solve_unsteady( setup, V₀, p₀, @@ -135,9 +135,9 @@ pressure_solver, inplace = false, ) - @test norm(V - V_exact) / norm(V_exact) < 1e-4 + @test norm(state.u - u_exact) / norm(u_exact) < 1e-4 @info "Testing Adams-Bashforth Crank-Nicolson method, in-place version" - Vip, pip, outputs = solve_unsteady( + stateip, outputs = solve_unsteady( setup, V₀, p₀, @@ -147,8 +147,8 @@ pressure_solver, inplace = true, ) - @test Vip ≈ V - @test pip ≈ p + @test stateip.u ≈ state.u + @test stateip.p ≈ state.p end end end From 1e11285f183dbb4b69cd2bd89cf94b3fdd36275b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Tue, 28 Nov 2023 18:34:12 +0100 Subject: [PATCH 166/379] Add workgroupsize to setup --- src/IncompressibleNavierStokes.jl | 4 --- src/boundary_conditions.jl | 18 +++++----- src/filter.jl | 4 +-- src/operators.jl | 60 +++++++++++++++---------------- src/setup.jl | 3 ++ src/solvers/pressure/poisson.jl | 5 +-- src/solvers/pressure/solvers.jl | 4 +-- 7 files changed, 49 insertions(+), 49 deletions(-) diff --git a/src/IncompressibleNavierStokes.jl b/src/IncompressibleNavierStokes.jl index 46733dc54..683684fe1 100644 --- a/src/IncompressibleNavierStokes.jl +++ b/src/IncompressibleNavierStokes.jl @@ -27,10 +27,6 @@ using Zygote # Must be loaded inside for Tullio to work correctly using CUDA -# Workgroup size for kernels -# Let this be constant for now -const WORKGROUP = 64 - # # Easily retrieve value from Val # (::Val{x})() where {x} = x diff --git a/src/boundary_conditions.jl b/src/boundary_conditions.jl index 391ea539a..2ba9416a0 100644 --- a/src/boundary_conditions.jl +++ b/src/boundary_conditions.jl @@ -127,7 +127,7 @@ function apply_bc_p!(p, t, setup; kwargs...) end function apply_bc_u!(::PeriodicBC, u, β, t, setup; atend, kwargs...) - (; grid) = setup + (; grid, workgroupsize) = setup (; dimension, N) = grid D = dimension() δ = Offset{D}() @@ -142,16 +142,16 @@ function apply_bc_u!(::PeriodicBC, u, β, t, setup; atend, kwargs...) ndrange = ntuple(γ -> γ == β ? 1 : N[γ], D) for α = 1:D if atend - _bc_b!(get_backend(u[1]), WORKGROUP)(u, Val(α), Val(β); ndrange) + _bc_b!(get_backend(u[1]), workgroupsize)(u, Val(α), Val(β); ndrange) else - _bc_a!(get_backend(u[1]), WORKGROUP)(u, Val(α), Val(β); ndrange) + _bc_a!(get_backend(u[1]), workgroupsize)(u, Val(α), Val(β); ndrange) end end u end function apply_bc_p!(::PeriodicBC, p, β, t, setup; atend, kwargs...) - (; grid) = setup + (; grid, workgroupsize) = setup (; dimension, N) = grid D = dimension() δ = Offset{D}() @@ -165,9 +165,9 @@ function apply_bc_p!(::PeriodicBC, p, β, t, setup; atend, kwargs...) end ndrange = ntuple(γ -> γ == β ? 1 : N[γ], D) if atend - _bc_b(get_backend(p), WORKGROUP)(p, Val(β); ndrange) + _bc_b(get_backend(p), workgroupsize)(p, Val(β); ndrange) else - _bc_a(get_backend(p), WORKGROUP)(p, Val(β); ndrange) + _bc_a(get_backend(p), workgroupsize)(p, Val(β); ndrange) end p end @@ -247,7 +247,7 @@ function apply_bc_p!(::SymmetricBC, p, β, t, setup; atend, kwargs...) end function apply_bc_u!(bc::PressureBC, u, β, t, setup; atend, kwargs...) - (; grid) = setup + (; grid, workgroupsize) = setup (; dimension, N, Nu, Iu) = grid D = dimension() δ = Offset{D}() @@ -266,11 +266,11 @@ function apply_bc_u!(bc::PressureBC, u, β, t, setup; atend, kwargs...) if atend I0 = CartesianIndex(ntuple(γ -> γ == β ? N[β] : 1, D)) I0 -= oneunit(I0) - _bc_b!(get_backend(u[1]), WORKGROUP)(u, Val(α), Val(β), I0; ndrange) + _bc_b!(get_backend(u[1]), workgroupsize)(u, Val(α), Val(β), I0; ndrange) else I0 = CartesianIndex(ntuple(γ -> γ == β && α != β ? 2 : 1, D)) I0 -= oneunit(I0) - _bc_a!(get_backend(u[1]), WORKGROUP)(u, Val(α), Val(β), I0; ndrange) + _bc_a!(get_backend(u[1]), workgroupsize)(u, Val(α), Val(β), I0; ndrange) end end u diff --git a/src/filter.jl b/src/filter.jl index e468a8a27..38b577e1a 100644 --- a/src/filter.jl +++ b/src/filter.jl @@ -4,7 +4,7 @@ Average `u` over volume faces. Put result in `v`. """ function face_average!(v, u, setup_les, comp) - (; grid) = setup_les + (; grid, workgroupsize) = setup_les (; Nu, Iu) = grid D = length(u) δ = Offset{D}() @@ -22,7 +22,7 @@ function face_average!(v, u, setup_les, comp) I0 = first(Iu[α]) I0 -= oneunit(I0) face = CartesianIndices(ntuple(β -> β == α ? (comp:comp) : (1:comp), D)) - Φ!(get_backend(v[1]), WORKGROUP)(v, u, Val(α), face, I0; ndrange) + Φ!(get_backend(v[1]), workgroupsize)(v, u, Val(α), face, I0; ndrange) end # synchronize(get_backend(u[1])) v diff --git a/src/operators.jl b/src/operators.jl index 554f6551f..c2cff8517 100644 --- a/src/operators.jl +++ b/src/operators.jl @@ -18,7 +18,7 @@ struct Offset{D} end Compute divergence of velocity field (in-place version). """ function divergence!(div, u, setup) - (; grid) = setup + (; grid, workgroupsize) = setup (; Δ, N, Ip) = grid D = length(u) δ = Offset{D}() @@ -39,7 +39,7 @@ function divergence!(div, u, setup) # ndrange = Np # I0 = first(Ip) I0 -= oneunit(I0) - div!(get_backend(div), WORKGROUP)(div, u, I0; ndrange) + div!(get_backend(div), workgroupsize)(div, u, I0; ndrange) div end @@ -70,7 +70,7 @@ Compute vorticity field. vorticity!(ω, u, setup) = vorticity!(setup.grid.dimension, ω, u, setup) function vorticity!(::Dimension{2}, ω, u, setup) - (; grid) = setup + (; grid, workgroupsize) = setup (; dimension, Δu, N) = grid D = dimension() δ = Offset{D}() @@ -82,12 +82,12 @@ function vorticity!(::Dimension{2}, ω, u, setup) end I0 = CartesianIndex(ntuple(Returns(1), D)) I0 -= oneunit(I0) - ω!(get_backend(ω), WORKGROUP)(ω, u, I0; ndrange = N .- 1) + ω!(get_backend(ω), workgroupsize)(ω, u, I0; ndrange = N .- 1) ω end function vorticity!(::Dimension{3}, ω, u, setup) - (; grid) = setup + (; grid, workgroupsize) = setup (; dimension, Δu, N) = grid D = dimension() δ = Offset{D}() @@ -105,7 +105,7 @@ function vorticity!(::Dimension{3}, ω, u, setup) end I0 = CartesianIndex(ntuple(Returns(1), D)) I0 -= oneunit(I0) - ω!(get_backend(ω[1]), WORKGROUP)(ω, u, I0; ndrange = N .- 1) + ω!(get_backend(ω[1]), workgroupsize)(ω, u, I0; ndrange = N .- 1) ω end @@ -115,7 +115,7 @@ end Compute convective term. """ function convection!(F, u, setup) - (; grid) = setup + (; grid, workgroupsize) = setup (; dimension, Δ, Δu, Nu, Iu, A) = grid D = dimension() δ = Offset{D}() @@ -136,7 +136,7 @@ function convection!(F, u, setup) for α = 1:D I0 = first(Iu[α]) I0 -= oneunit(I0) - conv!(get_backend(F[1]), WORKGROUP)(F, u, Val(α), Val(1:D), I0; ndrange = Nu[α]) + conv!(get_backend(F[1]), workgroupsize)(F, u, Val(α), Val(1:D), I0; ndrange = Nu[α]) end F end @@ -147,7 +147,7 @@ end Compute diffusive term. """ function diffusion!(F, u, setup) - (; grid, Re) = setup + (; grid, workgroupsize, Re) = setup (; dimension, Δ, Δu, Nu, Iu) = grid D = dimension() δ = Offset{D}() @@ -168,13 +168,13 @@ function diffusion!(F, u, setup) for α = 1:D I0 = first(Iu[α]) I0 -= oneunit(I0) - diff!(get_backend(F[1]), WORKGROUP)(F, u, Val(α), Val(1:D), I0; ndrange = Nu[α]) + diff!(get_backend(F[1]), workgroupsize)(F, u, Val(α), Val(1:D), I0; ndrange = Nu[α]) end F end function convectiondiffusion!(F, u, setup) - (; grid, Re) = setup + (; grid, workgroupsize, Re) = setup (; dimension, Δ, Δu, Nu, Iu, A) = grid D = dimension() δ = Offset{D}() @@ -200,7 +200,7 @@ function convectiondiffusion!(F, u, setup) for α = 1:D I0 = first(Iu[α]) I0 -= oneunit(I0) - cd!(get_backend(F[1]), WORKGROUP)(F, u, Val(α), Val(1:D), I0; ndrange = Nu[α]) + cd!(get_backend(F[1]), workgroupsize)(F, u, Val(α), Val(1:D), I0; ndrange = Nu[α]) end F end @@ -211,7 +211,7 @@ end Compute body force. """ function bodyforce!(F, u, t, setup) - (; grid, bodyforce) = setup + (; grid, workgroupsize, bodyforce) = setup (; dimension, Δ, Δu, Nu, Iu, x, xp) = grid isnothing(bodyforce) && return F D = dimension() @@ -225,7 +225,7 @@ function bodyforce!(F, u, t, setup) for α = 1:D I0 = first(Iu[α]) I0 -= oneunit(I0) - f!(get_backend(F[1]), WORKGROUP)(F, bodyforce, Val(α), t, I0; ndrange = Nu[α]) + f!(get_backend(F[1]), workgroupsize)(F, bodyforce, Val(α), t, I0; ndrange = Nu[α]) end F end @@ -269,7 +269,7 @@ momentum(u, t, setup) = momentum!(zero.(u), u, t, setup) Compute pressure gradient (in-place). """ function pressuregradient!(G, p, setup) - (; grid) = setup + (; grid, workgroupsize) = setup (; dimension, Δu, Nu, Iu) = grid D = dimension() δ = Offset{D}() @@ -282,7 +282,7 @@ function pressuregradient!(G, p, setup) for α = 1:D I0 = first(Iu[α]) I0 -= oneunit(I0) - G!(get_backend(G[1]), WORKGROUP)(G, p, Val(α), I0; ndrange = Nu[α]) + G!(get_backend(G[1]), workgroupsize)(G, p, Val(α), I0; ndrange = Nu[α]) end G end @@ -304,7 +304,7 @@ pressuregradient(p, setup) = pressuregradient!( Compute Laplacian of pressure field (in-place version). """ function laplacian!(L, p, setup) - (; grid) = setup + (; grid, workgroupsize) = setup (; dimension, Δ, Δu, N, Np, Ip, Ω) = grid D = dimension() δ = Offset{D}() @@ -325,7 +325,7 @@ function laplacian!(L, p, setup) ndrange = Np I0 = first(Ip) I0 -= oneunit(I0) - lap!(get_backend(L), WORKGROUP)(L, p, I0; ndrange) + lap!(get_backend(L), workgroupsize)(L, p, I0; ndrange) L end @@ -441,7 +441,7 @@ interpolate_u_p(u, setup) = Interpolate velocity to pressure points. """ function interpolate_u_p!(up, u, setup) - (; boundary_conditions, grid, Re, bodyforce) = setup + (; boundary_conditions, grid, workgroupsize, Re, bodyforce) = setup (; dimension, Np, Ip) = grid D = dimension() δ = Offset{D}() @@ -453,7 +453,7 @@ function interpolate_u_p!(up, u, setup) for α = 1:D I0 = first(Ip) I0 -= oneunit(I0) - int!(get_backend(up[1]), WORKGROUP)(up, u, Val(α), I0; ndrange = Np) + int!(get_backend(up[1]), workgroupsize)(up, u, Val(α), I0; ndrange = Np) end up end @@ -478,7 +478,7 @@ Interpolate vorticity to pressure points. interpolate_ω_p!(ωp, ω, setup) = interpolate_ω_p!(setup.grid.dimension, ωp, ω, setup) function interpolate_ω_p!(::Dimension{2}, ωp, ω, setup) - (; boundary_conditions, grid, Re, bodyforce) = setup + (; boundary_conditions, grid, workgroupsize, Re, bodyforce) = setup (; dimension, Np, Ip) = grid D = dimension() δ = Offset{D}() @@ -489,12 +489,12 @@ function interpolate_ω_p!(::Dimension{2}, ωp, ω, setup) end I0 = first(Ip) I0 -= oneunit(I0) - int!(get_backend(ωp), WORKGROUP)(ωp, ω, I0; ndrange = Np) + int!(get_backend(ωp), workgroupsize)(ωp, ω, I0; ndrange = Np) ωp end function interpolate_ω_p!(::Dimension{3}, ωp, ω, setup) - (; boundary_conditions, grid, Re) = setup + (; boundary_conditions, grid, workgroupsize, Re) = setup (; dimension, Np, Ip) = grid D = dimension() δ = Offset{D}() @@ -508,7 +508,7 @@ function interpolate_ω_p!(::Dimension{3}, ωp, ω, setup) I0 = first(Ip) I0 -= oneunit(I0) for α = 1:D - int!(get_backend(ωp[1]), WORKGROUP)(ωp, ω, Val(α), I0; ndrange = Np) + int!(get_backend(ωp[1]), workgroupsize)(ωp, ω, Val(α), I0; ndrange = Np) end ωp end @@ -523,7 +523,7 @@ D = \\frac{2 | \\nabla p |}{\\nabla^2 p}. ``` """ function Dfield!(d, G, p, setup; ϵ = eps(eltype(p))) - (; boundary_conditions, grid) = setup + (; boundary_conditions, grid, workgroupsize) = setup (; dimension, Np, Ip, Δ) = grid T = eltype(p) D = dimension() @@ -554,7 +554,7 @@ function Dfield!(d, G, p, setup; ϵ = eps(eltype(p))) pressuregradient!(G, p, setup) I0 = first(Ip) I0 -= oneunit(I0) - D!(get_backend(p), WORKGROUP)(d, G, p, I0; ndrange = Np) + D!(get_backend(p), workgroupsize)(d, G, p, I0; ndrange = Np) d end @@ -581,7 +581,7 @@ Q = - \\frac{1}{2} \\sum_{α, β} \\frac{\\partial u^α}{\\partial x^β} ``` """ function Qfield!(Q, u, setup; ϵ = eps(eltype(Q))) - (; boundary_conditions, grid) = setup + (; boundary_conditions, grid, workgroupsize) = setup (; dimension, Np, Ip, Δ) = grid D = dimension() δ = Offset{D}() @@ -598,7 +598,7 @@ function Qfield!(Q, u, setup; ϵ = eps(eltype(Q))) end I0 = first(Ip) I0 -= oneunit(I0) - Q!(get_backend(u[1]), WORKGROUP)(Q, u, I0; ndrange = Np) + Q!(get_backend(u[1]), workgroupsize)(Q, u, I0; ndrange = Np) Q end @@ -615,7 +615,7 @@ Qfield(u, setup) = Qfield!(similar(u[1], setup.grid.N), u, setup) Compute the second eigenvalue of ``S^2 + \\Omega^2``. """ function eig2field!(λ, u, setup) - (; boundary_conditions, grid) = setup + (; boundary_conditions, grid, workgroupsize) = setup (; dimension, Np, Ip, Δ, Δu) = grid D = dimension() δ = Offset{D}() @@ -648,7 +648,7 @@ function eig2field!(λ, u, setup) end I0 = first(Ip) I0 -= oneunit(I0) - λ!(get_backend(u[1]), WORKGROUP)(λ, u, I0; ndrange = Np) + λ!(get_backend(u[1]), workgroupsize)(λ, u, I0; ndrange = Np) λ end diff --git a/src/setup.jl b/src/setup.jl index 59048d24d..304d783d0 100644 --- a/src/setup.jl +++ b/src/setup.jl @@ -7,6 +7,7 @@ bodyforce = nothing, closure_model = nothing, ArrayType = Array, + workgroupsize = 64, ) Create setup. @@ -19,6 +20,7 @@ Setup( bodyforce = nothing, closure_model = nothing, ArrayType = Array, + workgroupsize = 64, ) = (; grid = Grid(x, boundary_conditions; ArrayType), boundary_conditions, @@ -27,4 +29,5 @@ Setup( bodyforce, closure_model, ArrayType, + workgroupsize, ) diff --git a/src/solvers/pressure/poisson.jl b/src/solvers/pressure/poisson.jl index 8027673d7..33bcf5ada 100644 --- a/src/solvers/pressure/poisson.jl +++ b/src/solvers/pressure/poisson.jl @@ -60,7 +60,8 @@ end # For initial guess, we already know the average is zero. function poisson!(solver::CGPressureSolver, p, f) (; setup, abstol, reltol, maxiter, r, L, q, preconditioner) = solver - (; Np, Ip, Ω) = setup.grid + (; grid, workgroupsize) = setup + (; Np, Ip, Ω) = setup T = typeof(reltol) function innerdot(a, b) @@ -78,7 +79,7 @@ function poisson!(solver::CGPressureSolver, p, f) eltype(a), ntuple(Returns(1), length(I0)), ) - innerdot!(get_backend(a), WORKGROUP)(d, a, b, I0; ndrange = Np) + innerdot!(get_backend(a), workgroupsize)(d, a, b, I0; ndrange = Np) d[] end diff --git a/src/solvers/pressure/solvers.jl b/src/solvers/pressure/solvers.jl index cbc7403c4..68687b4b0 100644 --- a/src/solvers/pressure/solvers.jl +++ b/src/solvers/pressure/solvers.jl @@ -89,7 +89,7 @@ struct CGPressureSolver{T,S,A,F} <: AbstractPressureSolver{T} end function create_laplace_diag(setup) - (; grid) = setup + (; grid, workgroupsize) = setup (; dimension, Δ, Δu, N, Np, Ip, Ω) = grid D = dimension() δ = Offset{D}() @@ -106,7 +106,7 @@ function create_laplace_diag(setup) I0 = first(Ip) I0 -= oneunit(I0) function laplace_diag(z, p) - _laplace_diag!(get_backend(z), WORKGROUP)(z, p, I0; ndrange) + _laplace_diag!(get_backend(z), workgroupsize)(z, p, I0; ndrange) # synchronize(get_backend(z)) end end From 90c40558d1b33090afe66ba49b5a6b845b5d2d66 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Wed, 29 Nov 2023 15:32:06 +0100 Subject: [PATCH 167/379] Factorize functions --- scratch/train_model.jl | 100 ++++++++++++++--------------- src/IncompressibleNavierStokes.jl | 2 +- src/closures/closure.jl | 76 +++++++++++++++++++++- src/closures/cnn.jl | 77 ++-------------------- src/closures/create_les_data.jl | 8 ++- src/closures/fno.jl | 103 +++++------------------------- 6 files changed, 152 insertions(+), 214 deletions(-) diff --git a/scratch/train_model.jl b/scratch/train_model.jl index ac240fb41..1f6e06d35 100644 --- a/scratch/train_model.jl +++ b/scratch/train_model.jl @@ -45,8 +45,8 @@ params = (; D = 2, Re = T(6_000), lims = (T(0), T(1)), - nles = 64, - compression = 8, + nles = 128, + ndns = 1024, tburn = T(0.05), tsim = T(0.2), Δt = T(1e-4), @@ -60,19 +60,6 @@ data_train = create_les_data(T; params..., nsim = 10); data_valid = create_les_data(T; params..., nsim = 1); data_test = create_les_data(T; params..., nsim = 1); -# Inspect data -isim = 1 -α = 1 -# j = 13 -o = Observable(data_train.u[isim][1][α]) -# o = Observable(data_train.u[isim][1][α][:, :, j]) -heatmap(o) -for i = 1:length(data_train.u[isim]) - o[] = data_train.u[isim][i][α] - # o[] = data_train.u[isim][i][α][:, :, j] - sleep(0.001) -end - # # Save filtered DNS data # jldsave("output/forced/data.jld2"; data_train, data_valid, data_test) @@ -83,47 +70,56 @@ end x = ntuple(α -> LinRange(params.lims..., params.nles + 1), params.D) setup = Setup(x...; params.Re, ArrayType); +# Inspect data +(; Ip) = setup.grid; +field = data_train.u[1]; +α = 1 +# j = 13 +o = Observable(field[1][α][Ip]) +# o = Observable(field[1][α][:, :, j]) +heatmap(o) +for i = 1:length(field) + o[] = field[i][α][Ip] + # o[] = field[i][α][:, :, j] + sleep(0.001) +end + # Uniform periodic grid pressure_solver = SpectralPressureSolver(setup); -closure, θ₀ = cnn(; - setup, - radii = [2, 2, 2, 2], - channels = [5, 5, 5, params.D], - activations = [leakyrelu, leakyrelu, leakyrelu, identity], - use_bias = [true, true, true, false], -); -closure.NN +# Create input/output arrays +io_train = create_io_arrays(data_train, setup); +io_valid = create_io_arrays(data_valid, setup); +io_test = create_io_arrays(data_test, setup); -sample = io_train[1][:, :, :, 1:5] -closure(sample, θ₀) |> size +size(io_train[1]) -θ.layer_5 -θ.layer_6 +Base.summarysize(io_train) / 1e9 -# closure, θ₀ = fno(; +# closure, θ₀ = cnn(; # setup, -# -# # Cut-off wavenumbers -# k = [8, 8, 8, 8], -# -# # Channel sizes -# c = [16, 16, 16, 16], -# -# # Fourier layer activations -# σ = [gelu, gelu, gelu, identity], -# -# # Dense activation -# ψ = gelu, +# radii = [2, 2, 2, 2], +# channels = [5, 5, 5, params.D], +# activations = [leakyrelu, leakyrelu, leakyrelu, identity], +# use_bias = [true, true, true, false], # ); # closure.NN +# +# sample = io_train[1][:, :, :, 1:5] +# closure(sample, θ₀) |> size +# +# θ.layer_5 +# θ.layer_6 -# Create input/output arrays -io_train = create_io_arrays(data_train, setup); -io_valid = create_io_arrays(data_valid, setup); -io_test = create_io_arrays(data_test, setup); +closure, θ₀ = fno(; + setup, + kmax = [8, 8, 8, 8], + c = [5, 5, 5, 5], + σ = [gelu, gelu, gelu, identity], + ψ = gelu, +); -size(io_train[1]) +closure.NN # Prepare training θ = T(1.0e-1) * device(θ₀); @@ -197,24 +193,24 @@ relerr_track(uref, setup) = end u, u₀, p₀ = nothing, nothing, nothing -u = device.(data_test.u[1]) -u₀ = device(data_test.u[1][1]) -p₀ = pressure(pressure_solver, u₀, T(0), setup) +u = device.(data_test.u[1]); +u₀ = device(data_test.u[1][1]); +p₀ = IncompressibleNavierStokes.pressure(pressure_solver, u₀, T(0), setup); length(u) -u_nm, p_nm, outputs = solve_unsteady( +state_nm, outputs = solve_unsteady( setup, u₀, p₀, (T(0), params.tsim); Δt = data_test.Δt, pressure_solver, - processors = (relerr = relerr_track(u, setup), log = timelogger(; nupdate = 1)), + processors = (; relerr = relerr_track(u, setup), log = timelogger(; nupdate = 1)), ) relerr_nm = outputs.relerr[] -u_cnn, p_cnn, outputs = solve_unsteady( - (; setup..., closure_model = create_neural_closure(closure, θ, setup)), +state_cnn, outputs = solve_unsteady( + (; setup..., closure_model = wrappedclosure(closure, θ, setup)), u₀, p₀, (T(0), params.tsim); diff --git a/src/IncompressibleNavierStokes.jl b/src/IncompressibleNavierStokes.jl index 683684fe1..6dd06e890 100644 --- a/src/IncompressibleNavierStokes.jl +++ b/src/IncompressibleNavierStokes.jl @@ -123,7 +123,7 @@ export cnn, fno, FourierLayer export train export mean_squared_error, relative_error export create_randloss, create_callback, create_les_data, create_io_arrays -export create_neural_closure +export wrappedclosure # ODE methods diff --git a/src/closures/closure.jl b/src/closures/closure.jl index e3f765889..4675395c9 100644 --- a/src/closures/closure.jl +++ b/src/closures/closure.jl @@ -1,4 +1,4 @@ -function create_neural_closure(m, θ, setup) +function wrappedclosure(m, θ, setup) (; dimension, Iu) = setup.grid D = dimension() function neural_closure(u) @@ -11,3 +11,77 @@ function create_neural_closure(m, θ, setup) mu = ntuple(α -> mu[i..., α, 1], D) end end + +function create_closure(layers...) + chain = Chain(layers...) + + # Create parameter vector (empty state) + params, state = Lux.setup(rng, chain) + θ = ComponentArray(params) + + # Compute closure term for given parameters + closure(u, θ) = first(chain(u, θ, state)) + + closure, θ +end + +function collocate(u) + sz..., D, _ = size(u) + # for α = 1:D + # v = selectdim(u, D + 1, α) + # v = (v + circshift(v, ntuple(β -> α == β ? -1 : 0, D + 1))) / 2 + # end + if D == 2 + a = selectdim(u, 3, 1) + b = selectdim(u, 3, 2) + a = (a .+ circshift(a, (-1, 0, 0))) ./ 2 + b = (b .+ circshift(b, (0, -1, 0))) ./ 2 + a = reshape(a, sz..., 1, :) + b = reshape(b, sz..., 1, :) + cat(a, b; dims = 3) + elseif D == 3 + a = selectdim(u, 4, 1) + b = selectdim(u, 4, 2) + c = selectdim(u, 4, 3) + a = (a .+ circshift(a, (-1, 0, 0, 0))) ./ 2 + b = (b .+ circshift(b, (0, -1, 0, 0))) ./ 2 + c = (c .+ circshift(c, (0, 0, -1, 0))) ./ 2 + a = reshape(a, sz..., 1, :) + b = reshape(b, sz..., 1, :) + c = reshape(c, sz..., 1, :) + cat(a, b, c; dims = 4) + end +end + +function decollocate(u) + sz..., D, _ = size(u) + # for α = 1:D + # v = selectdim(u, D + 1, α) + # v = (v + circshift(v, ntuple(β -> α == β ? -1 : 0, D + 1))) / 2 + # end + if D == 2 + a = selectdim(u, 3, 1) + b = selectdim(u, 3, 2) + a = (a .+ circshift(a, (1, 0, 0))) ./ 2 + b = (b .+ circshift(b, (0, 1, 0))) ./ 2 + # a = circshift(a, (1, 0, 0)) .- a + # b = circshift(b, (0, 1, 0)) .- b + a = reshape(a, sz..., 1, :) + b = reshape(b, sz..., 1, :) + cat(a, b; dims = 3) + elseif D == 3 + a = selectdim(u, 4, 1) + b = selectdim(u, 4, 2) + c = selectdim(u, 4, 3) + a = (a .+ circshift(a, (1, 0, 0, 0))) ./ 2 + b = (b .+ circshift(b, (0, 1, 0, 0))) ./ 2 + c = (c .+ circshift(c, (0, 0, 1, 0))) ./ 2 + # a = circshift(a, (1, 0, 0, 0)) .- a + # b = circshift(b, (0, 1, 0, 0)) .- b + # c = circshift(c, (0, 0, 1, 0)) .- c + a = reshape(a, sz..., 1, :) + b = reshape(b, sz..., 1, :) + c = reshape(c, sz..., 1, :) + cat(a, b, c; dims = 4) + end +end diff --git a/src/closures/cnn.jl b/src/closures/cnn.jl index f95be102c..afcaeca3b 100644 --- a/src/closures/cnn.jl +++ b/src/closures/cnn.jl @@ -23,9 +23,11 @@ function cnn(; ) r, c, σ, b = radii, channels, activations, use_bias (; grid) = setup - (; dimension, x) = grid + (; dimension, x, Δu) = grid D = dimension() + dx = map(d -> d[2:end-1], Δu) + # Weight initializer T = eltype(x[1]) glorot_uniform_T(rng::AbstractRNG, dims...) = glorot_uniform(rng, T, dims...) @@ -37,34 +39,9 @@ function cnn(; c = [D; c] # Create convolutional closure model - NN = Chain( - function (u) - sz..., _, _ = size(u) - # for α = 1:D - # v = selectdim(u, D + 1, α) - # v = (v + circshift(v, ntuple(β -> α == β ? -1 : 0, D + 1))) / 2 - # end - if D == 2 - a = selectdim(u, 3, 1) - b = selectdim(u, 3, 2) - a = (a + circshift(a, (-1, 0, 0))) / 2 - b = (b + circshift(b, (0, -1, 0))) / 2 - a = reshape(a, sz..., 1, :) - b = reshape(b, sz..., 1, :) - cat(a, b; dims = 3) - elseif D == 3 - a = selectdim(u, 4, 1) - b = selectdim(u, 4, 2) - c = selectdim(u, 4, 3) - a = (a + circshift(a, (-1, 0, 0, 0))) / 2 - b = (b + circshift(b, (0, -1, 0, 0))) / 2 - c = (c + circshift(c, (0, 0, -1, 0))) / 2 - a = reshape(a, sz..., 1, :) - b = reshape(b, sz..., 1, :) - c = reshape(c, sz..., 1, :) - cat(a, b, c; dims = 4) - end - end, + create_closure( + # Put inputs in pressure points + collocate, # Add padding so that output has same shape as commutator error u -> pad_circular(u, sum(r)), @@ -81,46 +58,6 @@ function cnn(; )..., # Differentiate output to velocity points - function (u) - sz..., _, _ = size(u) - # for α = 1:D - # v = selectdim(u, D + 1, α) - # v = (v + circshift(v, ntuple(β -> α == β ? -1 : 0, D + 1))) / 2 - # end - if D == 2 - a = selectdim(u, 3, 1) - b = selectdim(u, 3, 2) - # a = (a + circshift(a, (1, 0, 0, 0))) / 2 - # b = (b + circshift(b, (0, 1, 0, 0))) / 2 - a = circshift(a, (1, 0, 0)) - a - b = circshift(b, (0, 1, 0)) - b - a = reshape(a, sz..., 1, :) - b = reshape(b, sz..., 1, :) - cat(a, b; dims = 3) - elseif D == 3 - a = selectdim(u, 4, 1) - b = selectdim(u, 4, 2) - c = selectdim(u, 4, 3) - # a = (a + circshift(a, (1, 0, 0, 0))) / 2 - # b = (b + circshift(b, (0, 1, 0, 0))) / 2 - # c = (c + circshift(c, (0, 0, 1, 0))) / 2 - a = circshift(a, (1, 0, 0, 0)) - a - b = circshift(b, (0, 1, 0, 0)) - b - c = circshift(c, (0, 0, 1, 0)) - c - a = reshape(a, sz..., 1, :) - b = reshape(b, sz..., 1, :) - c = reshape(c, sz..., 1, :) - cat(a, b, c; dims = 4) - end - end, + decollocate, ) - - # Create parameter vector (empty state) - params, state = Lux.setup(rng, NN) - θ = ComponentArray(params) - - # Compute closure term for given parameters - closure(u, θ) = first(NN(u, θ, state)) - - closure, θ end diff --git a/src/closures/create_les_data.jl b/src/closures/create_les_data.jl index 5419e446d..17edf9053 100644 --- a/src/closures/create_les_data.jl +++ b/src/closures/create_les_data.jl @@ -85,7 +85,7 @@ _filter_saver(dns, les, comp, pressure_solver; nupdate = 1) = Re = T(2_000), lims = (T(0), T(1)), nles = 64, - compression = 4, + ndns = 256, nsim = 10, tburn = T(0.1), tsim = T(0.1), @@ -102,7 +102,7 @@ function create_les_data( Re = T(2_000), lims = (T(0), T(1)), nles = 64, - compression = 4, + ndns = 256, nsim = 10, tburn = T(0.1), tsim = T(0.1), @@ -110,7 +110,9 @@ function create_les_data( ArrayType = Array, ic_params = (;), ) - ndns = compression * nles + compression = ndns ÷ nles + @assert compression * nles == ndns + xdns = ntuple(α -> LinRange(lims..., ndns + 1), D) xles = ntuple(α -> LinRange(lims..., nles + 1), D) diff --git a/src/closures/fno.jl b/src/closures/fno.jl index a5f8a23e4..db595f8b1 100644 --- a/src/closures/fno.jl +++ b/src/closures/fno.jl @@ -23,35 +23,9 @@ function fno(; setup, kmax, c, σ, ψ, rng = Random.default_rng(), kwargs...) init_weight(rng::AbstractRNG, dims...) = glorot_uniform(rng, T, dims...) # Create FNO closure model - NN = Chain( + create_closure( # Put inputs in pressure points - function (u) - sz..., _, _ = size(u) - # for α = 1:D - # v = selectdim(u, D + 1, α) - # v = (v + circshift(v, ntuple(β -> α == β ? -1 : 0, D + 1))) / 2 - # end - if D == 2 - a = selectdim(u, 3, 1) - b = selectdim(u, 3, 2) - a = (a + circshift(a, (-1, 0, 0))) / 2 - b = (b + circshift(b, (0, -1, 0))) / 2 - a = reshape(a, sz..., 1, :) - b = reshape(b, sz..., 1, :) - cat(a, b; dims = 3) - elseif D == 3 - a = selectdim(u, 4, 1) - b = selectdim(u, 4, 2) - c = selectdim(u, 4, 3) - a = (a + circshift(a, (-1, 0, 0, 0))) / 2 - b = (b + circshift(b, (0, -1, 0, 0))) / 2 - c = (c + circshift(c, (0, 0, -1, 0))) / 2 - a = reshape(a, sz..., 1, :) - b = reshape(b, sz..., 1, :) - c = reshape(c, sz..., 1, :) - cat(a, b, c; dims = 4) - end - end, + collocate, # Some Fourier layers ( @@ -59,59 +33,14 @@ function fno(; setup, kmax, c, σ, ψ, rng = Random.default_rng(), kwargs...) i ∈ eachindex(σ) )..., - # Put channels in first dimension - u -> permutedims(u, (D + 1, (1:D)..., D + 2)), - # Compress with a final dense layer - Dense(c[end] => 2 * c[end], ψ), - Dense(2 * c[end] => 2; use_bias = false), - - # Put channels back after spatial dimensions - u -> permutedims(u, ((2:D+1)..., 1, D + 2)), + Conv(ntuple(Returns(1), D), c[end] => D; use_bias = false), + # Conv(ntuple(Returns(1), D), c[end] => 2 * c[end], ψ; use_bias = false), + # Conv(ntuple(Returns(1), D), 2 * c[end] => D; use_bias = false), # Differentiate output to velocity points - function (u) - sz..., _, _ = size(u) - # for α = 1:D - # v = selectdim(u, D + 1, α) - # v = (v + circshift(v, ntuple(β -> α == β ? -1 : 0, D + 1))) / 2 - # end - if D == 2 - a = selectdim(u, 3, 1) - b = selectdim(u, 3, 2) - # a = (a + circshift(a, (1, 0, 0, 0))) / 2 - # b = (b + circshift(b, (0, 1, 0, 0))) / 2 - a = circshift(a, (1, 0, 0)) - a - b = circshift(b, (0, 1, 0)) - b - a = reshape(a, sz..., 1, :) - b = reshape(b, sz..., 1, :) - cat(a, b; dims = 3) - elseif D == 3 - a = selectdim(u, 4, 1) - b = selectdim(u, 4, 2) - c = selectdim(u, 4, 3) - # a = (a + circshift(a, (1, 0, 0, 0))) / 2 - # b = (b + circshift(b, (0, 1, 0, 0))) / 2 - # c = (c + circshift(c, (0, 0, 1, 0))) / 2 - a = circshift(a, (1, 0, 0, 0)) - a - b = circshift(b, (0, 1, 0, 0)) - b - c = circshift(c, (0, 0, 1, 0)) - c - a = reshape(a, sz..., 1, :) - b = reshape(b, sz..., 1, :) - c = reshape(c, sz..., 1, :) - cat(a, b, c; dims = 4) - end - end, + decollocate, ) - - # Create parameter vector (empty state) - params, state = Lux.setup(rng, NN) - θ = ComponentArray(params) - - # Compute closure term for given parameters - closure(u, θ) = first(NN(u, θ, state)) - - closure, θ end """ @@ -130,8 +59,8 @@ Some important sizes: - `kmax`: Cut-off wavenumber - `nsample`: Number of input samples (treated independently) """ -struct FourierLayer{N,A,F} <: Lux.AbstractExplicitLayer - dimension::Dimension{N} +struct FourierLayer{D,A,F} <: Lux.AbstractExplicitLayer + dimension::Dimension{D} kmax::Int cin::Int cout::Int @@ -177,7 +106,7 @@ function ((; dimension, kmax, cout, cin, σ)::FourierLayer)(x, params, state) # TODO: Set FFT normalization so that layer is truly grid independent # Spatial dimension - N = dimension() + D = dimension() nx..., _cin, nsample = size(x) @assert _cin == cin "Number of input channels must be compatible with weights" @@ -188,12 +117,12 @@ function ((; dimension, kmax, cout, cin, σ)::FourierLayer)(x, params, state) # The real and imaginary parts of R are stored in two separate channels W = params.spatial_weight R = params.spectral_weights - R = selectdim(R, N + 3, 1) .+ im .* selectdim(R, N + 3, 2) + R = selectdim(R, D + 3, 1) .+ im .* selectdim(R, D + 3, 2) # Spatial part (applied point-wise) - if N == 2 + if D == 2 @tullio y[i₁, i₂, b, s] := W[b, a] * x[i₁, i₂, a, s] - elseif N == 3 + elseif D == 3 @tullio y[i₁, i₂, i₃, b, s] := W[b, a] * x[i₁, i₂, i₃, a, s] end @@ -206,13 +135,13 @@ function ((; dimension, kmax, cout, cin, σ)::FourierLayer)(x, params, state) # - multiply with weights mode-wise # - pad with zeros to restore original shape # - go back to real valued spatial representation - ikeep = ntuple(Returns(1:kmax+1), N) - dims = ntuple(identity, N) + ikeep = ntuple(Returns(1:kmax+1), D) + dims = ntuple(identity, D) xhat = fft(x, dims) xhat = xhat[ikeep..., :, :] - if N == 2 + if D == 2 @tullio z[k₁, k₂, b, s] := R[k₁, k₂, b, a] * xhat[k₁, k₂, a, s] - elseif N == 3 + elseif D == 3 @tullio z[k₁, k₂, k₃, b, s] := R[k₁, k₂, k₃, b, a] * xhat[k₁, k₂, k₃, a, s] end z = pad_zeros(z, ntuple(i -> isodd(i) ? 0 : first(nx) - kmax - 1, 2N); dims) From edf648f563300f7676b9ee1609fd225f85ff73ae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Wed, 29 Nov 2023 15:34:42 +0100 Subject: [PATCH 168/379] Fix typo --- src/solvers/pressure/poisson.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/solvers/pressure/poisson.jl b/src/solvers/pressure/poisson.jl index 33bcf5ada..b68452f05 100644 --- a/src/solvers/pressure/poisson.jl +++ b/src/solvers/pressure/poisson.jl @@ -61,7 +61,7 @@ end function poisson!(solver::CGPressureSolver, p, f) (; setup, abstol, reltol, maxiter, r, L, q, preconditioner) = solver (; grid, workgroupsize) = setup - (; Np, Ip, Ω) = setup + (; Np, Ip, Ω) = grid T = typeof(reltol) function innerdot(a, b) From 21268aaf1e32774ac219fb527ee6076376f37cc1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Wed, 29 Nov 2023 15:41:07 +0100 Subject: [PATCH 169/379] docs: Add docstrings --- docs/src/features/closure.md | 4 ++++ src/closures/closure.jl | 20 ++++++++++++++++++++ 2 files changed, 24 insertions(+) diff --git a/docs/src/features/closure.md b/docs/src/features/closure.md index 8103b1572..6ff6016ec 100644 --- a/docs/src/features/closure.md +++ b/docs/src/features/closure.md @@ -80,6 +80,10 @@ create_callback We provide two neural architectures: A convolutional neural network (CNN) and a Fourier neural operator (FNO). ```@docs +wrappedclosure +create_closure +collocate +decollocate cnn fno FourierLayer diff --git a/src/closures/closure.jl b/src/closures/closure.jl index 4675395c9..3718299d0 100644 --- a/src/closures/closure.jl +++ b/src/closures/closure.jl @@ -1,3 +1,8 @@ +""" + wrappedclosure(m, θ, setup) + +Wrap closure model and parameters so that it can be used in the solver. +""" function wrappedclosure(m, θ, setup) (; dimension, Iu) = setup.grid D = dimension() @@ -12,6 +17,11 @@ function wrappedclosure(m, θ, setup) end end +""" + create_closure(layers...) + +Create neural closure model from layers. +""" function create_closure(layers...) chain = Chain(layers...) @@ -25,6 +35,11 @@ function create_closure(layers...) closure, θ end +""" + collocate(u) + +Interpolate velocity components to volume centers. +""" function collocate(u) sz..., D, _ = size(u) # for α = 1:D @@ -53,6 +68,11 @@ function collocate(u) end end +""" + decollocate(u) + +Interpolate closure force from volume centers to volume faces. +""" function decollocate(u) sz..., D, _ = size(u) # for α = 1:D From b8d5174348c29c6ab1673247f272c82d929a85c9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Wed, 29 Nov 2023 16:02:29 +0100 Subject: [PATCH 170/379] Do not assume 0 --- src/boundary_conditions.jl | 25 +++++++++++++++++++------ src/create_initial_conditions.jl | 1 - 2 files changed, 19 insertions(+), 7 deletions(-) diff --git a/src/boundary_conditions.jl b/src/boundary_conditions.jl index 2ba9416a0..a44bde52d 100644 --- a/src/boundary_conditions.jl +++ b/src/boundary_conditions.jl @@ -176,15 +176,15 @@ function apply_bc_u!(bc::DirichletBC, u, β, t, setup; atend, dudt = false, kwar (; dimension, x, xp, N) = setup.grid D = dimension() δ = Offset{D}() - isnothing(bc.u) && return + # isnothing(bc.u) && return bcfunc = dudt ? bc.dudt : bc.u for α = 1:D - if atend - I = CartesianIndices( + I = if atend + CartesianIndices( ntuple(γ -> γ == β ? α == β ? (N[γ]-1:N[γ]-1) : (N[γ]:N[γ]) : (1:N[γ]), D), ) else - I = CartesianIndices(ntuple(γ -> γ == β ? (1:1) : (1:N[γ]), D)) + CartesianIndices(ntuple(γ -> γ == β ? (1:1) : (1:N[γ]), D)) end xI = ntuple( γ -> reshape( @@ -195,7 +195,11 @@ function apply_bc_u!(bc::DirichletBC, u, β, t, setup; atend, dudt = false, kwar ), D, ) - u[α][I] .= bcfunc.((Dimension(α),), xI..., t) + if isnothing(bc.u) + u[α][I] .= 0 + else + u[α][I] .= bcfunc.((Dimension(α),), xI..., t) + end end u end @@ -277,6 +281,15 @@ function apply_bc_u!(bc::PressureBC, u, β, t, setup; atend, kwargs...) end function apply_bc_p!(bc::PressureBC, p, β, t, setup; atend, kwargs...) - # p is already zero at boundary + (; dimension, N) = setup.grid + D = dimension() + I = if atend + CartesianIndices( + ntuple(γ -> γ == β ? (N[γ]:N[γ]) : (1:N[γ]), D), + ) + else + CartesianIndices(ntuple(γ -> γ == β ? (2:2) : (1:N[γ]), D)) + end + p[I] .= 0 p end diff --git a/src/create_initial_conditions.jl b/src/create_initial_conditions.jl index b87bf89a1..5c1ad1b08 100644 --- a/src/create_initial_conditions.jl +++ b/src/create_initial_conditions.jl @@ -35,7 +35,6 @@ function create_initial_conditions( ) u[α][Iu[α]] .= initial_velocity.((Dimension(α),), xin...)[Iu[α]] end - apply_bc_u!(u, t, setup) # Make velocity field divergence free From a42f6b5bcb5fb7e6f8dfbed73cfe9ced18c3a9b5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Fri, 1 Dec 2023 18:00:16 +0100 Subject: [PATCH 171/379] Fix interpolation --- src/boundary_conditions.jl | 4 +--- src/grid/grid.jl | 2 +- src/operators.jl | 23 ++++++++++++++--------- 3 files changed, 16 insertions(+), 13 deletions(-) diff --git a/src/boundary_conditions.jl b/src/boundary_conditions.jl index a44bde52d..7001d95c3 100644 --- a/src/boundary_conditions.jl +++ b/src/boundary_conditions.jl @@ -284,9 +284,7 @@ function apply_bc_p!(bc::PressureBC, p, β, t, setup; atend, kwargs...) (; dimension, N) = setup.grid D = dimension() I = if atend - CartesianIndices( - ntuple(γ -> γ == β ? (N[γ]:N[γ]) : (1:N[γ]), D), - ) + CartesianIndices(ntuple(γ -> γ == β ? (N[γ]:N[γ]) : (1:N[γ]), D)) else CartesianIndices(ntuple(γ -> γ == β ? (2:2) : (1:N[γ]), D)) end diff --git a/src/grid/grid.jl b/src/grid/grid.jl index 9382e93ab..ee0ef73e1 100644 --- a/src/grid/grid.jl +++ b/src/grid/grid.jl @@ -115,7 +115,7 @@ function Grid(x, boundary_conditions; ArrayType = Array) Aαβ2[end] = 1 else # Interpolation from α-face center to left (1) or right (2) α-face β-edge - Aαβ1 = [(xp[β][i] - x[β][i]) / Δu[β][i-1] for i = 2:N[β]] + Aαβ1 = [(x[β][i] - xp[β][i-1]) / Δu[β][i-1] for i = 2:N[β]] Aαβ2 = 1 .- Aαβ1 pushfirst!(Aαβ1, 1) push!(Aαβ2, 1) diff --git a/src/operators.jl b/src/operators.jl index c2cff8517..9e5e6ebb9 100644 --- a/src/operators.jl +++ b/src/operators.jl @@ -126,10 +126,11 @@ function convection!(F, u, setup) KernelAbstractions.Extras.LoopInfo.@unroll for β in βrange Δuαβ = α == β ? Δu[β] : Δ[β] uαβ1 = A[α][β][2][I[β]-1] * u[α][I-δ(β)] + A[α][β][1][I[β]] * u[α][I] - uαβ2 = A[α][β][2][I[β]] * u[α][I] + A[α][β][1][I[β]+1] * u[α][I+δ(β)] uβα1 = - A[β][α][2][I[α]-1] * u[β][I-δ(β)] + A[β][α][1][I[α]+1] * u[β][I-δ(β)+δ(α)] - uβα2 = A[β][α][2][I[α]] * u[β][I] + A[β][α][1][I[α]] * u[β][I+δ(α)] + A[β][α][2][I[α]-(α==β)] * u[β][I-δ(β)] + + A[β][α][1][I[α]+(α!=β)] * u[β][I-δ(β)+δ(α)] + uαβ2 = A[α][β][2][I[β]] * u[α][I] + A[α][β][1][I[β]+1] * u[α][I+δ(β)] + uβα2 = A[β][α][2][I[α]] * u[β][I] + A[β][α][1][I[α]+1] * u[β][I+δ(α)] F[α][I] -= (uαβ2 * uβα2 - uαβ1 * uβα1) / Δuαβ[I[β]] end end @@ -140,6 +141,7 @@ function convection!(F, u, setup) end F end +convection(u, setup) = convection!(zero.(u), u, setup) """ diffusion!(F, u, setup) @@ -172,6 +174,7 @@ function diffusion!(F, u, setup) end F end +diffusion(u, setup) = diffusion!(zero.(u), u, setup) function convectiondiffusion!(F, u, setup) (; grid, workgroupsize, Re) = setup @@ -186,10 +189,11 @@ function convectiondiffusion!(F, u, setup) KernelAbstractions.Extras.LoopInfo.@unroll for β in βrange Δuαβ = α == β ? Δu[β] : Δ[β] uαβ1 = A[α][β][2][I[β]-1] * u[α][I-δ(β)] + A[α][β][1][I[β]] * u[α][I] - uαβ2 = A[α][β][2][I[β]] * u[α][I] + A[α][β][1][I[β]+1] * u[α][I+δ(β)] uβα1 = - A[β][α][2][I[α]-1] * u[β][I-δ(β)] + A[β][α][1][I[α]+1] * u[β][I-δ(β)+δ(α)] - uβα2 = A[β][α][2][I[α]] * u[β][I] + A[β][α][1][I[α]] * u[β][I+δ(α)] + A[β][α][2][I[α]-(α==β)] * u[β][I-δ(β)] + + A[β][α][1][I[α]+(α!=β)] * u[β][I-δ(β)+δ(α)] + uαβ2 = A[α][β][2][I[β]] * u[α][I] + A[α][β][1][I[β]+1] * u[α][I+δ(β)] + uβα2 = A[β][α][2][I[α]] * u[β][I] + A[β][α][1][I[α]+1] * u[β][I+δ(α)] uαuβ1 = uαβ1 * uβα1 uαuβ2 = uαβ2 * uβα2 ∂βuα1 = (u[α][I] - u[α][I-δ(β)]) / (β == α ? Δ[β][I[β]] : Δu[β][I[β]-1]) @@ -204,6 +208,7 @@ function convectiondiffusion!(F, u, setup) end F end +convectiondiffusion(u, setup) = convectiondiffusion!(zero.(u), u, setup) """ bodyforce!(F, u, setup) @@ -243,9 +248,9 @@ function momentum!(F, u, t, setup) for α = 1:D F[α] .= 0 end - diffusion!(F, u, setup) - convection!(F, u, setup) - # convectiondiffusion!(F, u, setup) + # diffusion!(F, u, setup) + # convection!(F, u, setup) + convectiondiffusion!(F, u, setup) bodyforce!(F, u, t, setup) if !isnothing(closure_model) m = closure_model(u) From e707a1a64cc7a8463a5b2a56018544d3bf8eb23b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Fri, 1 Dec 2023 18:00:56 +0100 Subject: [PATCH 172/379] Fix BC in matrix --- src/operators.jl | 59 +++++++++---------- .../step_explicit_runge_kutta.jl | 2 +- 2 files changed, 30 insertions(+), 31 deletions(-) diff --git a/src/operators.jl b/src/operators.jl index 9e5e6ebb9..3901174ef 100644 --- a/src/operators.jl +++ b/src/operators.jl @@ -360,53 +360,52 @@ function laplacian_mat(setup) ia = Ip[ntuple(β -> α == β ? (1:1) : (:), D)...][:] ib = Ip[ntuple(β -> α == β ? (Np[α]:Np[α]) : (:), D)...][:] for (aa, bb, j) in [(a, nothing, ia), (nothing, nothing, i), (nothing, b, ib)] - ja = if isnothing(aa) - j .- [δ(α)] + vala = @.(Ω[j] / Δ[α][getindex.(j, α)] / Δu[α][getindex.(j, α)-1]) + if isnothing(aa) + J = [J; j .- [δ(α)]; j] + I = [I; j; j] + val = [val; vala; -vala] elseif aa isa PressureBC - # The weight of the "left" BC is zero, but still needs a J inside Ip, so - # just set it to ia - ia + J = [J; j] + I = [I; j] + val = [val; -vala] elseif aa isa PeriodicBC - ib + J = [J; ib; j] + I = [I; j; j] + val = [val; vala; -vala] elseif aa isa SymmetricBC - ia + J = [J; ia; j] + I = [I; j; j] + val = [val; vala; -vala] elseif aa isa DirichletBC - # The weight of the "left" BC is zero, but still needs a J inside Ip, so - # just set it to ia - ia end - jb = if isnothing(bb) - j .+ [δ(α)] + + valb = @.(Ω[j] / Δ[α][getindex.(j, α)] / Δu[α][getindex.(j, α)]) + if isnothing(bb) + J = [J; j; j .+ [δ(α)]] + I = [I; j; j] + val = [val; -valb; valb] elseif bb isa PressureBC # The weight of the "right" BC is zero, but still needs a J inside Ip, so # just set it to ib - ib + J = [J; j] + I = [I; j] + val = [val; -valb] elseif bb isa PeriodicBC - ia + J = [J; j; ia] + I = [I; j; j] + val = [val; -valb; valb] elseif bb isa SymmetricBC - ib + J = [J; j; ib] + I = [I; j; j] + val = [val; -valb; valb] elseif bb isa DirichletBC - # The weight of the "right" BC is zero, but still needs a J inside Ip, so - # just set it to ib - ib end - J = [J; ja; j; jb] - I = [I; j; j; j] # val = vcat( # val, # map(I -> Ω[I] / Δ[α][I[α]] / Δu[α][I[α]-1], j), # map(I -> -Ω[I] / Δ[α][I[α]] * (1 / Δu[α][I[α]] + 1 / Δu[α][I[α]-1]), j), # map(I -> Ω[I] / Δ[α][I[α]] / Δu[α][I[α]], j), - # ) - val = vcat( - val, - @.(Ω[j] / Δ[α][getindex.(j, α)] / Δu[α][getindex.(j, α)-1]), - @.( - -Ω[j] / Δ[α][getindex.(j, α)] * - (1 / Δu[α][getindex.(j, α)] + 1 / Δu[α][getindex.(j, α)-1]) - ), - @.(Ω[j] / Δ[α][getindex.(j, α)] / Δu[α][getindex.(j, α)]), - ) end end # Go back to CPU, otherwise get following error: diff --git a/src/time_steppers/step_explicit_runge_kutta.jl b/src/time_steppers/step_explicit_runge_kutta.jl index fce21437d..390cd44e9 100644 --- a/src/time_steppers/step_explicit_runge_kutta.jl +++ b/src/time_steppers/step_explicit_runge_kutta.jl @@ -56,9 +56,9 @@ function timestep!(method::ExplicitRungeKuttaMethod, stepper, Δt; cache) # Solve the Poisson equation poisson!(pressure_solver, p, M) + apply_bc_p!(p, t, setup) # Compute pressure correction term - apply_bc_p!(p, t, setup) pressuregradient!(G, p, setup) # Update velocity current stage, which is now divergence free From 7dd6b471e306478c319206cc8c02f3ff398a1c6f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Fri, 1 Dec 2023 18:05:23 +0100 Subject: [PATCH 173/379] Use multithreading for CI --- .github/workflows/CI.yml | 2 +- docs/make.jl | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index c04fc2fa9..cb4fb67b1 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -55,7 +55,7 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - julia --project=docs -e ' + julia --project=docs -t auto -e ' using Documenter: DocMeta, doctest using IncompressibleNavierStokes DocMeta.setdocmeta!( diff --git a/docs/make.jl b/docs/make.jl index bb034f57e..30ed82c1c 100644 --- a/docs/make.jl +++ b/docs/make.jl @@ -5,6 +5,8 @@ if isdefined(@__MODULE__, :LanguageServer) using .IncompressibleNavierStokes end +@info "" Threads.nthreads() + using IncompressibleNavierStokes using Literate using Documenter From e16b2a58b85505405b2f1ee503d8a84c07272074 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Fri, 1 Dec 2023 19:07:36 +0100 Subject: [PATCH 174/379] Put examples back --- docs/make.jl | 20 +++++++-------- examples/BackwardFacingStep2D.jl | 4 +-- examples/PlanarMixing2D.jl | 2 +- examples/PlaneJets2D.jl | 43 ++++++++++++++++---------------- 4 files changed, 34 insertions(+), 35 deletions(-) diff --git a/docs/make.jl b/docs/make.jl index 30ed82c1c..ff5b6fef6 100644 --- a/docs/make.jl +++ b/docs/make.jl @@ -25,16 +25,16 @@ bib = CitationBibliography(joinpath(@__DIR__, "references.bib")) examples = [ "Tutorial: Lid-Driven Cavity (2D)" => "LidDrivenCavity2D", "Convergence: Taylor-Green Vortex (2D)" => "TaylorGreenVortex2D", - # "Unsteady inflow: Actuator (2D)" => "Actuator2D", - # # "Actuator (3D)" => "Actuator3D", - # "Walls: Backward Facing Step (2D)" => "BackwardFacingStep2D", - # # "Backward Facing Step (3D)" => "BackwardFacingStep3D", - # "Decaying Turbulunce (2D)" => "DecayingTurbulence2D", - # # "Decaying Turbulunce (3D)" => "DecayingTurbulence3D", - # # "Lid-Driven Cavity (3D)" => "LidDrivenCavity3D", - # "Planar Mixing (2D)" => "PlanarMixing2D", - # # "Shear Layer (2D)" => "ShearLayer2D", - # # "Taylor-Green Vortex (3D)" => "TaylorGreenVortex3D", + "Unsteady inflow: Actuator (2D)" => "Actuator2D", + # "Actuator (3D)" => "Actuator3D", + "Walls: Backward Facing Step (2D)" => "BackwardFacingStep2D", + # "Backward Facing Step (3D)" => "BackwardFacingStep3D", + "Decaying Turbulunce (2D)" => "DecayingTurbulence2D", + # "Decaying Turbulunce (3D)" => "DecayingTurbulence3D", + # "Lid-Driven Cavity (3D)" => "LidDrivenCavity3D", + "Planar Mixing (2D)" => "PlanarMixing2D", + # "Shear Layer (2D)" => "ShearLayer2D", + # "Taylor-Green Vortex (3D)" => "TaylorGreenVortex3D", ] output = "generated" diff --git a/examples/BackwardFacingStep2D.jl b/examples/BackwardFacingStep2D.jl index 452aeda2a..00638e6c8 100644 --- a/examples/BackwardFacingStep2D.jl +++ b/examples/BackwardFacingStep2D.jl @@ -59,7 +59,7 @@ plotgrid(x, y) # Build setup and assemble operators setup = Setup(x, y; Re, boundary_conditions, ArrayType); -pressure_solver = CGPressureSolver(setup); +pressure_solver = DirectPressureSolver(setup); # Initial conditions (extend inflow) u₀, p₀ = @@ -69,7 +69,7 @@ u₀, p₀ = ## u, p = solve_steady_state(setup, u₀, p₀); # Solve unsteady problem -u, p, outputs = solve_unsteady( +state, outputs = solve_unsteady( setup, u₀, p₀, diff --git a/examples/PlanarMixing2D.jl b/examples/PlanarMixing2D.jl index 5ac11884d..289cb4c97 100644 --- a/examples/PlanarMixing2D.jl +++ b/examples/PlanarMixing2D.jl @@ -42,7 +42,7 @@ boundary_conditions = ( (DirichletBC(U, dUdt), PressureBC()), ## y rear, y front - (SymmetricBC(), SymmetricBC()), + (PressureBC(), PressureBC()), ) # A 2D grid is a Cartesian product of two vectors diff --git a/examples/PlaneJets2D.jl b/examples/PlaneJets2D.jl index f8acba66b..dcb8b0927 100644 --- a/examples/PlaneJets2D.jl +++ b/examples/PlaneJets2D.jl @@ -105,35 +105,31 @@ u₀, p₀ = create_initial_conditions( ); # Real time plot: Streamwise average and spectrum -function meanplot(; setup, state) - (; Ip) = setup.grid - - umean = @lift begin - (; u, p, t) = $state - up = IncompressibleNavierStokes.interpolate_u_p(u, setup) - u1 = u[1] - reshape(sum(u1[Ip]; dims = 1), :) ./ size(u1, 1) ./ V() +function meanplot(state; setup) + (; xp, Iu, Ip, Nu, N) = setup.grid + + umean = lift(state) do (; u, p, t) + reshape(sum(u[1][Iu[1]]; dims = 1), :) ./ Nu[1][1] ./ V() end - K = size(Ip, 1) ÷ 2 + K = Nu[1][2] ÷ 2 k = 1:(K-1) # Find energy spectrum where y = 0 - n₀ = size(Ip, 2) ÷ 2 - E₀ = @lift begin - (; u, p, t) = $state + n₀ = findmin(abs, xp[2])[2] + E₀ = lift(state) do (; u, p, t) u_y = u[1][:, n₀] abs.(fft(u_y .^ 2))[k.+1] end + y₀ = xp[2][n₀] # Find energy spectrum where y = 1 - n₁ = argmin(n -> abs(yin[n] .- 1), 1:Nuy_in) - E₁ = @lift begin - (; V, p, t) = $state - u = V[indu] - u_y = reshape(u, size(yu))[:, n₁] + n₁ = findmin(y -> abs(y - 1), xp[2])[2] + E₁ = lift(state) do (; u, p, t) + u_y = u[1][:, n₁] abs.(fft(u_y .^ 2))[k.+1] end + y₁ = xp[2][n₁] fig = Figure() ax = Axis( @@ -142,7 +138,7 @@ function meanplot(; setup, state) xlabel = "y", ylabel = L"\langle u \rangle / U_0", ) - lines!(ax, yu[1, :], umean) + lines!(ax, xp[2][2:end-1], umean) ax = Axis( fig[1, 2]; title = "Streamwise energy spectrum", @@ -153,11 +149,12 @@ function meanplot(; setup, state) ) # ylims!(ax, (10^(0.0), 10^4.0)) ksub = k[10:end] - lines!(ax, ksub, 1000 .* ksub .^ (-3 / 5); label = L"k^{-3/5}") + # lines!(ax, ksub, 1000 .* ksub .^ (-5 / 3); label = L"k^{-5/3}") lines!(ax, ksub, 1e7 .* ksub .^ -3; label = L"k^{-3}") - scatter!(ax, k, E₀; label = "y = $(yin[n₀])") - scatter!(ax, k, E₁; label = "y = $(yin[n₁])") + scatter!(ax, k, E₀; label = "y = $y₀") + scatter!(ax, k, E₁; label = "y = $y₁") axislegend(ax; position = :lb) + # on(_ -> autolimits!(ax), E₁) fig end @@ -199,8 +196,10 @@ save_vtk(setup, state.u, state.p, "$output/solution") # Plot pressure fieldplot(state; setup, fieldname = :pressure) -# Plot velocity +# Plot initial velocity fieldplot((; u = u₀, p = p₀, t = T(0)); setup, fieldname = :velocity) + +# Plot final velocity fieldplot(state; setup, fieldname = :velocity) # Plot vorticity From 593f5cabbbe861eccf3a0504b3c1411975663d0c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Fri, 1 Dec 2023 19:58:55 +0100 Subject: [PATCH 175/379] Fix: Wrong cell separators --- examples/Actuator2D.jl | 4 ++-- examples/PlaneJets2D.jl | 16 ++++++++-------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/examples/Actuator2D.jl b/examples/Actuator2D.jl index 4250b0b30..f67b8570f 100644 --- a/examples/Actuator2D.jl +++ b/examples/Actuator2D.jl @@ -75,8 +75,8 @@ state, outputs = solve_unsteady( Δt = 0.05, processors = ( rtp = realtimeplotter(; setup, plot = fieldplot, nupdate = 1), - # ehist = realtimeplotter(; setup, plot = energy_history_plot, nupdate = 1), - # espec = realtimeplotter(; setup, plot = energy_spectrum_plot, nupdate = 1), + ## ehist = realtimeplotter(; setup, plot = energy_history_plot, nupdate = 1), + ## espec = realtimeplotter(; setup, plot = energy_spectrum_plot, nupdate = 1), ## anim = animator(; setup, path = "$output/vorticity.mkv", nupdate = 20), ## vtk = vtk_writer(; setup, nupdate = 10, dir = "$output", filename = "solution"), ## field = fieldsaver(; setup, nupdate = 10), diff --git a/examples/PlaneJets2D.jl b/examples/PlaneJets2D.jl index dcb8b0927..7b3e72ebb 100644 --- a/examples/PlaneJets2D.jl +++ b/examples/PlaneJets2D.jl @@ -115,7 +115,7 @@ function meanplot(state; setup) K = Nu[1][2] ÷ 2 k = 1:(K-1) - # Find energy spectrum where y = 0 + ## Find energy spectrum where y = 0 n₀ = findmin(abs, xp[2])[2] E₀ = lift(state) do (; u, p, t) u_y = u[1][:, n₀] @@ -123,7 +123,7 @@ function meanplot(state; setup) end y₀ = xp[2][n₀] - # Find energy spectrum where y = 1 + ## Find energy spectrum where y = 1 n₁ = findmin(y -> abs(y - 1), xp[2])[2] E₁ = lift(state) do (; u, p, t) u_y = u[1][:, n₁] @@ -147,14 +147,14 @@ function meanplot(state; setup) xlabel = L"k_x", ylabel = L"\hat{U}_{cl} / U_0", ) - # ylims!(ax, (10^(0.0), 10^4.0)) + ## ylims!(ax, (10^(0.0), 10^4.0)) ksub = k[10:end] - # lines!(ax, ksub, 1000 .* ksub .^ (-5 / 3); label = L"k^{-5/3}") + ## lines!(ax, ksub, 1000 .* ksub .^ (-5 / 3); label = L"k^{-5/3}") lines!(ax, ksub, 1e7 .* ksub .^ -3; label = L"k^{-3}") scatter!(ax, k, E₀; label = "y = $y₀") scatter!(ax, k, E₁; label = "y = $y₁") axislegend(ax; position = :lb) - # on(_ -> autolimits!(ax), E₁) + ## on(_ -> autolimits!(ax), E₁) fig end @@ -171,9 +171,9 @@ state, outputs = solve_unsteady( processors = ( rtp = realtimeplotter(; setup, - # plot = fieldplot, - # plot = energy_history_plot, - # plot = energy_spectrum_plot, + ## plot = fieldplot, + ## plot = energy_history_plot, + ## plot = energy_spectrum_plot, plot = meanplot, nupdate = 1, ), From 312ba1704e0b481463e17fbe3a42ca439f88a19a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Sat, 2 Dec 2023 21:57:43 +0100 Subject: [PATCH 176/379] Skip save steps --- src/closures/create_les_data.jl | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/closures/create_les_data.jl b/src/closures/create_les_data.jl index 17edf9053..fd2b3079e 100644 --- a/src/closures/create_les_data.jl +++ b/src/closures/create_les_data.jl @@ -49,7 +49,8 @@ _filter_saver(dns, les, comp, pressure_solver; nupdate = 1) = _t = fill(zero(eltype(x[1])), 0) _u = fill(Array.(Φu), 0) _c = fill(Array.(Φu), 0) - on(state) do (; u, p, t) + on(state) do (; u, p, t, n) + n % nupdate == 0 || return momentum!(F, u, t, dns) pressuregradient!(G, p, dns) for α = 1:D @@ -107,6 +108,7 @@ function create_les_data( tburn = T(0.1), tsim = T(0.1), Δt = T(1e-4), + savefreq = 1, ArrayType = Array, ic_params = (;), ) @@ -182,7 +184,7 @@ function create_les_data( (T(0), tsim); Δt, processors = ( - f = _filter_saver(_dns, _les, compression, pressure_solver_les), + f = _filter_saver(_dns, _les, compression, pressure_solver_les; nupdate = savefreq), # step_logger(; nupdate = 10), ), pressure_solver, From 8ea03af75ce61cbe4fe4ceaee4b9bc5e9fa09fe7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Sat, 2 Dec 2023 21:58:04 +0100 Subject: [PATCH 177/379] Correct size info --- src/closures/create_les_data.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/closures/create_les_data.jl b/src/closures/create_les_data.jl index fd2b3079e..e15982d51 100644 --- a/src/closures/create_les_data.jl +++ b/src/closures/create_les_data.jl @@ -140,7 +140,7 @@ function create_les_data( ) # @info "Generating $(Base.summarysize(filtered) / 1e6) Mb of LES data" - @info "Generating $(nsim * (nt + 1) * nles * 3 * 2 * length(bitstring(zero(T))) / 8 / 1e6) Mb of LES data" + @info "Generating $(nsim * (nt ÷ savefreq + 1) * nles^D * 3 * 2 * length(bitstring(zero(T))) / 8 / 1e6) Mb of LES data" for isim = 1:nsim # @info "Generating data for simulation $isim of $nsim" From 7643be36d2e5a74f6bd86b515dddff8e0006b1fa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Sat, 2 Dec 2023 21:59:15 +0100 Subject: [PATCH 178/379] Add missing RNG --- src/closures/closure.jl | 2 +- src/closures/cnn.jl | 4 +++- src/closures/fno.jl | 3 ++- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/src/closures/closure.jl b/src/closures/closure.jl index 3718299d0..00d22ef74 100644 --- a/src/closures/closure.jl +++ b/src/closures/closure.jl @@ -22,7 +22,7 @@ end Create neural closure model from layers. """ -function create_closure(layers...) +function create_closure(layers...; rng) chain = Chain(layers...) # Create parameter vector (empty state) diff --git a/src/closures/cnn.jl b/src/closures/cnn.jl index afcaeca3b..255a9f875 100644 --- a/src/closures/cnn.jl +++ b/src/closures/cnn.jl @@ -39,7 +39,7 @@ function cnn(; c = [D; c] # Create convolutional closure model - create_closure( + layers = ( # Put inputs in pressure points collocate, @@ -60,4 +60,6 @@ function cnn(; # Differentiate output to velocity points decollocate, ) + + create_closure(layers...; rng) end diff --git a/src/closures/fno.jl b/src/closures/fno.jl index db595f8b1..f5e103340 100644 --- a/src/closures/fno.jl +++ b/src/closures/fno.jl @@ -23,7 +23,7 @@ function fno(; setup, kmax, c, σ, ψ, rng = Random.default_rng(), kwargs...) init_weight(rng::AbstractRNG, dims...) = glorot_uniform(rng, T, dims...) # Create FNO closure model - create_closure( + layers = ( # Put inputs in pressure points collocate, @@ -41,6 +41,7 @@ function fno(; setup, kmax, c, σ, ψ, rng = Random.default_rng(), kwargs...) # Differentiate output to velocity points decollocate, ) + create_closure(layers...; rng) end """ From 2c46a4141730d2deaba2d928eb1ad3b725deea4e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Sat, 2 Dec 2023 22:00:01 +0100 Subject: [PATCH 179/379] Update format --- .JuliaFormatter.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/.JuliaFormatter.toml b/.JuliaFormatter.toml index fddb52631..c2ccc2df2 100644 --- a/.JuliaFormatter.toml +++ b/.JuliaFormatter.toml @@ -1,2 +1,3 @@ +whitespace_in_kwargs = true remove_extra_newlines = true trailing_comma = true From 45d7bdd989307185de4f015f2ab7198ab64d19cb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Sat, 2 Dec 2023 22:24:36 +0100 Subject: [PATCH 180/379] Add plot for individual components --- src/processors/real_time_plot.jl | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/processors/real_time_plot.jl b/src/processors/real_time_plot.jl index 685f3c1c3..d5eafedee 100644 --- a/src/processors/real_time_plot.jl +++ b/src/processors/real_time_plot.jl @@ -95,7 +95,10 @@ function fieldplot( xf = Array.(getindex.(setup.grid.xp, Ip.indices)) (; u, p, t) = state[] - _f = if fieldname == :velocity + _f = if fieldname in (1, 2) + up = interpolate_u_p(u, setup) + up[fieldname] + elseif fieldname == :velocity up = interpolate_u_p(u, setup) upnorm = zero(p) elseif fieldname == :vorticity @@ -108,7 +111,10 @@ function fieldplot( end _f = Array(_f)[Ip] field = lift(state) do (; u, p, t) - f = if fieldname == :velocity + f = if fieldname in (1, 2) + interpolate_u_p!(up, u, setup) + up[fieldname] + elseif fieldname == :velocity interpolate_u_p!(up, u, setup) map((u, v) -> √sum(u^2 + v^2), up...) @. upnorm = sqrt(up[1]^2 + up[2]^2) From d3cc3e52f993a40d7b2f4608115728ab59ddd381 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Sun, 3 Dec 2023 01:21:40 +0100 Subject: [PATCH 181/379] Create dataloader --- src/IncompressibleNavierStokes.jl | 2 +- src/closures/closure.jl | 2 +- src/closures/training.jl | 48 ++++++++++++++++++------------- 3 files changed, 30 insertions(+), 22 deletions(-) diff --git a/src/IncompressibleNavierStokes.jl b/src/IncompressibleNavierStokes.jl index 6dd06e890..99fd08519 100644 --- a/src/IncompressibleNavierStokes.jl +++ b/src/IncompressibleNavierStokes.jl @@ -122,7 +122,7 @@ export plotmat export cnn, fno, FourierLayer export train export mean_squared_error, relative_error -export create_randloss, create_callback, create_les_data, create_io_arrays +export createloss, createdataloader, create_callback, create_les_data, create_io_arrays export wrappedclosure # ODE methods diff --git a/src/closures/closure.jl b/src/closures/closure.jl index 00d22ef74..ce887e33e 100644 --- a/src/closures/closure.jl +++ b/src/closures/closure.jl @@ -6,7 +6,7 @@ Wrap closure model and parameters so that it can be used in the solver. function wrappedclosure(m, θ, setup) (; dimension, Iu) = setup.grid D = dimension() - function neural_closure(u) + function neuralclosure(u) u = stack(ntuple(α -> u[α][Iu[α]], D)) u = reshape(u, size(u)..., 1) # One sample mu = m(u, θ) diff --git a/src/closures/training.jl b/src/closures/training.jl index b0d3f0495..e53e2cb03 100644 --- a/src/closures/training.jl +++ b/src/closures/training.jl @@ -1,5 +1,23 @@ +""" + createdataloader(data; nuse = 50, device = identity) + +Create dataloader that uses a batch of `batchsize` random samples from +`data` at each evaluation. +The batch is moved to `device`. +""" +createdataloader(data; batchsize = 50, device = identity) = function dataloader() + x, y = data + nsample = size(x)[end] + d = ndims(x) + i = sort(shuffle(1:nsample)[1:batchsize]) + xuse = device(Array(selectdim(x, d, i))) + yuse = device(Array(selectdim(y, d, i))) + xuse, yuse +end + """ train( + dataloader, loss, opt, θ; @@ -8,13 +26,14 @@ callback = (i, θ) -> println("Iteration \$i of \$niter"), ) -Update parameters `θ` to minimize `loss(θ)` using the optimiser `opt` for -`niter` iterations. +Update parameters `θ` to minimize `loss(dataloader(), θ)` using the +optimiser `opt` for `niter` iterations. -Return the a new named tuple `(; opt, θ, callbackstate)` with updated state and -parameters. +Return the a new named tuple `(; opt, θ, callbackstate)` with +updated state and parameters. """ function train( + dataloader, loss, opt, θ; @@ -24,7 +43,8 @@ function train( callbackstate = nothing, ) for i = 1:niter - g = first(gradient(loss, θ)) + b = dataloader() + g = first(gradient(θ -> loss(b, θ), θ)) opt, θ = Optimisers.update(opt, θ, g) if i % ncallback == 0 callbackstate = callback(callbackstate, i, θ) @@ -34,25 +54,13 @@ function train( end """ - create_randloss(loss, f, x, y; nuse = size(x, 2), device = identity) + createloss(loss, f) -Create loss function `randloss(θ)` that uses a batch of `nuse` random samples from -`(x, y)` at each evaluation. +Wrap loss function `loss(batch, θ)`. The function `loss` should take inputs like `loss(f, x, y, θ)`. - -The batch is moved to `device` before the loss is evaluated. """ -function create_randloss(loss, f, x, y; nuse = 50, device = identity) - nsample = size(x)[end] - d = ndims(x) - function randloss(θ) - i = Zygote.@ignore sort(shuffle(1:nsample)[1:nuse]) - xuse = Zygote.@ignore device(Array(selectdim(x, d, i))) - yuse = Zygote.@ignore device(Array(selectdim(y, d, i))) - loss(f, xuse, yuse, θ) - end -end +createloss(loss, f) = ((x, y), θ) -> loss(f, x, y, θ) """ mean_squared_error(f, x, y, θ; normalize = y -> sum(abs2, y), λ = sqrt(eps(eltype(x)))) From 73318335c17400ad809e217d578f7b9fb7c48d94 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Sun, 3 Dec 2023 10:32:48 +0100 Subject: [PATCH 182/379] Rename --- src/closures/create_les_data.jl | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/src/closures/create_les_data.jl b/src/closures/create_les_data.jl index e15982d51..231a50c29 100644 --- a/src/closures/create_les_data.jl +++ b/src/closures/create_les_data.jl @@ -32,23 +32,25 @@ function gaussian_force( force end -_filter_saver(dns, les, comp, pressure_solver; nupdate = 1) = +_filter_saver(dns, les, compression, pressure_solver; nupdate = 1) = processor() do state (; dimension, x) = dns.grid T = eltype(x[1]) D = dimension() F = zero.(state[].u) G = zero.(state[].u) - Φu = zero.(face_average(state[].u, les, comp)) + Φu = zero.(face_average(state[].u, les, compression)) q = zero(pressure(pressure_solver, Φu, state[].t, les)) M = zero(q) ΦF = zero.(Φu) FΦ = zero.(Φu) GΦ = zero.(Φu) c = zero.(Φu) - _t = fill(zero(eltype(x[1])), 0) - _u = fill(Array.(Φu), 0) - _c = fill(Array.(Φu), 0) + results = (; + t = fill(zero(eltype(x[1])), 0), + u = fill(Array.(Φu), 0), + c = fill(Array.(Φu), 0), + ) on(state) do (; u, p, t, n) n % nupdate == 0 || return momentum!(F, u, t, dns) @@ -56,9 +58,9 @@ _filter_saver(dns, les, comp, pressure_solver; nupdate = 1) = for α = 1:D F[α] .-= G[α] end - face_average!(Φu, u, les, comp) + face_average!(Φu, u, les, compression) apply_bc_u!(Φu, t, les) - face_average!(ΦF, F, les, comp) + face_average!(ΦF, F, les, compression) momentum!(FΦ, Φu, t, les) apply_bc_u!(FΦ, t, les; dudt = true) divergence!(M, FΦ, les) @@ -71,12 +73,12 @@ _filter_saver(dns, les, comp, pressure_solver; nupdate = 1) = FΦ[α] .-= GΦ[α] c[α] .= ΦF[α] .- FΦ[α] end - push!(_t, t) - push!(_u, Array.(Φu)) - push!(_c, Array.(c)) + push!(results.t, t) + push!(results.u, Array.(Φu)) + push!(results.c, Array.(c)) end state[] = state[] # Save initial conditions - (; t = _t, u = _u, c = _c) + results end """ From ef917101864d58c096b2a961084e0285a80555a6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Mon, 4 Dec 2023 13:37:02 +0100 Subject: [PATCH 183/379] Add Smagorinsky model --- src/IncompressibleNavierStokes.jl | 1 + src/operators.jl | 147 ++++++++++++++++++++++++++++++ src/setup.jl | 1 + 3 files changed, 149 insertions(+) diff --git a/src/IncompressibleNavierStokes.jl b/src/IncompressibleNavierStokes.jl index 99fd08519..38c08688e 100644 --- a/src/IncompressibleNavierStokes.jl +++ b/src/IncompressibleNavierStokes.jl @@ -119,6 +119,7 @@ export plotgrid, save_vtk export plotmat # Closure models +export smagorinsky_closure export cnn, fno, FourierLayer export train export mean_squared_error, relative_error diff --git a/src/operators.jl b/src/operators.jl index 3901174ef..c0fc2ec9d 100644 --- a/src/operators.jl +++ b/src/operators.jl @@ -431,6 +431,153 @@ function laplacian_mat(setup) # Ω isa CuArray ? cu(L) : L end +# @inline function ∂x(uα, I, ::Val{α}, ::Val{β}, Δβ, Δuβ) where {α,β} +# D = length(I) +# δ = Offset{D}() +# α == β ? (uα[I] - uα[I-δ(β)]) / Δβ[I[β]] : +# ( +# (uα[I+δ(β)] - uα[I]) / Δuβ[I[β]] + +# (uα[I-δ(α)+δ(β)] - uα[I-δ(α)]) / Δuβ[I[β]] + +# (uα[I] - uα[I-δ(β)]) / Δuβ[I[β]-1] + +# (uα[I-δ(α)] - uα[I-δ(α)-δ(β)]) / Δuβ[I[β]-1] +# ) / 4 +# end +# @inline ∇(::Val{2}, u, I, Δ, Δu) = SMatrix{2,2,eltype(u[1]),4}( +# ∂x(u[1], I, Val(1), Val(1), Δ[1], Δu[1]), +# ∂x(u[2], I, Val(2), Val(1), Δ[1], Δu[1]), +# ∂x(u[1], I, Val(1), Val(2), Δ[2], Δu[2]), +# ∂x(u[2], I, Val(2), Val(2), Δ[2], Δu[2]), +# ) +# @inline ∇(::Val{3}, u, I, Δ, Δu) = SMatrix{3,3,eltype(u[1]),9}( +# ∂x(u[1], I, Val(1), Val(1), Δ[1], Δu[1]), +# ∂x(u[2], I, Val(2), Val(1), Δ[1], Δu[1]), +# ∂x(u[3], I, Val(3), Val(1), Δ[1], Δu[1]), +# ∂x(u[1], I, Val(1), Val(2), Δ[2], Δu[2]), +# ∂x(u[2], I, Val(2), Val(2), Δ[2], Δu[2]), +# ∂x(u[3], I, Val(3), Val(2), Δ[2], Δu[2]), +# ∂x(u[1], I, Val(1), Val(3), Δ[3], Δu[3]), +# ∂x(u[2], I, Val(2), Val(3), Δ[3], Δu[3]), +# ∂x(u[3], I, Val(3), Val(3), Δ[3], Δu[3]), +# ) +# +# @inline function strain(valD, u, I, Δ, Δu) +# ∇u = ∇(valD, u, I, Δ, Δu) +# (∇u + ∇u') / 2 +# end +# +# @inline gridsize(::Val{2}, Δ, I) = sqrt(Δ[1][I[1]]^2 + Δ[2][I[2]]^2) +# @inline gridsize(::Val{3}, Δ, I) = sqrt(Δ[1][I[1]]^2 + Δ[2][I[2]]^2 + Δ[3][I[3]]^2) + +function smagtensor!(σ, u, θ, setup) + # TODO: Combine with normal diffusion tensor + (; boundary_conditions, grid, workgroupsize) = setup + (; dimension, Np, Ip, Δ, Δu) = grid + D = dimension() + δ = Offset{D}() + @assert D == 2 + @kernel function σ!(σ, u, I0) + ∂x(uα, I, ::Val{α}, ::Val{β}, Δβ, Δuβ) where {α,β} = + α == β ? (uα[I] - uα[I-δ(β)]) / Δβ[I[β]] : + ( + (uα[I+δ(β)] - uα[I]) / Δuβ[I[β]] + + (uα[I-δ(α)+δ(β)] - uα[I-δ(α)]) / Δuβ[I[β]] + + (uα[I] - uα[I-δ(β)]) / Δuβ[I[β]-1] + + (uα[I-δ(α)] - uα[I-δ(α)-δ(β)]) / Δuβ[I[β]-1] + ) / 4 + I = @index(Global, Cartesian) + I = I + I0 + if D == 2 + ∇u = SMatrix{2,2,eltype(u[1]),4}( + ∂x(u[1], I, Val(1), Val(1), Δ[1], Δu[1]), + ∂x(u[2], I, Val(2), Val(1), Δ[1], Δu[1]), + ∂x(u[1], I, Val(1), Val(2), Δ[2], Δu[2]), + ∂x(u[2], I, Val(2), Val(2), Δ[2], Δu[2]), + ) + d = sqrt(Δ[1][I[1]]^2 + Δ[2][I[2]]^2) + elseif D == 3 + # TODO: Figure out why KA doesn't like I[3] when D == 2 + # ∇u = SMatrix{3,3,eltype(u[1]),9}( + # ∂x(u[1], I, Val(1), Val(1), Δ[1], Δu[1]), + # ∂x(u[2], I, Val(2), Val(1), Δ[1], Δu[1]), + # ∂x(u[3], I, Val(3), Val(1), Δ[1], Δu[1]), + # ∂x(u[1], I, Val(1), Val(2), Δ[2], Δu[2]), + # ∂x(u[2], I, Val(2), Val(2), Δ[2], Δu[2]), + # ∂x(u[3], I, Val(3), Val(2), Δ[2], Δu[2]), + # ∂x(u[1], I, Val(1), Val(3), Δ[3], Δu[3]), + # ∂x(u[2], I, Val(2), Val(3), Δ[3], Δu[3]), + # ∂x(u[3], I, Val(3), Val(3), Δ[3], Δu[3]), + # ) + # d = sqrt(Δ[1][I[1]]^2 + Δ[2][I[2]]^2 + Δ[3][I[3]]^2) + end + # S = strain(Val(D), u, I, Δ, Δu) + # d = gridsize(Val(D), Δ, I) + S = (∇u + ∇u') / 2 + # νt = θ^2 * d^2 * sqrt(2 * dot(S, S)) + νt = θ^2 * d^2 * sqrt(2 * sum(S .* S)) + σ[I] = 2 * νt * S + end + I0 = first(Ip) + I0 -= oneunit(I0) + σ!(get_backend(u[1]), workgroupsize)(σ, u, I0; ndrange = Np) + σ +end + +function smagorinsky!(s, σ, setup) + (; boundary_conditions, grid, workgroupsize) = setup + (; dimension, Nu, Iu, Δ, Δu, A) = grid + D = dimension() + δ = Offset{D}() + @kernel function s!(s, σ, ::Val{α}, ::Val{βrange}, I0) where {α,βrange} + I = @index(Global, Cartesian) + I = I + I0 + s[α][I] = 0 + # for β = 1:D + KernelAbstractions.Extras.LoopInfo.@unroll for β in βrange + Δuαβ = α == β ? Δu[β] : Δ[β] + if α == β + σαβ2 = σ[I+δ(β)][α, β] + σαβ1 = σ[I][α, β] + else + # TODO: Add interpolation weights for non-uniform case + σαβ2 = + ( + σ[I][α, β] + + σ[I+δ(β)][α, β] + + σ[I+δ(α)+δ(β)][α, β] + + σ[I+δ(α)][α, β] + ) / 4 + σαβ1 = + ( + σ[I-δ(β)][α, β] + + σ[I][α, β] + + σ[I+δ(α)-δ(β)][α, β] + + σ[I+δ(α)][α, β] + ) / 4 + end + s[α][I] += (σαβ2 - σαβ1) / Δuαβ[I[β]] + end + end + for α = 1:D + I0 = first(Iu[α]) + I0 -= oneunit(I0) + s!(get_backend(s[1]), workgroupsize)(s, σ, Val(α), Val(1:D), I0; ndrange = Nu[α]) + end + s +end + +function smagorinsky_closure(setup) + (; dimension, x, N) = setup.grid + D = dimension() + backend = get_backend(x[1]) + T = eltype(x[1]) + σ = KernelAbstractions.zeros(backend, SMatrix{D,D,T,D * D}, N) + s = ntuple(α -> KernelAbstractions.zeros(backend, T, N), D) + function closure(u, θ) + smagtensor!(σ, u, θ, setup) + smagorinsky!(s, σ, setup) + end +end + """ interpolate_u_p(u, setup) diff --git a/src/setup.jl b/src/setup.jl index 304d783d0..f2250ce91 100644 --- a/src/setup.jl +++ b/src/setup.jl @@ -29,5 +29,6 @@ Setup( bodyforce, closure_model, ArrayType, + T = eltype(x[1]), workgroupsize, ) From 08fbc479490623b08b0eb57a91060f8310a1343f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Mon, 4 Dec 2023 13:37:35 +0100 Subject: [PATCH 184/379] Add multigrid training --- src/closures/create_les_data.jl | 229 ++++++++++++++++---------------- src/closures/training.jl | 35 ++++- 2 files changed, 143 insertions(+), 121 deletions(-) diff --git a/src/closures/create_les_data.jl b/src/closures/create_les_data.jl index 231a50c29..35fcf2b2b 100644 --- a/src/closures/create_les_data.jl +++ b/src/closures/create_les_data.jl @@ -32,25 +32,53 @@ function gaussian_force( force end -_filter_saver(dns, les, compression, pressure_solver; nupdate = 1) = +function lesdatagen(dnsobs, les, compression, pressure_solver) + Φu = zero.(face_average(dnsobs[].u, les, compression)) + q = zero(pressure(pressure_solver, Φu, dnsobs[].t, les)) + M = zero(q) + ΦF = zero.(Φu) + FΦ = zero.(Φu) + GΦ = zero.(Φu) + c = zero.(Φu) + results = (; u = fill(Array.(dnsobs[].u), 0), c = fill(Array.(dnsobs[].u), 0)) + on(dnsobs) do (; u, F, t) + face_average!(Φu, u, les, compression) + apply_bc_u!(Φu, t, les) + face_average!(ΦF, F, les, compression) + momentum!(FΦ, Φu, t, les) + apply_bc_u!(FΦ, t, les; dudt = true) + divergence!(M, FΦ, les) + @. M *= les.grid.Ω + poisson!(pressure_solver, q, M) + apply_bc_p!(q, t, les) + pressuregradient!(GΦ, q, les) + for α = 1:length(u) + FΦ[α] .-= GΦ[α] + c[α] .= ΦF[α] .- FΦ[α] + end + push!(results.u, Array.(Φu)) + push!(results.c, Array.(c)) + end + results +end + +filtersaver(dns, les, compression, pressure_solver; nupdate = 1) = processor() do state (; dimension, x) = dns.grid T = eltype(x[1]) D = dimension() F = zero.(state[].u) G = zero.(state[].u) - Φu = zero.(face_average(state[].u, les, compression)) - q = zero(pressure(pressure_solver, Φu, state[].t, les)) - M = zero(q) - ΦF = zero.(Φu) - FΦ = zero.(Φu) - GΦ = zero.(Φu) - c = zero.(Φu) + dnsobs = Observable((; state[].u, F, state[].t)) + data = [ + lesdatagen(dnsobs, les[i], compression[i], pressure_solver[i]) for + i = 1:length(les) + ] results = (; - t = fill(zero(eltype(x[1])), 0), - u = fill(Array.(Φu), 0), - c = fill(Array.(Φu), 0), - ) + t = fill(zero(eltype(x[1])), 0), + u = [d.u for d in data], + c = [d.c for d in data], + ) on(state) do (; u, p, t, n) n % nupdate == 0 || return momentum!(F, u, t, dns) @@ -58,24 +86,8 @@ _filter_saver(dns, les, compression, pressure_solver; nupdate = 1) = for α = 1:D F[α] .-= G[α] end - face_average!(Φu, u, les, compression) - apply_bc_u!(Φu, t, les) - face_average!(ΦF, F, les, compression) - momentum!(FΦ, Φu, t, les) - apply_bc_u!(FΦ, t, les; dudt = true) - divergence!(M, FΦ, les) - @. M *= les.grid.Ω - - poisson!(pressure_solver, q, M) - apply_bc_p!(q, t, les) - pressuregradient!(GΦ, q, les) - for α = 1:D - FΦ[α] .-= GΦ[α] - c[α] .= ΦF[α] .- FΦ[α] - end push!(results.t, t) - push!(results.u, Array.(Φu)) - push!(results.c, Array.(c)) + dnsobs[] = (; u, F, t) end state[] = state[] # Save initial conditions results @@ -87,9 +99,8 @@ _filter_saver(dns, les, compression, pressure_solver; nupdate = 1) = D = 2, Re = T(2_000), lims = (T(0), T(1)), - nles = 64, + nles = [64], ndns = 256, - nsim = 10, tburn = T(0.1), tsim = T(0.1), Δt = T(1e-4), @@ -104,9 +115,8 @@ function create_les_data( D = 2, Re = T(2_000), lims = (T(0), T(1)), - nles = 64, + nles = [64], ndns = 256, - nsim = 10, tburn = T(0.1), tsim = T(0.1), Δt = T(1e-4), @@ -114,109 +124,94 @@ function create_les_data( ArrayType = Array, ic_params = (;), ) - compression = ndns ÷ nles - @assert compression * nles == ndns - - xdns = ntuple(α -> LinRange(lims..., ndns + 1), D) - xles = ntuple(α -> LinRange(lims..., nles + 1), D) + compression = @. ndns ÷ nles + @assert all(@.(compression * nles == ndns)) # Build setup and assemble operators - dns = Setup(xdns...; Re, ArrayType) - les = Setup(xles...; Re, ArrayType) + dns = Setup(ntuple(α -> LinRange(lims..., ndns + 1), D)...; Re, ArrayType) + les = [ + Setup(ntuple(α -> LinRange(lims..., nles + 1), D)...; Re, ArrayType) for + nles in nles + ] # Since the grid is uniform and identical for x and y, we may use a specialized # spectral pressure solver pressure_solver = SpectralPressureSolver(dns) - pressure_solver_les = SpectralPressureSolver(les) + pressure_solver_les = SpectralPressureSolver.(les) # Number of time steps to save nt = round(Int, tsim / Δt) Δt = tsim / nt - # Filtered quantities to store - (; N) = les.grid - filtered = (; + # datasize = Base.summarysize(filtered) / 1e6 + datasize = + (nt ÷ savefreq + 1) * sum(nles .^ D) * 3 * 2 * length(bitstring(zero(T))) / 8 / 1e6 + @info "Generating $datasize Mb of LES data" + + # Initial conditions + u₀, p₀ = random_field(dns, T(0); pressure_solver, ic_params...) + + # Random body force + # force_dns = + # gaussian_force(xdns...) + + # gaussian_force(xdns...) + + # # gaussian_force(xdns...) + + # # gaussian_force(xdns...) + + # gaussian_force(xdns...) + # force_dns = zero.(u₀) + # force_les = face_average(force_dns, les, compression) + + _dns = dns + _les = les + # _dns = (; dns..., bodyforce = force_dns) + # _les = (; les..., bodyforce = force_les) + + # Solve burn-in DNS + (; u, p, t), outputs = solve_unsteady(_dns, u₀, p₀, (T(0), tburn); Δt, pressure_solver) + + # Solve DNS and store filtered quantities + (; u, p, t), outputs = solve_unsteady( + _dns, + u, + p, + (T(0), tsim); Δt, - u = fill(fill(ntuple(α -> zeros(T, N...), D), 0), 0), - c = fill(fill(ntuple(α -> zeros(T, N...), D), 0), 0), - ) - - # @info "Generating $(Base.summarysize(filtered) / 1e6) Mb of LES data" - @info "Generating $(nsim * (nt ÷ savefreq + 1) * nles^D * 3 * 2 * length(bitstring(zero(T))) / 8 / 1e6) Mb of LES data" - - for isim = 1:nsim - # @info "Generating data for simulation $isim of $nsim" - - # Initial conditions - u₀, p₀ = random_field(dns, T(0); pressure_solver, ic_params...) - - # Random body force - # force_dns = - # gaussian_force(xdns...) + - # gaussian_force(xdns...) + - # # gaussian_force(xdns...) + - # # gaussian_force(xdns...) + - # gaussian_force(xdns...) - force_dns = zero.(u₀) - force_les = face_average(force_dns, les, compression) - - _dns = dns - _les = les - # _dns = (; dns..., bodyforce = force_dns) - # _les = (; les..., bodyforce = force_les) - - # Solve burn-in DNS - @info "Burn-in for simulation $isim of $nsim" - (; u, p, t), outputs = solve_unsteady( - _dns, - u₀, - p₀, - (T(0), tburn); - Δt, - # processors = (timelogger(; nupdate = 10),), - pressure_solver, - ) - - # Solve DNS and store filtered quantities - @info "Solving DNS for simulation $isim of $nsim" - (; u, p, t), outputs = solve_unsteady( - _dns, - u, - p, - (T(0), tsim); - Δt, - processors = ( - f = _filter_saver(_dns, _les, compression, pressure_solver_les; nupdate = savefreq), - # step_logger(; nupdate = 10), + processors = (; + f = filtersaver( + _dns, + _les, + compression, + pressure_solver_les; + nupdate = savefreq, ), - pressure_solver, - ) - - # Store result for current IC - push!(filtered.u, outputs.f.u) - push!(filtered.c, outputs.f.c) - end + ), + pressure_solver, + ) - filtered + # Store result for current IC + outputs[1] end """ - create_io_arrays(data, setup) + create_io_arrays(data, setups) Create ``(\\bar{u}, c)`` pairs for training. """ -function create_io_arrays(data, setup) - nsample = length(data.u) - nt = length(data.u[1]) - 1 - D = setup.grid.dimension() - T = eltype(data.u[1][1][1]) - (; N) = setup.grid - u = zeros(T, (N .- 2)..., D, nt + 1, nsample) - c = zeros(T, (N .- 2)..., D, nt + 1, nsample) - ifield = ntuple(Returns(:), D) - for i = 1:nsample, j = 1:nt+1, α = 1:D - copyto!(view(u, ifield..., α, j, i), view(data.u[i][j][α], setup.grid.Iu[α])) - copyto!(view(c, ifield..., α, j, i), view(data.c[i][j][α], setup.grid.Iu[α])) +function create_io_arrays(data, setups) + nsample = length(data) + ngrid = length(setups) + nt = length(data[1].u[1]) - 1 + T = eltype(data[1].u[1][1][1]) + map(1:ngrid) do ig + (; dimension, N, Iu) = setups[ig].grid + D = dimension() + u = zeros(T, (N .- 2)..., D, nt + 1, nsample) + c = zeros(T, (N .- 2)..., D, nt + 1, nsample) + ifield = ntuple(Returns(:), D) + for is = 1:nsample, it = 1:nt+1, α = 1:D + copyto!(view(u, ifield..., α, it, is), view(data[is].u[ig][it][α], Iu[α])) + copyto!(view(c, ifield..., α, it, is), view(data[is].c[ig][it][α], Iu[α])) + end + (; u = reshape(u, (N .- 2)..., D, :), c = reshape(c, (N .- 2)..., D, :)) end - reshape(u, (N .- 2)..., D, :), reshape(c, (N .- 2)..., D, :) end diff --git a/src/closures/training.jl b/src/closures/training.jl index e53e2cb03..b57cf084f 100644 --- a/src/closures/training.jl +++ b/src/closures/training.jl @@ -17,7 +17,7 @@ end """ train( - dataloader, + dataloaders, loss, opt, θ; @@ -33,7 +33,7 @@ Return the a new named tuple `(; opt, θ, callbackstate)` with updated state and parameters. """ function train( - dataloader, + dataloaders, loss, opt, θ; @@ -43,8 +43,10 @@ function train( callbackstate = nothing, ) for i = 1:niter - b = dataloader() - g = first(gradient(θ -> loss(b, θ), θ)) + g = sum(dataloaders) do d + b = d() + first(gradient(θ -> loss(b, θ), θ)) + end opt, θ = Optimisers.update(opt, θ, g) if i % ncallback == 0 callbackstate = callback(callbackstate, i, θ) @@ -80,6 +82,31 @@ Compute average column relative error between matrices `x` and `y`. relative_error(x, y) = sum(norm(x - y) / norm(y) for (x, y) ∈ zip(eachcol(x), eachcol(y))) / size(x, 2) +""" + relerr_trajectory(uref, setup) + +Processor to compute relative error between `uref` and `u` at each iteration. +""" +relerr_trajectory(uref, setup; nupdate = 1) = + processor() do state + (; dimension, x, Ip) = setup.grid + D = dimension() + T = eltype(x[1]) + e = Ref(T(0)) + on(state) do (; u, n) + n % nupdate == 0 || return + neff = n ÷ nupdate + a, b = T(0), T(0) + for α = 1:D + # @show size(uref[n + 1]) + a += sum(abs2, u[α][Ip] - uref[neff+1][α][Ip]) + b += sum(abs2, uref[neff+1][α][Ip]) + end + e[] += sqrt(a) / sqrt(b) / (length(uref) - 1) + end + e + end + """ create_callback( f, From 88316bca69d83c8f418bc9c62340f01673ec45a4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Mon, 4 Dec 2023 13:38:04 +0100 Subject: [PATCH 185/379] Add fixed length solver loop --- src/closures/cnn.jl | 5 ++--- src/solvers/solve_unsteady.jl | 38 ++++++++++++++++------------------- 2 files changed, 19 insertions(+), 24 deletions(-) diff --git a/src/closures/cnn.jl b/src/closures/cnn.jl index 255a9f875..a33f8fd88 100644 --- a/src/closures/cnn.jl +++ b/src/closures/cnn.jl @@ -22,14 +22,13 @@ function cnn(; rng = Random.default_rng(), ) r, c, σ, b = radii, channels, activations, use_bias - (; grid) = setup - (; dimension, x, Δu) = grid + (; T, grid) = setup + (; dimension) = grid D = dimension() dx = map(d -> d[2:end-1], Δu) # Weight initializer - T = eltype(x[1]) glorot_uniform_T(rng::AbstractRNG, dims...) = glorot_uniform(rng, T, dims...) # Make sure there are two force fields in output diff --git a/src/solvers/solve_unsteady.jl b/src/solvers/solve_unsteady.jl index 4e10ff0e6..601fb9941 100644 --- a/src/solvers/solve_unsteady.jl +++ b/src/solvers/solve_unsteady.jl @@ -9,7 +9,6 @@ Δt = zero(eltype(u₀[1])), cfl = 1, n_adapt_Δt = 1, - inplace = true, docopy = true, processors = (;), ) @@ -40,7 +39,6 @@ function solve_unsteady( Δt = zero(eltype(u₀[1])), cfl = 1, n_adapt_Δt = 1, - inplace = true, docopy = true, processors = (;), ) @@ -51,27 +49,19 @@ function solve_unsteady( t_start, t_end = tlims isadaptive = isnothing(Δt) - if !isadaptive - nstep = round(Int, (t_end - t_start) / Δt) - Δt = (t_end - t_start) / nstep - end - if inplace - cache = ode_method_cache(method, setup, u₀, p₀) - end + # Cache arrays for intermediate computations + cache = ode_method_cache(method, setup, u₀, p₀) # Time stepper stepper = create_stepper(method; setup, pressure_solver, u = u₀, p = p₀, t = t_start) - # Get initial time step - isadaptive && (Δt = get_timestep(stepper, cfl)) - # Initialize processors for iteration results state = Observable(get_state(stepper)) initialized = (; (k => v.initialize(state) for (k, v) in pairs(processors))...) - while stepper.t < t_end - if isadaptive + if isadaptive + while stepper.t < t_end if stepper.n % n_adapt_Δt == 0 # Change timestep based on operators Δt = get_timestep(stepper, cfl) @@ -79,17 +69,23 @@ function solve_unsteady( # Make sure not to step past `t_end` Δt = min(Δt, t_end - stepper.t) - end - # Perform a single time step with the time integration method - if inplace + # Perform a single time step with the time integration method stepper = timestep!(method, stepper, Δt; cache) - else - stepper = timestep(method, stepper, Δt) + + # Process iteration results with each processor + state[] = get_state(stepper) end + else + nstep = round(Int, (t_end - t_start) / Δt) + Δt = (t_end - t_start) / nstep + for it = 1:nstep + # Perform a single time step with the time integration method + stepper = timestep!(method, stepper, Δt; cache) - # Process iteration results with each processor - state[] = get_state(stepper) + # Process iteration results with each processor + state[] = get_state(stepper) + end end # Final state From a04205a2ab630328b89e5b11477d63b23f5902da Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Mon, 4 Dec 2023 13:39:19 +0100 Subject: [PATCH 186/379] Add multigrid experiments --- scratch/multigrid.jl | 549 +++++++++++++++++++++++------- scratch/train_model.jl | 226 ++++++------ src/IncompressibleNavierStokes.jl | 2 +- 3 files changed, 533 insertions(+), 244 deletions(-) diff --git a/scratch/multigrid.jl b/scratch/multigrid.jl index c3c9019e1..cdb8e65c4 100644 --- a/scratch/multigrid.jl +++ b/scratch/multigrid.jl @@ -10,6 +10,7 @@ end #src # Here, we consider a periodic box ``[0, 1]^2``. It is discretized with a # uniform Cartesian grid with square cells. +using CairoMakie using GLMakie using IncompressibleNavierStokes using JLD2 @@ -19,9 +20,18 @@ using NNlib using Optimisers using Random using Zygote +using SparseArrays +using KernelAbstractions +using FFTW + +GLMakie.activate!() set_theme!(; GLMakie = (; scalefactor = 1.5)) +# Random number generator +rng = Random.default_rng() +Random.seed!(rng, 123) + # Floating point precision T = Float64 @@ -46,33 +56,21 @@ get_params(nles) = (; Re = T(6_000), lims = (T(0), T(1)), tburn = T(0.05), - tsim = T(0.05), + tsim = T(0.5), Δt = T(1e-4), nles, - compression = 2048 ÷ nles, + ndns = 2048, ArrayType, - # ic_params = (; A = T(20_000_000), σ = T(5.0), s = T(3)), - # ic_params = (; A = T(10)), ) -# Create LES data from DNS -data_train = [create_les_data(T; get_params(nles)..., nsim = 5) for nles in [32, 64, 128]]; -data_valid = [create_les_data(T; get_params(nles)..., nsim = 1) for nles in [128]]; -ntest = [8, 16, 32, 64, 128, 256, 512] -data_test = [create_les_data(T; get_params(nles)..., nsim = 1) for nles in ntest]; -data_train = data_test +params_train = (; get_params([64, 128, 256])..., savefreq = 5); +params_valid = (; get_params([128])..., savefreq = 10); +params_test = (; get_params([32, 64, 128, 256, 512, 1024])..., tsim = T(0.1), savefreq = 5); -# Inspect data -g = 4 -j = 1 -α = 1 -data_train[g].u[j][1][α] -o = Observable(data_train[g].u[j][1][α]) -heatmap(o) -for i = 1:1:length(data_train[g].u[j]) - o[] = data_train[g].u[j][i][α] - sleep(0.001) -end +# Create LES data from DNS +data_train = [create_les_data(T; params_train...) for _ = 1:5]; +data_valid = [create_les_data(T; params_valid...) for _ = 1:1]; +data_test = create_les_data(T; params_test...); # # Save filtered DNS data # jldsave("output/forced/data.jld2"; data_train, data_valid, data_test) @@ -80,146 +78,443 @@ end # # Load previous LES data # data_train, data_valid, data_test = load("output/forced/data.jld2", "data_train", "data_valid", "data_test") -relerr_track(uref, setup) = - processor() do state - (; dimension, x, Ip) = setup.grid - D = dimension() - T = eltype(x[1]) - e = Ref(T(0)) - on(state) do (; u, n) - a, b = T(0), T(0) - for α = 1:D - # @show size(uref[n + 1]) - a += sum(abs2, u[α][Ip] - uref[n+1][α][Ip]) - b += sum(abs2, uref[n+1][α][Ip]) - end - e[] += sqrt(a) / sqrt(b) / (length(uref) - 1) - end - e - end +# Build LES setup and assemble operators +getsetups(params) = [ + Setup( + ntuple(α -> LinRange(params.lims..., nles + 1), params.D)...; + params.Re, + params.ArrayType, + ) for nles in params.nles +] +setups_train = getsetups(params_train); +setups_valid = getsetups(params_valid); +setups_test = getsetups(params_test); -e_nm = zeros(T, length(ntest)) -for (i, n) in enumerate(ntest) - params = get_params(n) - x = ntuple(α -> LinRange(params.lims..., params.nles + 1), params.D) - setup = Setup(x...; params.Re, ArrayType) - pressure_solver = SpectralPressureSolver(setup) - u = device.(data_test[i].u[1]) - u₀ = device(data_test[i].u[1][1]) - p₀ = pressure(pressure_solver, u₀, T(0), setup) - tlims = (T(0), params.tsim) - (; Δt) = data_test[i] - processors = (; relerr = relerr_track(u, setup)) - _, _, o = solve_unsteady(setup, u₀, p₀, tlims; Δt, pressure_solver, processors) - e_nm[i] = o.relerr[] - _, _, o = solve_unsteady(setup, u₀, p₀, tlims; Δt, pressure_solver, processors) - e_cnn[i] = o.relerr[] +# Create input/output arrays +io_train = create_io_arrays(data_train, setups_train); +io_valid = create_io_arrays(data_valid, setups_valid); + +# Inspect data +ig = 1 +field, setup = data_train[1].u[ig], setups_train[ig]; +# field, setup = data_valid[1].u[ig], setups_valid[ig]; +# field, setup = data_test.u[ig], setups_test[ig]; +u = device(field[1]); +o = Observable((; u, p = similar(u[1], setup.grid.N), t = nothing)); +fieldplot( + o; + setup, + # fieldname = :velocity, + # fieldname = 2, +) +# energy_spectrum_plot( o; setup,) +for i = 1:length(field) + o[] = (; o[]..., u = device(field[i])) + sleep(0.001) end -e_nm -e_cnn = ones(T, length(ntest)) -e_fno_share = ones(T, length(ntest)) -e_fno_spec = ones(T, length(ntest)) -using CairoMakie +GLMakie.activate!() CairoMakie.activate!() -# Plot convergence -with_theme(; - # linewidth = 5, - # markersize = 20, - # fontsize = 20, - palette = (; color = ["#3366cc", "#cc0000", "#669900", "#ffcc00"]), -) do +# Training data plot +fig = with_theme() do + sample = data_train[1] fig = Figure() + for (i, it) in enumerate((1, length(sample.t))) + for (j, ig) in enumerate((1, 2, 3)) + setup = setups_train[ig] + xf = Array.(getindex.(setup.grid.xp, setup.grid.Ip.indices)) + u = sample.u[ig][it] |> device + ωp = + IncompressibleNavierStokes.interpolate_ω_p( + IncompressibleNavierStokes.vorticity(u, setup), + setup, + )[setup.grid.Ip] |> Array + colorrange = IncompressibleNavierStokes.get_lims(ωp) + opts = (; + xticksvisible = false, + xticklabelsvisible = false, + yticklabelsvisible = false, + yticksvisible = false, + ) + i == 2 && ( + opts = (; + opts..., + xlabel = "x", + xticksvisible = true, + xticklabelsvisible = true, + ) + ) + j == 1 && ( + opts = + (; + opts..., + ylabel = "y", + yticklabelsvisible = true, + yticksvisible = true, + ) + ) + ax = Axis( + fig[i, j]; + opts..., + title = "n = $(params_train.nles[ig]), t = $(round(sample.t[it]; digits = 1))", + aspect = DataAspect(), + limits = (params_test.lims..., params_test.lims...), + ) + heatmap!(ax, xf..., ωp; colorrange) + end + end + fig +end + +save("training_data.pdf", fig) + +# Plot training spectra +fig = with_theme(; palette = (; color = ["#3366cc", "#cc0000", "#669900", "#ffcc00"])) do + ig = 3 + setup = setups_train[ig] + D = params_train.D + backend = KernelAbstractions.get_backend(setup.grid.x[1]) + sample = data_train[1] + K = size(setup.grid.Ip) .÷ 2 + kx = ntuple(α -> 1:K[α]-1, params_train.D) + k = zero(similar(setup.grid.x[1], length.(kx))) + for α = 1:D + kα = reshape(kx[α], ntuple(Returns(1), α - 1)..., :, ntuple(Returns(1), D - α)...) + k .+= kα .^ 2 + end + k .= sqrt.(k) + k = reshape(k, :) + # Make averaging matrix + naverage = 5^D + i = sortperm(k) + nbin, r = divrem(length(i), naverage) + ib = KernelAbstractions.zeros(backend, Int, nbin * naverage) + ia = KernelAbstractions.zeros(backend, Int, nbin * naverage) + for j = 1:naverage + copyto!(view(ia, (j-1)*nbin+1:j*nbin), collect(1:nbin)) + ib[(j-1)*nbin+1:j*nbin] = i[j:naverage:end-r] + end + vals = KernelAbstractions.ones(backend, T, nbin * naverage) / naverage + A = sparse(ia, ib, vals, nbin, length(i)) + k = Array(A * k) + # Build inertial slope above energy + @show length(k) + krange = collect(extrema(k[20:end])) + # slope, slopelabel = D == 2 ? (-T(3), L"$k^{-3}") : (-T(5 / 3), L"$k^{-5/3}") + slope, slopelabel = D == 2 ? (-T(3), "||k||₂⁻³") : (-T(5 / 3), L"k⁻⁵³") + slopeconst = T(0) + # Make plot + fig = Figure(; size = (500, 400)) ax = Axis( fig[1, 1]; + xlabel = "||k||₂", + ylabel = "e(k)", + title = "Kinetic energy (n = $(params_train.nles[ig]))", xscale = log10, yscale = log10, - xticks = ntest, - xlabel = "n", - title = "Relative error (DNS: n = 2048)", + xticks = [4, 8, 16, 32, 64, 128], + limits = (extrema(k)..., T(1e-8), T(1)), ) - scatterlines!(ntest, e_nm; label = "No closure") - scatterlines!(ntest, e_cnn; label = "CNN") - scatterlines!(ntest, e_cnn_share; label = "CNN (shared parameters)") - scatterlines!(ntest, e_fno_spec; label = "FNO (retrained)") - scatterlines!(ntest, e_fno_share; label = "FNO (shared parameters)") - lines!(collect(extrema(ntest)), n -> 100n^-2.0; linestyle = :dash, label = "n^-2") - axislegend(; position = :lb) + for (i, it) in enumerate((1, length(sample.t))) + u = device.(sample.u[ig][it]) + up = IncompressibleNavierStokes.interpolate_u_p(u, setup) + e = sum(up -> up[setup.grid.Ip] .^ 2, up) + ehat = Array( + A * reshape(abs.(fft(e)[ntuple(α -> kx[α] .+ 1, D)...]) ./ size(e, 1), :), + ) + slopeconst = max(slopeconst, maximum(ehat ./ k .^ slope)) + lines!(ax, k, ehat; label = "t = $(round(sample.t[it]; digits = 1))") + end + inertia = 2 .* slopeconst .* krange .^ slope + lines!(ax, krange, inertia; linestyle = :dash, label = slopelabel) + axislegend(ax; position = :lb) + autolimits!(ax) fig end -save("convergence.pdf", current_figure()) +save("training_spectra.pdf", fig) -closure, θ₀ = cnn( - setup, +closure, θ₀ = cnn(; + setup = setups_train[1], radii = [2, 2, 2, 2], channels = [5, 5, 5, params.D], activations = [leakyrelu, leakyrelu, leakyrelu, identity], use_bias = [true, true, true, false], ); -closure.NN - -# closure, θ₀ = fno( -# setup, -# -# # Cut-off wavenumbers -# [8, 8, 8, 8], -# -# # Channel sizes -# [8, 8, 8, 8], -# -# # Fourier layer activations -# [gelu, gelu, gelu, identity], -# -# # Dense activation -# gelu, -# ); -# closure.NN +closure.chain -# Create input/output arrays -io_train = create_io_arrays(data_train[end], setup); - -size(io_train[1]) +# Prepare training +loss = createloss(mean_squared_error, closure); +dataloaders = createdataloader.(io_train; batchsize = 50, device); +# dataloaders[1]() +loss(dataloaders[1](), θ) +it = rand(1:size(io_valid[1].u, 4), 50); +validset = map(v -> v[:, :, :, it], io_valid[1]); # Prepare training θ = T(1.0e-1) * device(θ₀); -# θ = device(θ₀); opt = Optimisers.setup(Adam(T(1.0e-3)), θ); callbackstate = Point2f[]; -randloss = create_randloss(mean_squared_error, closure, io_train...; nuse = 50, device); - -# Warm-up -randloss(θ) -@time randloss(θ); -first(gradient(randloss, θ)); -@time first(gradient(randloss, θ)); -GC.gc() -CUDA.reclaim() -# Training -# Note: The states `opt`, `θ`, and `callbackstate` -# will not be overwritten until training is finished. -# This allows for cancelling with "Control-C" should errors explode. +# Training with multiple grids at the same time (; opt, θ, callbackstate) = train( - randloss, + dataloaders, + loss, opt, θ; - niter = 50, - ncallback = 10, + niter = 1000, + ncallback = 20, callbackstate, - callback = create_callback(closure, device(io_valid)...; state = callbackstate), + callback = create_callback(closure, device(validset)...; state = callbackstate), ); GC.gc() CUDA.reclaim() -Array(θ) +# Extract parameters +θ_cnn_shared = θ; -# # Save trained parameters -# jldsave("output/forced/theta_cnn.jld2"; theta = Array(θ)) -# jldsave("output/forced/theta_fno.jld2"; theta = Array(θ)) +# Train grid-specialized closure models +θ_cnn = map(dataloaders) do d + # Prepare training + θ = T(1.0e-1) * device(θ₀) + opt = Optimisers.setup(Adam(T(1.0e-3)), θ) + callbackstate = Point2f[] -# # Load trained parameters -# θθ = load("output/theta_cnn.jld2") -# θθ = load("output/theta_fno.jld2") -# copyto!(θ, θθ["theta"]) + # Training with multiple grids at the same time + (; opt, θ, callbackstate) = train( + [d], + loss, + opt, + θ; + niter = 2000, + ncallback = 20, + callbackstate, + callback = create_callback(closure, device(validset)...; state = callbackstate), + ) + θ +end +GC.gc() +CUDA.reclaim() + +# Train Smagorinsky closure model +ig = 2; +setup = setups_train[ig]; +sample = data_train[1]; +m = smagorinsky_closure(setup); +θ = T(0.05) +e_smag = sum(2:length(sample.t)) do it + It = setup.grid.Ip + u = sample.u[ig][it] |> device; + c = sample.c[ig][it] |> device; + mu = m(u, θ) + e = zero(eltype(u[1])) + for α = 1:D + # e += sum(abs2, mu[α][Ip] .- c[α][Ip]) / sum(abs2, c[α][Ip]) + e += norm(mu[α][Ip] .- c[α][Ip]) / norm(c[α][Ip]) + end + e / D +end / length(sample.t) + # for θ in LinRange(T(0), T(1), 100)]; + +# lines(LinRange(T(0), T(1), 100), e_smag) + +# Errors for grid-specialized closure models +e_cnn = zeros(T, length(θ_cnn)) +i_traintest = 2:4 +offset_i = 1 +for (i, setup) in enumerate(setups_test[2:4]) + ig = i + offset_i + params = params_test[ig] + pressure_solver = SpectralPressureSolver(setup) + u = device.(data_test.u[ig]) + u₀ = device(data_test.u[ig][1]) + p₀ = IncompressibleNavierStokes.pressure(pressure_solver, u₀, T(0), setup) + Δt = params_test.Δt * params_test.savefreq + tlims = extrema(data_test.t) + nupdate = 4 + Δt /= nupdate + processors = (; relerr = relerr_trajectory(u, setup; nupdate)) + closedsetup = (; setup..., closure_model = wrappedclosure(closure, θ_cnn[i], setup)) + _, outputs = solve_unsteady(closedsetup, u₀, p₀, tlims; Δt, pressure_solver, processors) + e_cnn[i] = outputs.relerr[] +end + +# Errors for all test grids +e_nm = zeros(T, length(setups_test)) +e_smag = zeros(T, length(setups_test)) +e_cnn_shared = zeros(T, length(setups_test)) +for (ig, setup) in enumerate(setups_test) + @show ig + params = params_test[ig] + pressure_solver = SpectralPressureSolver(setup) + u = device.(data_test.u[ig]) + u₀ = device(data_test.u[ig][1]) + p₀ = IncompressibleNavierStokes.pressure(pressure_solver, u₀, T(0), setup) + Δt = params_test.Δt * params_test.savefreq + tlims = extrema(data_test.t) + nupdate = 4 + Δt /= nupdate + processors = (; relerr = relerr_trajectory(u, setup; nupdate)) + _, outputs = solve_unsteady(setup, u₀, p₀, tlims; Δt, pressure_solver, processors) + e_nm[ig] = outputs.relerr[] + m = smagorinsky_closure(setup) + closedsetup = (; setup..., closure_model = u -> m(u, T(0.1))) + _, outputs = solve_unsteady(closedsetup, u₀, p₀, tlims; Δt, pressure_solver, processors) + e_smag[ig] = outputs.relerr[] + closedsetup = (; setup..., closure_model = wrappedclosure(closure, θ_cnn_shared, setup)) + _, outputs = solve_unsteady(closedsetup, u₀, p₀, tlims; Δt, pressure_solver, processors) + e_cnn_shared[ig] = outputs.relerr[] +end + +e_nm +e_smag +e_cnn +e_cnn_shared +# e_cnn = ones(T, length(setups_test)) +# e_fno_shared = ones(T, length(setups_test)) +# e_fno_spec = ones(T, length(setups_test)) + +CairoMakie.activate!() +GLMakie.activate!() + +# Plot convergence +with_theme(; + # linewidth = 5, + # markersize = 10, + # markersize = 20, + # fontsize = 20, + palette = (; color = ["#3366cc", "#cc0000", "#669900", "#ffcc00"]), +) do + nles = params_test.nles + fig = Figure(; size = (500, 400)) + ax = Axis( + fig[1, 1]; + xscale = log10, + yscale = log10, + xticks = nles, + xlabel = "n", + title = "Relative error (DNS: n = $(params_test.ndns))", + ) + scatterlines!(nles, e_nm; marker = :circle, label = "No closure") + scatterlines!(nles, e_smag; marker = :utriangle, label = "Smagorinsky") + scatterlines!(nles, e_cnn_shared; marker = :diamond, label = "CNN (shared)") + scatterlines!(params_train.nles, e_cnn; marker = :rect, label = "CNN (specialized)") + # scatterlines!(nles, e_fno_spec; label = "FNO (retrained)") + # scatterlines!(nles, e_fno_share; label = "FNO (shared parameters)") + lines!( + collect(extrema(nles[3:end])), + n -> 2e4 * n^-2.0; + linestyle = :dash, + label = "n⁻²", + ) + axislegend(; position = :lb) + fig +end + +save("convergence.pdf", current_figure()) + +markers_labels = [ + (:circle, ":circle"), + (:rect, ":rect"), + (:diamond, ":diamond"), + (:hexagon, ":hexagon"), + (:cross, ":cross"), + (:xcross, ":xcross"), + (:utriangle, ":utriangle"), + (:dtriangle, ":dtriangle"), + (:ltriangle, ":ltriangle"), + (:rtriangle, ":rtriangle"), + (:pentagon, ":pentagon"), + (:star4, ":star4"), + (:star5, ":star5"), + (:star6, ":star6"), + (:star8, ":star8"), + (:vline, ":vline"), + (:hline, ":hline"), + ('a', "'a'"), + ('B', "'B'"), + ('↑', "'\\uparrow'"), + ('😄', "'\\:smile:'"), + ('✈', "'\\:airplane:'"), +] + +# Final spectra +ig = 4 +setup = setups_test[ig]; +params = params_test +pressure_solver = SpectralPressureSolver(setup); +uref = device(data_test.u[ig][end]); +u₀ = device(data_test.u[ig][1]); +p₀ = IncompressibleNavierStokes.pressure(pressure_solver, u₀, T(0), setup); +Δt = params_test.Δt * params_test.savefreq; +tlims = extrema(data_test.t); +nupdate = 4; +Δt /= nupdate; +state_nm, outputs = solve_unsteady(setup, u₀, p₀, tlims; Δt, pressure_solver); +m = smagorinsky_closure(setup); +closedsetup = (; setup..., closure_model = u -> m(u, T(0.1))); +state_smag, outputs = solve_unsteady(closedsetup, u₀, p₀, tlims; Δt, pressure_solver); +closedsetup = (; setup..., closure_model = wrappedclosure(closure, θ_cnn_shared, setup)); +state_cnn, outputs = solve_unsteady(closedsetup, u₀, p₀, tlims; Δt, pressure_solver); + +# Plot training spectra +fig = with_theme(; palette = (; color = ["#3366cc", "#cc0000", "#669900", "#ffcc00"])) do + D = params.D + backend = KernelAbstractions.get_backend(setup.grid.x[1]) + K = size(setup.grid.Ip) .÷ 2 + kx = ntuple(α -> 1:K[α]-1, params_train.D) + k = zero(similar(setup.grid.x[1], length.(kx))) + for α = 1:D + kα = reshape(kx[α], ntuple(Returns(1), α - 1)..., :, ntuple(Returns(1), D - α)...) + k .+= kα .^ 2 + end + k .= sqrt.(k) + k = reshape(k, :) + # Make averaging matrix + naverage = 5^D + i = sortperm(k) + nbin, r = divrem(length(i), naverage) + ib = KernelAbstractions.zeros(backend, Int, nbin * naverage) + ia = KernelAbstractions.zeros(backend, Int, nbin * naverage) + for j = 1:naverage + copyto!(view(ia, (j-1)*nbin+1:j*nbin), collect(1:nbin)) + ib[(j-1)*nbin+1:j*nbin] = i[j:naverage:end-r] + end + vals = KernelAbstractions.ones(backend, T, nbin * naverage) / naverage + A = sparse(ia, ib, vals, nbin, length(i)) + k = Array(A * k) + # Build inertial slope above energy + @show length(k) + krange = collect(extrema(k[20:end])) + # slope, slopelabel = D == 2 ? (-T(3), L"$k^{-3}") : (-T(5 / 3), L"$k^{-5/3}") + slope, slopelabel = D == 2 ? (-T(3), "||k||₂⁻³") : (-T(5 / 3), L"k⁻⁵³") + slopeconst = T(0) + # Make plot + fig = Figure(; size = (500, 400)) + ax = Axis( + fig[1, 1]; + xlabel = "||k||₂", + ylabel = "e(k)", + # title = "Kinetic energy (n = $(params.nles[ig])) at time t = $(round(data_test.t[end]; digits = 1))", + title = "Kinetic energy (n = $(params.nles[ig]))", + xscale = log10, + yscale = log10, + xticks = [4, 8, 16, 32, 64, 128], + limits = (extrema(k)..., T(1e-8), T(1)), + ) + for (u, label) in ((uref, "Reference"), (state_nm.u, "No closure"), (state_smag.u, "Smagorinsky"), (state_cnn.u, "CNN")) + up = IncompressibleNavierStokes.interpolate_u_p(u, setup) + e = sum(up -> up[setup.grid.Ip] .^ 2, up) + ehat = Array( + A * reshape(abs.(fft(e)[ntuple(α -> kx[α] .+ 1, D)...]) ./ size(e, 1), :), + ) + slopeconst = max(slopeconst, maximum(ehat ./ k .^ slope)) + lines!(ax, k, ehat; label) + end + inertia = 2 .* slopeconst .* krange .^ slope + lines!(ax, krange, inertia; linestyle = :dash, label = slopelabel) + axislegend(ax; position = :lb) + autolimits!(ax) + fig +end diff --git a/scratch/train_model.jl b/scratch/train_model.jl index 1f6e06d35..4d0a30292 100644 --- a/scratch/train_model.jl +++ b/scratch/train_model.jl @@ -20,6 +20,10 @@ using Optimisers using Random using Zygote +# Random number generator +rng = Random.default_rng() +Random.seed!(rng, 123) + set_theme!(; GLMakie = (; scalefactor = 1.5)) # Floating point precision @@ -41,24 +45,30 @@ CUDA.allowscalar(false); device = cu # Parameters +# nles = 50 +nles = 128 +# ndns = 200 +# nles = 256 params = (; D = 2, Re = T(6_000), lims = (T(0), T(1)), - nles = 128, - ndns = 1024, + nles = [nles], + ndns = 512, + # ndns = 1024, tburn = T(0.05), - tsim = T(0.2), + tsim = T(0.5), Δt = T(1e-4), + savefreq = 5, ArrayType, - # ic_params = (; A = T(20_000_000), σ = T(5.0), s = T(3)), - # ic_params = (; A = T(10_000_000)), ) # Create LES data from DNS -data_train = create_les_data(T; params..., nsim = 10); -data_valid = create_les_data(T; params..., nsim = 1); -data_test = create_les_data(T; params..., nsim = 1); +data_train = [create_les_data(T; params...) for _ = 1:3]; +data_valid = [create_les_data(T; params...) for _ = 1:1]; +data_test = [create_les_data(T; params..., tsim = T(0.2)) for _ = 1:1]; + +data_test[1].u[1][1][1] # # Save filtered DNS data # jldsave("output/forced/data.jld2"; data_train, data_valid, data_test) @@ -67,13 +77,16 @@ data_test = create_les_data(T; params..., nsim = 1); # data_train, data_valid, data_test = load("output/forced/data.jld2", "data_train", "data_valid", "data_test") # Build LES setup and assemble operators -x = ntuple(α -> LinRange(params.lims..., params.nles + 1), params.D) +x = ntuple(α -> LinRange(params.lims..., nles + 1), params.D) setup = Setup(x...; params.Re, ArrayType); +# Uniform periodic grid +pressure_solver = SpectralPressureSolver(setup); + # Inspect data (; Ip) = setup.grid; -field = data_train.u[1]; -α = 1 +field = data_train[1].u[1]; +α = 2 # j = 13 o = Observable(field[1][α][Ip]) # o = Observable(field[1][α][:, :, j]) @@ -84,55 +97,90 @@ for i = 1:length(field) sleep(0.001) end -# Uniform periodic grid -pressure_solver = SpectralPressureSolver(setup); +# Inspect data +field = data_train[1].u[1]; +u = device(field[1]) +o = Observable((; + u, + p = IncompressibleNavierStokes.pressure(pressure_solver, u, T(0), setup), + t = nothing, +)) +fieldplot( + o; + setup, + # fieldname = :velocity, + fieldname = 2, +) +for i = 1:length(field) + o[] = (; o[]..., u = device(field[i])) + sleep(0.001) +end # Create input/output arrays -io_train = create_io_arrays(data_train, setup); -io_valid = create_io_arrays(data_valid, setup); -io_test = create_io_arrays(data_test, setup); +io_train = create_io_arrays(data_train, [setup]); +io_valid = create_io_arrays(data_valid, [setup]); +io_test = create_io_arrays(data_test, [setup]); -size(io_train[1]) +size(io_train[1].u) +size(io_valid[1].u) +size(io_test[1].u) Base.summarysize(io_train) / 1e9 -# closure, θ₀ = cnn(; +closure, θ₀ = cnn(; + setup, + radii = [2, 2, 2, 2], + channels = [5, 5, 5, params.D], + activations = [leakyrelu, leakyrelu, leakyrelu, identity], + use_bias = [true, true, true, false], +); +closure.chain + +sample = io_train[1].u[:, :, :, 1:5] +closure(sample, θ₀) |> size + +θ₀.layer_5 +θ₀.layer_6 + +# closure, θ₀ = fno(; # setup, -# radii = [2, 2, 2, 2], -# channels = [5, 5, 5, params.D], -# activations = [leakyrelu, leakyrelu, leakyrelu, identity], -# use_bias = [true, true, true, false], +# kmax = [20, 20, 20, 20], +# c = [24, 12, 8, 8], +# # σ = [gelu, gelu, gelu, identity], +# σ = [leakyrelu, leakyrelu, leakyrelu, identity], +# # σ = [tanh, tanh, tanh, identity], +# # ψ = gelu, +# ψ = tanh, +# rng, # ); -# closure.NN # -# sample = io_train[1][:, :, :, 1:5] -# closure(sample, θ₀) |> size +# closure.chain # -# θ.layer_5 -# θ.layer_6 +# θ₀.layer_4.spectral_weights |> size -closure, θ₀ = fno(; - setup, - kmax = [8, 8, 8, 8], - c = [5, 5, 5, 5], - σ = [gelu, gelu, gelu, identity], - ψ = gelu, -); - -closure.NN +# θ₀ +# θ₀ = 2*θ₀ +# θ₀.layer_6 ./= 20 # Prepare training θ = T(1.0e-1) * device(θ₀); # θ = device(θ₀); +# θ = 2 * device(θ₀); opt = Optimisers.setup(Adam(T(1.0e-3)), θ); callbackstate = Point2f[]; -randloss = create_randloss(mean_squared_error, closure, io_train...; nuse = 50, device); +loss = createloss(mean_squared_error, closure); +dataloader = createdataloader(io_train[1]; batchsize = 50, device); +dataloader() +loss(dataloader(), θ) +it = rand(1:size(io_valid[1].u, 4), 50); +validset = map(v -> v[:, :, :, it], io_valid[1]); # Warm-up -randloss(θ) -@time randloss(θ); -first(gradient(randloss, θ)); -@time first(gradient(randloss, θ)); +loss(dataloader(), θ) +@time loss(dataloader(), θ); +b = dataloader() +first(gradient(θ -> loss(b, θ), θ)); +@time first(gradient(θ -> loss(b, θ), θ)); GC.gc() CUDA.reclaim() @@ -141,13 +189,14 @@ CUDA.reclaim() # will not be overwritten until training is finished. # This allows for cancelling with "Control-C" should errors explode. (; opt, θ, callbackstate) = train( - randloss, + dataloader, + loss, opt, θ; niter = 1000, - ncallback = 20, + ncallback = 10, callbackstate, - callback = create_callback(closure, device(io_valid)...; state = callbackstate), + callback = create_callback(closure, device(validset)...; state = callbackstate), ); GC.gc() CUDA.reclaim() @@ -174,38 +223,23 @@ function relerr(u, uref, setup) sqrt(a) / sqrt(b) end -relerr_track(uref, setup) = - processor() do state - (; dimension, x, Ip) = setup.grid - D = dimension() - T = eltype(x[1]) - e = Ref(T(0)) - on(state) do (; u, n) - a, b = T(0), T(0) - for α = 1:D - # @show size(uref[n + 1]) - a += sum(abs2, u[α][Ip] - uref[n+1][α][Ip]) - b += sum(abs2, uref[n+1][α][Ip]) - end - e[] += sqrt(a) / sqrt(b) / (length(uref) - 1) - end - e - end - u, u₀, p₀ = nothing, nothing, nothing -u = device.(data_test.u[1]); -u₀ = device(data_test.u[1][1]); +u = device.(data_test[1].u[1]); +u₀ = device(data_test[1].u[1][1]); p₀ = IncompressibleNavierStokes.pressure(pressure_solver, u₀, T(0), setup); +Δt = data_test[1].t[2] - data_test[1].t[1] +tlims = extrema(data_test[1].t) length(u) +length(data_test[1].t) state_nm, outputs = solve_unsteady( setup, u₀, p₀, - (T(0), params.tsim); - Δt = data_test.Δt, + tlims; + Δt, pressure_solver, - processors = (; relerr = relerr_track(u, setup), log = timelogger(; nupdate = 1)), + processors = (; relerr = relerr_trajectory(u, setup), log = timelogger(; nupdate = 1000)), ) relerr_nm = outputs.relerr[] @@ -213,16 +247,22 @@ state_cnn, outputs = solve_unsteady( (; setup..., closure_model = wrappedclosure(closure, θ, setup)), u₀, p₀, - (T(0), params.tsim); - Δt = data_test.Δt, + tlims; + Δt, pressure_solver, - processors = (relerr = relerr_track(u, setup), log = timelogger(; nupdate = 1)), + processors = (relerr = relerr_trajectory(u, setup), log = timelogger(; nupdate = 1)), ) relerr_cnn = outputs.relerr[] relerr_nm relerr_cnn +# dnm = relerr_nm +# dcnn = relerr_cnn + +dnm +dcnn + function energy_history(setup, state) (; Ωp) = setup.grid points = Point2f[] @@ -261,52 +301,6 @@ V_nm, p_nm, outputs_nm = solve_unsteady( ) ehist_nm = outputs_nm[2] -setup_fno = (; forcedsetup..., closure_model = V -> closure(V, θ)) -devsetup = device(setup_fno); -V_fno, p_fno, outputs_fno = solve_unsteady( - setup_fno, - data_test.V[:, 1, isample], - data_test.p[:, 1, isample], - (T(0), tsim); - Δt = T(2e-4), - processors = ( - field_plotter(devsetup; type = heatmap, nupdate = 1), - energy_history_writer(forcedsetup), - step_logger(; nupdate = 10), - ), - pressure_solver, - inplace = false, - device, - devsetup, -) -ehist_fno = outputs_fno[2] - -state = Observable((; V = data_train.V[:, 1, 1], p = data_train.p[:, 1, 1], t = T(0))) -ehist = energy_history(forcedsetup, state) -for i = 2:nt+1 - t = (i - 1) / T(nt - 1) * tsim - V = data_test.V[:, i, isample] - p = data_test.p[:, i, isample] - state[] = (; V, p, t) -end -ehist - -fig = Figure() -ax = Axis(fig[1, 1]; xlabel = "t", ylabel = "Kinetic energy") -lines!(ax, ehist; label = "Reference") -lines!(ax, ehist_nm; label = "No closure") -lines!(ax, ehist_fno; label = "FNO") -axislegend(ax) -fig - -save("output/train/energy.png", fig) - -V = data_train.V[:, end, isample] -p = data_train.p[:, end, isample] - -relative_error(V_nm, V) -relative_error(V_fno, V) - box = [ Point2f(0.72, 0.42), Point2f(0.81, 0.42), diff --git a/src/IncompressibleNavierStokes.jl b/src/IncompressibleNavierStokes.jl index 38c08688e..53adf6017 100644 --- a/src/IncompressibleNavierStokes.jl +++ b/src/IncompressibleNavierStokes.jl @@ -122,7 +122,7 @@ export plotmat export smagorinsky_closure export cnn, fno, FourierLayer export train -export mean_squared_error, relative_error +export mean_squared_error, relative_error, relerr_trajectory export createloss, createdataloader, create_callback, create_les_data, create_io_arrays export wrappedclosure From a729855ab5fd82cd21cfe5b2204dfd583d0461bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Tue, 5 Dec 2023 10:32:53 +0100 Subject: [PATCH 187/379] fix: Missing reference --- src/closures/cnn.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/closures/cnn.jl b/src/closures/cnn.jl index a33f8fd88..8cd8ff1b8 100644 --- a/src/closures/cnn.jl +++ b/src/closures/cnn.jl @@ -26,7 +26,7 @@ function cnn(; (; dimension) = grid D = dimension() - dx = map(d -> d[2:end-1], Δu) + # dx = map(d -> d[2:end-1], Δu) # Weight initializer glorot_uniform_T(rng::AbstractRNG, dims...) = glorot_uniform(rng, T, dims...) From e282113ba2bb1a7627ac21a2590c49b5b5c77737 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Tue, 5 Dec 2023 10:33:14 +0100 Subject: [PATCH 188/379] Add plot --- scratch/multigrid.jl | 142 ++++++++++++++++++++++++++++++++++------- scratch/train_model.jl | 5 +- 2 files changed, 122 insertions(+), 25 deletions(-) diff --git a/scratch/multigrid.jl b/scratch/multigrid.jl index cdb8e65c4..bfa916b9d 100644 --- a/scratch/multigrid.jl +++ b/scratch/multigrid.jl @@ -95,10 +95,10 @@ io_train = create_io_arrays(data_train, setups_train); io_valid = create_io_arrays(data_valid, setups_valid); # Inspect data -ig = 1 -field, setup = data_train[1].u[ig], setups_train[ig]; +ig = 5 +# field, setup = data_train[1].u[ig], setups_train[ig]; # field, setup = data_valid[1].u[ig], setups_valid[ig]; -# field, setup = data_test.u[ig], setups_test[ig]; +field, setup = data_test.u[ig], setups_test[ig]; u = device(field[1]); o = Observable((; u, p = similar(u[1], setup.grid.N), t = nothing)); fieldplot( @@ -146,13 +146,12 @@ fig = with_theme() do ) ) j == 1 && ( - opts = - (; - opts..., - ylabel = "y", - yticklabelsvisible = true, - yticksvisible = true, - ) + opts = (; + opts..., + ylabel = "y", + yticklabelsvisible = true, + yticksvisible = true, + ) ) ax = Axis( fig[i, j]; @@ -237,10 +236,10 @@ save("training_spectra.pdf", fig) closure, θ₀ = cnn(; setup = setups_train[1], - radii = [2, 2, 2, 2], - channels = [5, 5, 5, params.D], - activations = [leakyrelu, leakyrelu, leakyrelu, identity], - use_bias = [true, true, true, false], + radii = [2, 2, 2, 2, 2, 2], + channels = [32, 32, 32, 32, 32, params_train.D], + activations = [leakyrelu, leakyrelu, leakyrelu, leakyrelu, leakyrelu, identity], + use_bias = [true, true, true, true, true, false], ); closure.chain @@ -248,7 +247,7 @@ closure.chain loss = createloss(mean_squared_error, closure); dataloaders = createdataloader.(io_train; batchsize = 50, device); # dataloaders[1]() -loss(dataloaders[1](), θ) +loss(dataloaders[1](), device(θ₀)) it = rand(1:size(io_valid[1].u, 4), 50); validset = map(v -> v[:, :, :, it], io_valid[1]); @@ -274,6 +273,13 @@ CUDA.reclaim() # Extract parameters θ_cnn_shared = θ; +# # Save trained parameters +# jldsave("output/multigrid/theta_cnn_shared.jld2"; theta = Array(θ_cnn_shared)) + +# # Load trained parameters +# θθ = load("output/multigrid/theta_cnn_shared.jld2") +# copyto!.(θ_cnn_shared, θθ["theta"]) + # Train grid-specialized closure models θ_cnn = map(dataloaders) do d # Prepare training @@ -287,7 +293,7 @@ CUDA.reclaim() loss, opt, θ; - niter = 2000, + niter = 5000, ncallback = 20, callbackstate, callback = create_callback(closure, device(validset)...; state = callbackstate), @@ -297,6 +303,13 @@ end GC.gc() CUDA.reclaim() +# # Save trained parameters +# jldsave("output/multigrid/theta_cnn.jld2"; theta = Array.(θ_cnn)) + +# # Load trained parameters +# θθ = load("output/multigrid/theta_cnn.jld2") +# copyto!.(θ_cnn, θθ["theta"]) + # Train Smagorinsky closure model ig = 2; setup = setups_train[ig]; @@ -305,8 +318,8 @@ m = smagorinsky_closure(setup); θ = T(0.05) e_smag = sum(2:length(sample.t)) do it It = setup.grid.Ip - u = sample.u[ig][it] |> device; - c = sample.c[ig][it] |> device; + u = sample.u[ig][it] |> device + c = sample.c[ig][it] |> device mu = m(u, θ) e = zero(eltype(u[1])) for α = 1:D @@ -315,7 +328,7 @@ e_smag = sum(2:length(sample.t)) do it end e / D end / length(sample.t) - # for θ in LinRange(T(0), T(1), 100)]; +# for θ in LinRange(T(0), T(1), 100)]; # lines(LinRange(T(0), T(1), 100), e_smag) @@ -367,6 +380,9 @@ for (ig, setup) in enumerate(setups_test) e_cnn_shared[ig] = outputs.relerr[] end +GC.gc() +CUDA.reclaim() + e_nm e_smag e_cnn @@ -455,10 +471,11 @@ state_nm, outputs = solve_unsteady(setup, u₀, p₀, tlims; Δt, pressure_solve m = smagorinsky_closure(setup); closedsetup = (; setup..., closure_model = u -> m(u, T(0.1))); state_smag, outputs = solve_unsteady(closedsetup, u₀, p₀, tlims; Δt, pressure_solver); -closedsetup = (; setup..., closure_model = wrappedclosure(closure, θ_cnn_shared, setup)); +# closedsetup = (; setup..., closure_model = wrappedclosure(closure, θ_cnn_shared, setup)); +closedsetup = (; setup..., closure_model = wrappedclosure(closure, θ_cnn[ig-1], setup)); state_cnn, outputs = solve_unsteady(closedsetup, u₀, p₀, tlims; Δt, pressure_solver); -# Plot training spectra +# Plot predicted spectra fig = with_theme(; palette = (; color = ["#3366cc", "#cc0000", "#669900", "#ffcc00"])) do D = params.D backend = KernelAbstractions.get_backend(setup.grid.x[1]) @@ -472,7 +489,7 @@ fig = with_theme(; palette = (; color = ["#3366cc", "#cc0000", "#669900", "#ffcc k .= sqrt.(k) k = reshape(k, :) # Make averaging matrix - naverage = 5^D + naverage = 20^D i = sortperm(k) nbin, r = divrem(length(i), naverage) ib = KernelAbstractions.zeros(backend, Int, nbin * naverage) @@ -486,7 +503,7 @@ fig = with_theme(; palette = (; color = ["#3366cc", "#cc0000", "#669900", "#ffcc k = Array(A * k) # Build inertial slope above energy @show length(k) - krange = collect(extrema(k[20:end])) + krange = collect(extrema(k[3:end-5])) # slope, slopelabel = D == 2 ? (-T(3), L"$k^{-3}") : (-T(5 / 3), L"$k^{-5/3}") slope, slopelabel = D == 2 ? (-T(3), "||k||₂⁻³") : (-T(5 / 3), L"k⁻⁵³") slopeconst = T(0) @@ -503,7 +520,13 @@ fig = with_theme(; palette = (; color = ["#3366cc", "#cc0000", "#669900", "#ffcc xticks = [4, 8, 16, 32, 64, 128], limits = (extrema(k)..., T(1e-8), T(1)), ) - for (u, label) in ((uref, "Reference"), (state_nm.u, "No closure"), (state_smag.u, "Smagorinsky"), (state_cnn.u, "CNN")) + for (u, label) in ( + # (uref, "Reference"), + (state_nm.u, "No closure"), + (state_smag.u, "Smagorinsky"), + (state_cnn.u, "CNN (specialized)"), + (uref, "Reference"), + ) up = IncompressibleNavierStokes.interpolate_u_p(u, setup) e = sum(up -> up[setup.grid.Ip] .^ 2, up) ehat = Array( @@ -518,3 +541,74 @@ fig = with_theme(; palette = (; color = ["#3366cc", "#cc0000", "#669900", "#ffcc autolimits!(ax) fig end + +save("predicted_spectra.pdf", fig) + +# Plot spectrum errors +fig = with_theme(; palette = (; color = ["#3366cc", "#cc0000", "#669900", "#ffcc00"])) do + D = params.D + backend = KernelAbstractions.get_backend(setup.grid.x[1]) + K = size(setup.grid.Ip) .÷ 2 + kx = ntuple(α -> 1:K[α]-1, params_train.D) + k = zero(similar(setup.grid.x[1], length.(kx))) + for α = 1:D + kα = reshape(kx[α], ntuple(Returns(1), α - 1)..., :, ntuple(Returns(1), D - α)...) + k .+= kα .^ 2 + end + k .= sqrt.(k) + k = reshape(k, :) + # Make averaging matrix + naverage = 20^D + i = sortperm(k) + nbin, r = divrem(length(i), naverage) + ib = KernelAbstractions.zeros(backend, Int, nbin * naverage) + ia = KernelAbstractions.zeros(backend, Int, nbin * naverage) + for j = 1:naverage + copyto!(view(ia, (j-1)*nbin+1:j*nbin), collect(1:nbin)) + ib[(j-1)*nbin+1:j*nbin] = i[j:naverage:end-r] + end + vals = KernelAbstractions.ones(backend, T, nbin * naverage) / naverage + A = sparse(ia, ib, vals, nbin, length(i)) + k = Array(A * k) + # Build inertial slope above energy + @show length(k) + krange = collect(extrema(k[20:end])) + # slope, slopelabel = D == 2 ? (-T(3), L"$k^{-3}") : (-T(5 / 3), L"$k^{-5/3}") + slope, slopelabel = D == 2 ? (-T(3), "||k||₂⁻³") : (-T(5 / 3), L"k⁻⁵³") + slopeconst = T(0) + # Make plot + fig = Figure(; size = (500, 400)) + ax = Axis( + fig[1, 1]; + xlabel = "||k||₂", + ylabel = "e(k)", + # title = "Kinetic energy (n = $(params.nles[ig])) at time t = $(round(data_test.t[end]; digits = 1))", + title = "Relative energy error (n = $(params.nles[ig]))", + xscale = log10, + yscale = log10, + xticks = [4, 8, 16, 32, 64, 128], + limits = (extrema(k)..., T(1e-8), T(1)), + ) + up = IncompressibleNavierStokes.interpolate_u_p(uref, setup) + e = sum(up -> up[setup.grid.Ip] .^ 2, up) + eref = + Array(A * reshape(abs.(fft(e)[ntuple(α -> kx[α] .+ 1, D)...]) ./ size(e, 1), :)) + for (u, label) in ( + (state_nm.u, "No closure"), + (state_smag.u, "Smagorinsky"), + (state_cnn.u, "CNN (specialized)"), + ) + up = IncompressibleNavierStokes.interpolate_u_p(u, setup) + e = sum(up -> up[setup.grid.Ip] .^ 2, up) + ehat = Array( + A * reshape(abs.(fft(e)[ntuple(α -> kx[α] .+ 1, D)...]) ./ size(e, 1), :), + ) + ee = @. abs(ehat - eref) / abs(eref) + lines!(ax, k, ee; label) + end + axislegend(ax; position = :lt) + autolimits!(ax) + fig +end + +save("spectrum_error.pdf", fig) diff --git a/scratch/train_model.jl b/scratch/train_model.jl index 4d0a30292..3d915dd4d 100644 --- a/scratch/train_model.jl +++ b/scratch/train_model.jl @@ -239,7 +239,10 @@ state_nm, outputs = solve_unsteady( tlims; Δt, pressure_solver, - processors = (; relerr = relerr_trajectory(u, setup), log = timelogger(; nupdate = 1000)), + processors = (; + relerr = relerr_trajectory(u, setup), + log = timelogger(; nupdate = 1000), + ), ) relerr_nm = outputs.relerr[] From bb722253cea86d28747980cc13547ed97bcd5c1c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Mon, 11 Dec 2023 17:40:42 +0100 Subject: [PATCH 189/379] Update forcing --- src/operators.jl | 24 +++++++++++++++++------- src/setup.jl | 36 +++++++++++++++++++++++++----------- 2 files changed, 42 insertions(+), 18 deletions(-) diff --git a/src/operators.jl b/src/operators.jl index c0fc2ec9d..3442d87c6 100644 --- a/src/operators.jl +++ b/src/operators.jl @@ -211,26 +211,36 @@ end convectiondiffusion(u, setup) = convectiondiffusion!(zero.(u), u, setup) """ - bodyforce!(F, u, setup) + bodyforce!(F, u, t, setup) Compute body force. """ function bodyforce!(F, u, t, setup) - (; grid, workgroupsize, bodyforce) = setup + (; grid, workgroupsize, bodyforce, issteadybodyforce) = setup (; dimension, Δ, Δu, Nu, Iu, x, xp) = grid isnothing(bodyforce) && return F D = dimension() δ = Offset{D}() - @kernel function f!(F, force, ::Val{α}, t, I0) where {α} + @assert D == 2 + @kernel function f!(F, ::Val{α}, t, I0) where {α} I = @index(Global, Cartesian) I = I + I0 - F[α][I] += - force(Dimension(α), ntuple(β -> α == β ? x[β][1+I[β]] : xp[β][I[β]], D)..., t) + # xI = ntuple(β -> α == β ? x[β][1+I[β]] : xp[β][I[β]], D) + xI = ( + α == 1 ? x[1][1+I[1]] : xp[1][I[1]], + α == 2 ? x[2][1+I[2]] : xp[2][I[2]], + # α == 3 ? x[3][1+I[3]] : xp[3][I[3]], + ) + F[α][I] += bodyforce(Dimension(α), xI..., t) end for α = 1:D I0 = first(Iu[α]) I0 -= oneunit(I0) - f!(get_backend(F[1]), workgroupsize)(F, bodyforce, Val(α), t, I0; ndrange = Nu[α]) + if issteadybodyforce + F[α] .+= bodyforce[α] + else + f!(get_backend(F[1]), workgroupsize)(F, Val(α), t, I0; ndrange = Nu[α]) + end end F end @@ -309,7 +319,7 @@ pressuregradient(p, setup) = pressuregradient!( Compute Laplacian of pressure field (in-place version). """ function laplacian!(L, p, setup) - (; grid, workgroupsize) = setup + (; grid, workgroupsize, boundary_conditions) = setup (; dimension, Δ, Δu, N, Np, Ip, Ω) = grid D = dimension() δ = Offset{D}() diff --git a/src/setup.jl b/src/setup.jl index f2250ce91..f10dc3222 100644 --- a/src/setup.jl +++ b/src/setup.jl @@ -5,6 +5,7 @@ Re = convert(eltype(x[1]), 1_000), viscosity_model = LaminarModel(), bodyforce = nothing, + issteadybodyforce = true, closure_model = nothing, ArrayType = Array, workgroupsize = 64, @@ -12,23 +13,36 @@ Create setup. """ -Setup( +function Setup( x...; boundary_conditions = ntuple(d -> (PeriodicBC(), PeriodicBC()), length(x)), Re = convert(eltype(x[1]), 1_000), viscosity_model = LaminarModel(), bodyforce = nothing, + issteadybodyforce = true, closure_model = nothing, ArrayType = Array, workgroupsize = 64, -) = (; - grid = Grid(x, boundary_conditions; ArrayType), - boundary_conditions, - Re, - viscosity_model, - bodyforce, - closure_model, - ArrayType, - T = eltype(x[1]), - workgroupsize, ) + setup = (; + grid = Grid(x, boundary_conditions; ArrayType), + boundary_conditions, + Re, + viscosity_model, + bodyforce, + issteadybodyforce = false, + closure_model, + ArrayType, + T = eltype(x[1]), + workgroupsize, + ) + if !isnothing(bodyforce) && issteadybodyforce + (; dimension, x, N) = setup.grid + T = eltype(x[1]) + F = ntuple(α -> zero(similar(x[1], N)), dimension()) + u = ntuple(α -> zero(similar(x[1], N)), dimension()) + bodyforce = bodyforce!(F, u, T(0), setup) + setup = (; setup..., issteadybodyforce = true, bodyforce) + end + setup +end From c83e204afce99ecc2ef13b034acc9e27a5b45dcc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Mon, 11 Dec 2023 18:00:06 +0100 Subject: [PATCH 190/379] Make sure to preserve "unified memory" trait --- src/create_initial_conditions.jl | 16 +++++++--------- src/operators.jl | 18 +++++++++--------- src/processors/real_time_plot.jl | 8 ++++---- src/solvers/pressure/poisson.jl | 6 +----- src/solvers/pressure/solvers.jl | 6 +++--- 5 files changed, 24 insertions(+), 30 deletions(-) diff --git a/src/create_initial_conditions.jl b/src/create_initial_conditions.jl index 5c1ad1b08..9b5b9e08a 100644 --- a/src/create_initial_conditions.jl +++ b/src/create_initial_conditions.jl @@ -57,18 +57,16 @@ function create_initial_conditions( u, p end -function create_spectrum(N; A, σ, s, backend) - T = typeof(A) - D = length(N) +function create_spectrum(; setup, A, σ, s) + (; dimension, x, N) = setup.grid + T = eltype(x[1]) + D = dimension() K = N .÷ 2 k = ntuple( α -> reshape(1:K[α], ntuple(Returns(1), α - 1)..., :, ntuple(Returns(1), D - α)...), D, ) - a = KernelAbstractions.ones(backend, Complex{T}, K) - AT = typeof(a) - # k = AT.(Array{Complex{T}}.(k)) - # k = AT.(k) + a = fill!(similar(x[1], Complex{T}, K), 1) τ = T(2π) a .*= prod(N) * A / sqrt(τ^2 * 2σ^2) for α = 1:D @@ -106,13 +104,13 @@ function random_field( s = convert(eltype(setup.grid.x[1]), 5), pressure_solver = DirectPressureSolver(setup), ) - (; dimension, x, N, Ip, Ω) = setup.grid + (; dimension, x, Ip, Ω) = setup.grid D = dimension() T = eltype(x[1]) backend = get_backend(x[1]) # Create random velocity field - u = ntuple(α -> real.(ifft(create_spectrum(N; A, σ, s, backend))), D) + u = ntuple(α -> real.(ifft(create_spectrum(; setup, A, σ, s))), D) apply_bc_u!(u, t, setup) # Make velocity field divergence free diff --git a/src/operators.jl b/src/operators.jl index 3442d87c6..2d6ff5119 100644 --- a/src/operators.jl +++ b/src/operators.jl @@ -236,7 +236,7 @@ function bodyforce!(F, u, t, setup) for α = 1:D I0 = first(Iu[α]) I0 -= oneunit(I0) - if issteadybodyforce + if issteadybodyforce F[α] .+= bodyforce[α] else f!(get_backend(F[1]), workgroupsize)(F, Val(α), t, I0; ndrange = Nu[α]) @@ -360,9 +360,9 @@ function laplacian_mat(setup) δ = Offset{D}() Ia = first(Ip) Ib = last(Ip) - I = KernelAbstractions.zeros(backend, CartesianIndex{D}, 0) - J = KernelAbstractions.zeros(backend, CartesianIndex{D}, 0) - val = KernelAbstractions.zeros(backend, T, 0) + I = similar(x[1], CartesianIndex{D}, 0) + J = similar(x[1], CartesianIndex{D}, 0) + val = similar(x[1], 0) I0 = Ia - oneunit(Ia) for α = 1:D a, b = boundary_conditions[α] @@ -426,15 +426,15 @@ function laplacian_mat(setup) # J = J .- I0 I = I .- [I0] J = J .- [I0] - # linear = copyto!(KernelAbstractions.zeros(backend, Int, Np), collect(LinearIndices(Ip))) + # linear = copyto!(similar(x[1], Int, Np), collect(LinearIndices(Ip))) linear = LinearIndices(Ip) I = linear[I] J = linear[J] # Assemble on CPU, since CUDA overwrites instead of adding L = sparse(I, J, Array(val)) - # II = copyto!(KernelAbstractions.zeros(backend, Int, length(I)), I) - # JJ = copyto!(KernelAbstractions.zeros(backend, Int, length(J)), J) + # II = copyto!(similar(x[1], Int, length(I)), I) + # JJ = copyto!(similar(x[1], Int, length(J)), J) # sparse(II, JJ, val) L @@ -580,8 +580,8 @@ function smagorinsky_closure(setup) D = dimension() backend = get_backend(x[1]) T = eltype(x[1]) - σ = KernelAbstractions.zeros(backend, SMatrix{D,D,T,D * D}, N) - s = ntuple(α -> KernelAbstractions.zeros(backend, T, N), D) + σ = similar(x[1], SMatrix{D,D,T,D * D}, N) + s = ntuple(α -> similar(x[1], N), D) function closure(u, θ) smagtensor!(σ, u, θ, setup) smagorinsky!(s, σ, setup) diff --git a/src/processors/real_time_plot.jl b/src/processors/real_time_plot.jl index d5eafedee..953a49f7a 100644 --- a/src/processors/real_time_plot.jl +++ b/src/processors/real_time_plot.jl @@ -293,7 +293,7 @@ function energy_spectrum_plot(state; setup, naverage = 5^setup.grid.dimension()) D = dimension() K = size(Ip) .÷ 2 kx = ntuple(α -> 1:K[α]-1, D) - k = KernelAbstractions.zeros(backend, T, length.(kx)) + k = fill!(similar(xp[1], length.(kx)), 0) for α = 1:D kα = reshape(kx[α], ntuple(Returns(1), α - 1)..., :, ntuple(Returns(1), D - α)...) k .+= kα .^ 2 @@ -304,13 +304,13 @@ function energy_spectrum_plot(state; setup, naverage = 5^setup.grid.dimension()) # Make averaging matrix i = sortperm(k) nbin, r = divrem(length(i), naverage) - ib = KernelAbstractions.zeros(backend, Int, nbin * naverage) - ia = KernelAbstractions.zeros(backend, Int, nbin * naverage) + ib = similar(xp[1], Int, nbin * naverage) + ia = similar(xp[1], Int, nbin * naverage) for j = 1:naverage copyto!(view(ia, (j-1)*nbin+1:j*nbin), collect(1:nbin)) ib[(j-1)*nbin+1:j*nbin] = i[j:naverage:end-r] end - vals = KernelAbstractions.ones(backend, T, nbin * naverage) / naverage + k = fill!(similar(xp[1], nbin * naverage), T(1) / naverage) A = sparse(ia, ib, vals, nbin, length(i)) k = Array(A * k) diff --git a/src/solvers/pressure/poisson.jl b/src/solvers/pressure/poisson.jl index b68452f05..73d3a3c16 100644 --- a/src/solvers/pressure/poisson.jl +++ b/src/solvers/pressure/poisson.jl @@ -74,11 +74,7 @@ function poisson!(solver::CGPressureSolver, p, f) # d = zero(eltype(a)) I0 = first(Ip) I0 -= oneunit(I0) - d = KernelAbstractions.zeros( - get_backend(a), - eltype(a), - ntuple(Returns(1), length(I0)), - ) + d = fill!(similar(a, ntuple(Returns(1), length(I0))), 0), innerdot!(get_backend(a), workgroupsize)(d, a, b, I0; ndrange = Np) d[] end diff --git a/src/solvers/pressure/solvers.jl b/src/solvers/pressure/solvers.jl index 68687b4b0..c9c0e43ad 100644 --- a/src/solvers/pressure/solvers.jl +++ b/src/solvers/pressure/solvers.jl @@ -20,8 +20,8 @@ struct DirectPressureSolver{T,S,F,A} <: AbstractPressureSolver{T} # T = eltype(x[1]) T = Float64 backend = get_backend(x[1]) - # f = KernelAbstractions.zeros(backend, T, prod(Np)) - # p = KernelAbstractions.zeros(backend, T, prod(Np)) + # f = similar(x[1], prod(Np)) + # p = similar(x[1], prod(Np)) f = zeros(T, prod(Np) + 1) p = zeros(T, prod(Np) + 1) L = laplacian_mat(setup) @@ -184,7 +184,7 @@ function SpectralPressureSolver(setup) D, ) - Ahat = KernelAbstractions.zeros(get_backend(x[1]), Complex{T}, Np...) + Ahat = fill!(similar(x[1], Complex{T}, Np), 0) Tπ = T(π) # CUDA doesn't like pi for d = 1:D @. Ahat += sin(k[d] * Tπ / Np[d])^2 / Δx[d]^2 From c27ba652792d0bdb42784ef55c10383741d91cfb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Tue, 12 Dec 2023 13:48:47 +0100 Subject: [PATCH 191/379] Add rng --- scratch/multigrid.jl | 1 + 1 file changed, 1 insertion(+) diff --git a/scratch/multigrid.jl b/scratch/multigrid.jl index bfa916b9d..76786aeb9 100644 --- a/scratch/multigrid.jl +++ b/scratch/multigrid.jl @@ -240,6 +240,7 @@ closure, θ₀ = cnn(; channels = [32, 32, 32, 32, 32, params_train.D], activations = [leakyrelu, leakyrelu, leakyrelu, leakyrelu, leakyrelu, identity], use_bias = [true, true, true, true, true, false], + rng, ); closure.chain From 91dc1858e134174a9eb27cb591761859df28abf3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Tue, 12 Dec 2023 17:41:26 +0100 Subject: [PATCH 192/379] Update spectra --- docs/references.bib | 13 +++++++ src/processors/real_time_plot.jl | 67 ++++++++++++++++++++++---------- 2 files changed, 60 insertions(+), 20 deletions(-) diff --git a/docs/references.bib b/docs/references.bib index 0eaf9143b..19efae8a4 100644 --- a/docs/references.bib +++ b/docs/references.bib @@ -1,3 +1,16 @@ +@article{San2012, + title={High-order methods for decaying two-dimensional homogeneous isotropic turbulence}, + volume={63}, + ISSN={0045-7930}, + url={http://dx.doi.org/10.1016/j.compfluid.2012.04.006}, + DOI={10.1016/j.compfluid.2012.04.006}, + journal={Computers & Fluids}, + publisher={Elsevier BV}, + author={San, Omer and Staples, Anne E.}, + year={2012}, + month={jun}, + pages={105–127} +} @article{Beck2021, author = {Beck, Andrea and Kurz, Marius}, doi = {10.1002/gamm.202100002}, diff --git a/src/processors/real_time_plot.jl b/src/processors/real_time_plot.jl index 953a49f7a..0799ae98b 100644 --- a/src/processors/real_time_plot.jl +++ b/src/processors/real_time_plot.jl @@ -280,19 +280,25 @@ function energy_history_plot(state; setup) end """ - energy_spectrum_plot(state; setup, naverage = 5^setup.grid.dimension()) + energy_spectrum_plot(state; setup, doaverage = false) Create energy spectrum plot. -The energy modes are averaged over the `naverage` nearest modes. +The energy at a scalar wavenumber level ``\\kappa \\in \\mathbb{N}`` is defined by + +```math +\\hat{e}(\\kappa) = \\int_{\\kappa \\leq \\| k \\|_2 < \\kappa + 1} | \\hat{e}(k) | \\mathrm{d} k, +``` + +as in San and Staples [San2012](@cite). """ -function energy_spectrum_plot(state; setup, naverage = 5^setup.grid.dimension()) +function energy_spectrum_plot(state; setup) state isa Observable || (state = Observable(state)) (; dimension, xp, Ip) = setup.grid backend = get_backend(xp[1]) T = eltype(xp[1]) D = dimension() K = size(Ip) .÷ 2 - kx = ntuple(α -> 1:K[α]-1, D) + kx = ntuple(α -> 0:K[α]-1, D) k = fill!(similar(xp[1], length.(kx)), 0) for α = 1:D kα = reshape(kx[α], ntuple(Returns(1), α - 1)..., :, ntuple(Returns(1), D - α)...) @@ -301,48 +307,69 @@ function energy_spectrum_plot(state; setup, naverage = 5^setup.grid.dimension()) k .= sqrt.(k) k = reshape(k, :) - # Make averaging matrix - i = sortperm(k) - nbin, r = divrem(length(i), naverage) - ib = similar(xp[1], Int, nbin * naverage) - ia = similar(xp[1], Int, nbin * naverage) - for j = 1:naverage - copyto!(view(ia, (j-1)*nbin+1:j*nbin), collect(1:nbin)) - ib[(j-1)*nbin+1:j*nbin] = i[j:naverage:end-r] + # Sum or average wavenumbers between k and k+1 + kmax = minimum(K) - 1 + nk = ceil(Int, maximum(k)) + kint = 1:kmax + ia = similar(xp[1], Int, 0) + ib = sortperm(k) + vals = similar(xp[1], 0) + ksort = k[ib] + jprev = 2 # Do not include constant mode + for ki = 1:kmax + j = findfirst(>(ki+1), ksort) + isnothing(j) && (j = length(k) + 1) + ia = [ia; fill!(similar(ia, j - jprev), ki)] + # val = doaverage ? T(1) / (j - jprev) : T(1) + val = T(π) * ((ki+1)^2 - ki^2) / (j - jprev) + vals = [vals; fill!(similar(vals, j - jprev), val)] + jprev = j end - k = fill!(similar(xp[1], nbin * naverage), T(1) / naverage) - A = sparse(ia, ib, vals, nbin, length(i)) - k = Array(A * k) + ib = ib[2:jprev-1] + A = sparse(ia, ib, vals, kmax, length(k)) + # Energy up = interpolate_u_p(state[].u, setup) ehat = lift(state) do (; u, p, t) interpolate_u_p!(up, u, setup) e = sum(up -> up[Ip] .^ 2, up) - Array(A * reshape(abs.(fft(e)[ntuple(α -> kx[α] .+ 1, D)...]) ./ size(e, 1), :)) + e = fft(e)[ntuple(α -> kx[α] .+ 1, D)...] + e = abs.(e) ./ size(e, 1) + e = A * reshape(e, :) + e = max.(e, eps(T)) # Avoid log(0) + Array(e) end # Build inertial slope above energy - krange = LinRange(extrema(k)..., 100) + # krange = LinRange(extrema(kint)..., 100) + # krange = collect(extrema(kint)) + krange = [cbrt(T(kmax)), T(kmax)] slope, slopelabel = D == 2 ? (-T(3), L"$k^{-3}") : (-T(5 / 3), L"$k^{-5/3}") inertia = lift(ehat) do ehat - slopeconst = maximum(ehat ./ k .^ slope) + slopeconst = maximum(ehat ./ kint .^ slope) 2 .* slopeconst .* krange .^ slope end + # Nice ticks + logmax = round(Int, log2(kmax + 1)) + xticks = T(2) .^ (0:logmax) + fig = Figure() ax = Axis( fig[1, 1]; + xticks, xlabel = "k", ylabel = "e(k)", xscale = log10, yscale = log10, - limits = (extrema(k)..., T(1e-8), T(1)), + limits = (extrema(kint)..., T(1e-8), T(1)), ) - lines!(ax, k, ehat; label = "Kinetic energy") + lines!(ax, kint, ehat; label = "Kinetic energy") lines!(ax, krange, inertia; label = slopelabel, color = :red) axislegend(ax) # autolimits!(ax) on(e -> autolimits!(ax), ehat) + autolimits!(ax) fig end From d326336d5131be78e4425639f15f3e7dbd198203 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Tue, 12 Dec 2023 22:19:55 +0100 Subject: [PATCH 193/379] Update files --- scratch/multigrid.jl | 254 ++++++++++++++++++------------ src/IncompressibleNavierStokes.jl | 2 + src/operators.jl | 188 ++++++++++------------ src/solvers/pressure/poisson.jl | 40 +++-- src/solvers/pressure/solvers.jl | 84 ++++++++-- 5 files changed, 337 insertions(+), 231 deletions(-) diff --git a/scratch/multigrid.jl b/scratch/multigrid.jl index 76786aeb9..1ce6d92db 100644 --- a/scratch/multigrid.jl +++ b/scratch/multigrid.jl @@ -14,6 +14,7 @@ using CairoMakie using GLMakie using IncompressibleNavierStokes using JLD2 +using LaTeXStrings using LinearAlgebra using Lux using NNlib @@ -101,12 +102,13 @@ ig = 5 field, setup = data_test.u[ig], setups_test[ig]; u = device(field[1]); o = Observable((; u, p = similar(u[1], setup.grid.N), t = nothing)); -fieldplot( - o; - setup, - # fieldname = :velocity, - # fieldname = 2, -) +energy_spectrum_plot(o; setup) +# fieldplot( +# o; +# setup, +# # fieldname = :velocity, +# # fieldname = 2, +# ) # energy_spectrum_plot( o; setup,) for i = 1:length(field) o[] = (; o[]..., u = device(field[i])) @@ -172,58 +174,77 @@ save("training_data.pdf", fig) fig = with_theme(; palette = (; color = ["#3366cc", "#cc0000", "#669900", "#ffcc00"])) do ig = 3 setup = setups_train[ig] + (; xp, Ip) = setup.grid D = params_train.D - backend = KernelAbstractions.get_backend(setup.grid.x[1]) sample = data_train[1] - K = size(setup.grid.Ip) .÷ 2 - kx = ntuple(α -> 1:K[α]-1, params_train.D) - k = zero(similar(setup.grid.x[1], length.(kx))) + K = size(Ip) .÷ 2 + kx = ntuple(α -> 0:K[α]-1, D) + k = fill!(similar(xp[1], length.(kx)), 0) for α = 1:D kα = reshape(kx[α], ntuple(Returns(1), α - 1)..., :, ntuple(Returns(1), D - α)...) k .+= kα .^ 2 end k .= sqrt.(k) k = reshape(k, :) - # Make averaging matrix - naverage = 5^D - i = sortperm(k) - nbin, r = divrem(length(i), naverage) - ib = KernelAbstractions.zeros(backend, Int, nbin * naverage) - ia = KernelAbstractions.zeros(backend, Int, nbin * naverage) - for j = 1:naverage - copyto!(view(ia, (j-1)*nbin+1:j*nbin), collect(1:nbin)) - ib[(j-1)*nbin+1:j*nbin] = i[j:naverage:end-r] + # Sum or average wavenumbers between k and k+1 + nk = ceil(Int, maximum(k)) + # kmax = minimum(K) - 1 + kmax = nk - 1 + kint = 1:kmax + ia = similar(xp[1], Int, 0) + ib = sortperm(k) + vals = similar(xp[1], 0) + ksort = k[ib] + jprev = 2 # Do not include constant mode + for ki = 1:kmax + j = findfirst(>(ki+1), ksort) + isnothing(j) && (j = length(k) + 1) + ia = [ia; fill!(similar(ia, j - jprev), ki)] + # val = doaverage ? T(1) / (j - jprev) : T(1) + val = T(π) * ((ki+1)^2 - ki^2) / (j - jprev) + vals = [vals; fill!(similar(vals, j - jprev), val)] + jprev = j end - vals = KernelAbstractions.ones(backend, T, nbin * naverage) / naverage - A = sparse(ia, ib, vals, nbin, length(i)) - k = Array(A * k) + ib = ib[2:jprev-1] + A = sparse(ia, ib, vals, kmax, length(k)) # Build inertial slope above energy - @show length(k) - krange = collect(extrema(k[20:end])) - # slope, slopelabel = D == 2 ? (-T(3), L"$k^{-3}") : (-T(5 / 3), L"$k^{-5/3}") - slope, slopelabel = D == 2 ? (-T(3), "||k||₂⁻³") : (-T(5 / 3), L"k⁻⁵³") + krange = [T(kmax)^(T(2)/3), T(kmax)] + # slope, slopelabel = D == 2 ? (-T(3), L"k^{-3}") : (-T(5 / 3), L"k^{-5/3}") + # slope, slopelabel = D == 2 ? (-T(3), "||k||₂⁻³") : (-T(5 / 3), "k⁻⁵³") + slope, slopelabel = D == 2 ? (-T(3), "|k|⁻³") : (-T(5 / 3), "|k|⁻⁵³") slopeconst = T(0) + # Nice ticks + logmax = round(Int, log2(kmax + 1)) + xticks = T(2) .^ (0:logmax) # Make plot fig = Figure(; size = (500, 400)) ax = Axis( fig[1, 1]; - xlabel = "||k||₂", - ylabel = "e(k)", + xticks, + # xlabel = "||k||₂", + xlabel = "|k|", + # xlabel = "k", + # xlabel = L"\| k \|_2", + # xlabel = L"|k|_2", + # ylabel = "e(k)", + ylabel = "e(|k|)", + # ylabel = L"e(\| k \|_2)", + # ylabel = L"e(|k|_2)", title = "Kinetic energy (n = $(params_train.nles[ig]))", xscale = log10, yscale = log10, - xticks = [4, 8, 16, 32, 64, 128], - limits = (extrema(k)..., T(1e-8), T(1)), + limits = (extrema(kint)..., T(1e-8), T(1)), ) for (i, it) in enumerate((1, length(sample.t))) u = device.(sample.u[ig][it]) up = IncompressibleNavierStokes.interpolate_u_p(u, setup) - e = sum(up -> up[setup.grid.Ip] .^ 2, up) - ehat = Array( - A * reshape(abs.(fft(e)[ntuple(α -> kx[α] .+ 1, D)...]) ./ size(e, 1), :), - ) - slopeconst = max(slopeconst, maximum(ehat ./ k .^ slope)) - lines!(ax, k, ehat; label = "t = $(round(sample.t[it]; digits = 1))") + e = sum(up -> up[Ip] .^ 2, up) + e = fft(e)[ntuple(α -> kx[α] .+ 1, D)...] + e = abs.(e) ./ size(e, 1) + e = A * reshape(e, :) + ehat = max.(e, eps(T)) # Avoid log(0) + slopeconst = max(slopeconst, maximum(ehat ./ kint .^ slope)) + lines!(ax, kint, Array(ehat); label = "t = $(round(sample.t[it]; digits = 1))") end inertia = 2 .* slopeconst .* krange .^ slope lines!(ax, krange, inertia; linestyle = :dash, label = slopelabel) @@ -236,10 +257,10 @@ save("training_spectra.pdf", fig) closure, θ₀ = cnn(; setup = setups_train[1], - radii = [2, 2, 2, 2, 2, 2], - channels = [32, 32, 32, 32, 32, params_train.D], - activations = [leakyrelu, leakyrelu, leakyrelu, leakyrelu, leakyrelu, identity], - use_bias = [true, true, true, true, true, false], + radii = [2, 2, 2, 2], + channels = [20, 20, 20, params_train.D], + activations = [leakyrelu, leakyrelu, leakyrelu, identity], + use_bias = [true, true, true, false], rng, ); closure.chain @@ -263,7 +284,7 @@ callbackstate = Point2f[]; loss, opt, θ; - niter = 1000, + niter = 5000, ncallback = 20, callbackstate, callback = create_callback(closure, device(validset)...; state = callbackstate), @@ -478,48 +499,60 @@ state_cnn, outputs = solve_unsteady(closedsetup, u₀, p₀, tlims; Δt, pressur # Plot predicted spectra fig = with_theme(; palette = (; color = ["#3366cc", "#cc0000", "#669900", "#ffcc00"])) do + (; xp, Ip) = setup.grid D = params.D - backend = KernelAbstractions.get_backend(setup.grid.x[1]) - K = size(setup.grid.Ip) .÷ 2 - kx = ntuple(α -> 1:K[α]-1, params_train.D) - k = zero(similar(setup.grid.x[1], length.(kx))) + K = size(Ip) .÷ 2 + kx = ntuple(α -> 0:K[α]-1, D) + k = fill!(similar(xp[1], length.(kx)), 0) for α = 1:D kα = reshape(kx[α], ntuple(Returns(1), α - 1)..., :, ntuple(Returns(1), D - α)...) k .+= kα .^ 2 end k .= sqrt.(k) k = reshape(k, :) - # Make averaging matrix - naverage = 20^D - i = sortperm(k) - nbin, r = divrem(length(i), naverage) - ib = KernelAbstractions.zeros(backend, Int, nbin * naverage) - ia = KernelAbstractions.zeros(backend, Int, nbin * naverage) - for j = 1:naverage - copyto!(view(ia, (j-1)*nbin+1:j*nbin), collect(1:nbin)) - ib[(j-1)*nbin+1:j*nbin] = i[j:naverage:end-r] + # Sum or average wavenumbers between k and k+1 + nk = ceil(Int, maximum(k)) + kmax = nk - 1 + # kmax = minimum(K) - 1 + kint = 1:kmax + ia = similar(xp[1], Int, 0) + ib = sortperm(k) + vals = similar(xp[1], 0) + ksort = k[ib] + jprev = 2 # Do not include constant mode + for ki = 1:kmax + j = findfirst(>(ki+1), ksort) + isnothing(j) && (j = length(k) + 1) + ia = [ia; fill!(similar(ia, j - jprev), ki)] + # val = doaverage ? T(1) / (j - jprev) : T(1) + val = T(π) * ((ki+1)^2 - ki^2) / (j - jprev) + # val = T(1) / (j - jprev) + vals = [vals; fill!(similar(vals, j - jprev), val)] + jprev = j end - vals = KernelAbstractions.ones(backend, T, nbin * naverage) / naverage - A = sparse(ia, ib, vals, nbin, length(i)) - k = Array(A * k) + ib = ib[2:jprev-1] + A = sparse(ia, ib, vals, kmax, length(k)) # Build inertial slope above energy - @show length(k) - krange = collect(extrema(k[3:end-5])) - # slope, slopelabel = D == 2 ? (-T(3), L"$k^{-3}") : (-T(5 / 3), L"$k^{-5/3}") - slope, slopelabel = D == 2 ? (-T(3), "||k||₂⁻³") : (-T(5 / 3), L"k⁻⁵³") + # krange = [cbrt(T(kmax)), T(kmax)] + krange = [T(kmax)^(T(2)/3), T(kmax)] + # slope, slopelabel = D == 2 ? (-T(3), L"k^{-3}") : (-T(5 / 3), L"k^{-5/3}") + slope, slopelabel = D == 2 ? (-T(3), "|k|⁻³") : (-T(5 / 3), "|k|⁻⁵³") slopeconst = T(0) + # Nice ticks + logmax = round(Int, log2(kmax + 1)) + xticks = T(2) .^ (0:logmax) # Make plot fig = Figure(; size = (500, 400)) ax = Axis( fig[1, 1]; - xlabel = "||k||₂", - ylabel = "e(k)", + xticks, + xlabel = "|k|", + ylabel = "e(|k|)", # title = "Kinetic energy (n = $(params.nles[ig])) at time t = $(round(data_test.t[end]; digits = 1))", title = "Kinetic energy (n = $(params.nles[ig]))", xscale = log10, yscale = log10, - xticks = [4, 8, 16, 32, 64, 128], - limits = (extrema(k)..., T(1e-8), T(1)), + limits = (extrema(kint)..., T(1e-8), T(1)), ) for (u, label) in ( # (uref, "Reference"), @@ -529,12 +562,13 @@ fig = with_theme(; palette = (; color = ["#3366cc", "#cc0000", "#669900", "#ffcc (uref, "Reference"), ) up = IncompressibleNavierStokes.interpolate_u_p(u, setup) - e = sum(up -> up[setup.grid.Ip] .^ 2, up) - ehat = Array( - A * reshape(abs.(fft(e)[ntuple(α -> kx[α] .+ 1, D)...]) ./ size(e, 1), :), - ) - slopeconst = max(slopeconst, maximum(ehat ./ k .^ slope)) - lines!(ax, k, ehat; label) + e = sum(up -> up[Ip] .^ 2, up) + e = fft(e)[ntuple(α -> kx[α] .+ 1, D)...] + e = abs.(e) ./ size(e, 1) + e = A * reshape(e, :) + ehat = max.(e, eps(T)) # Avoid log(0) + slopeconst = max(slopeconst, maximum(ehat ./ kint .^ slope)) + lines!(ax, kint, Array(ehat); label) end inertia = 2 .* slopeconst .* krange .^ slope lines!(ax, krange, inertia; linestyle = :dash, label = slopelabel) @@ -547,65 +581,79 @@ save("predicted_spectra.pdf", fig) # Plot spectrum errors fig = with_theme(; palette = (; color = ["#3366cc", "#cc0000", "#669900", "#ffcc00"])) do + (; xp, Ip) = setup.grid D = params.D - backend = KernelAbstractions.get_backend(setup.grid.x[1]) - K = size(setup.grid.Ip) .÷ 2 - kx = ntuple(α -> 1:K[α]-1, params_train.D) - k = zero(similar(setup.grid.x[1], length.(kx))) + K = size(Ip) .÷ 2 + kx = ntuple(α -> 0:K[α]-1, D) + k = fill!(similar(xp[1], length.(kx)), 0) for α = 1:D kα = reshape(kx[α], ntuple(Returns(1), α - 1)..., :, ntuple(Returns(1), D - α)...) k .+= kα .^ 2 end k .= sqrt.(k) k = reshape(k, :) - # Make averaging matrix - naverage = 20^D - i = sortperm(k) - nbin, r = divrem(length(i), naverage) - ib = KernelAbstractions.zeros(backend, Int, nbin * naverage) - ia = KernelAbstractions.zeros(backend, Int, nbin * naverage) - for j = 1:naverage - copyto!(view(ia, (j-1)*nbin+1:j*nbin), collect(1:nbin)) - ib[(j-1)*nbin+1:j*nbin] = i[j:naverage:end-r] + # Sum or average wavenumbers between k and k+1 + nk = ceil(Int, maximum(k)) + # kmax = nk - 1 + kmax = minimum(K) - 1 + kint = 1:kmax + ia = similar(xp[1], Int, 0) + ib = sortperm(k) + vals = similar(xp[1], 0) + ksort = k[ib] + jprev = 2 # Do not include constant mode + for ki = 1:kmax + j = findfirst(>(ki+1), ksort) + isnothing(j) && (j = length(k) + 1) + ia = [ia; fill!(similar(ia, j - jprev), ki)] + # val = doaverage ? T(1) / (j - jprev) : T(1) + val = T(π) * ((ki+1)^2 - ki^2) / (j - jprev) + vals = [vals; fill!(similar(vals, j - jprev), val)] + jprev = j end - vals = KernelAbstractions.ones(backend, T, nbin * naverage) / naverage - A = sparse(ia, ib, vals, nbin, length(i)) - k = Array(A * k) + ib = ib[2:jprev-1] + A = sparse(ia, ib, vals, kmax, length(k)) # Build inertial slope above energy - @show length(k) - krange = collect(extrema(k[20:end])) - # slope, slopelabel = D == 2 ? (-T(3), L"$k^{-3}") : (-T(5 / 3), L"$k^{-5/3}") - slope, slopelabel = D == 2 ? (-T(3), "||k||₂⁻³") : (-T(5 / 3), L"k⁻⁵³") + # krange = [cbrt(T(kmax)), T(kmax)] + krange = [T(kmax)^(T(2)/3), T(kmax)] + # slope, slopelabel = D == 2 ? (-T(3), L"k^{-3}") : (-T(5 / 3), L"k^{-5/3}") + slope, slopelabel = D == 2 ? (-T(3), "|k|⁻³") : (-T(5 / 3), "|k|⁻⁵³") slopeconst = T(0) + # Nice ticks + logmax = round(Int, log2(kmax + 1)) + xticks = T(2) .^ (0:logmax) # Make plot fig = Figure(; size = (500, 400)) ax = Axis( fig[1, 1]; - xlabel = "||k||₂", - ylabel = "e(k)", + xticks, + xlabel = "|k|", + ylabel = "e(|k|)", # title = "Kinetic energy (n = $(params.nles[ig])) at time t = $(round(data_test.t[end]; digits = 1))", title = "Relative energy error (n = $(params.nles[ig]))", xscale = log10, yscale = log10, - xticks = [4, 8, 16, 32, 64, 128], - limits = (extrema(k)..., T(1e-8), T(1)), + limits = (extrema(kint)..., T(1e-8), T(1)), ) up = IncompressibleNavierStokes.interpolate_u_p(uref, setup) - e = sum(up -> up[setup.grid.Ip] .^ 2, up) - eref = - Array(A * reshape(abs.(fft(e)[ntuple(α -> kx[α] .+ 1, D)...]) ./ size(e, 1), :)) + e = sum(up -> up[Ip] .^ 2, up) + e = fft(e)[ntuple(α -> kx[α] .+ 1, D)...] + e = abs.(e) ./ size(e, 1) + e = A * reshape(e, :) + eref = max.(e, eps(T)) # Avoid log(0) for (u, label) in ( (state_nm.u, "No closure"), (state_smag.u, "Smagorinsky"), (state_cnn.u, "CNN (specialized)"), ) up = IncompressibleNavierStokes.interpolate_u_p(u, setup) - e = sum(up -> up[setup.grid.Ip] .^ 2, up) - ehat = Array( - A * reshape(abs.(fft(e)[ntuple(α -> kx[α] .+ 1, D)...]) ./ size(e, 1), :), - ) + e = sum(up -> up[Ip] .^ 2, up) + e = fft(e)[ntuple(α -> kx[α] .+ 1, D)...] + e = abs.(e) ./ size(e, 1) + e = A * reshape(e, :) + ehat = max.(e, eps(T)) # Avoid log(0) ee = @. abs(ehat - eref) / abs(eref) - lines!(ax, k, ee; label) + lines!(ax, kint, Array(ee); label) end axislegend(ax; position = :lt) autolimits!(ax) diff --git a/src/IncompressibleNavierStokes.jl b/src/IncompressibleNavierStokes.jl index 53adf6017..fa43751f4 100644 --- a/src/IncompressibleNavierStokes.jl +++ b/src/IncompressibleNavierStokes.jl @@ -26,6 +26,8 @@ using Zygote # Must be loaded inside for Tullio to work correctly using CUDA +using CUDA.CUSPARSE +using CUSOLVERRF # # Easily retrieve value from Val # (::Val{x})() where {x} = x diff --git a/src/operators.jl b/src/operators.jl index 2d6ff5119..6e987d531 100644 --- a/src/operators.jl +++ b/src/operators.jl @@ -10,7 +10,7 @@ for writing kernel loops using Cartesian indices. """ struct Offset{D} end -(::Offset{D})(α) where {D} = CartesianIndex(ntuple(β -> β == α ? 1 : 0, D)) +@inline (::Offset{D})(α) where {D} = CartesianIndex(ntuple(β -> β == α ? 1 : 0, D)) """ divergence!(div, u, setup) @@ -323,16 +323,54 @@ function laplacian!(L, p, setup) (; dimension, Δ, Δu, N, Np, Ip, Ω) = grid D = dimension() δ = Offset{D}() - @kernel function lap!(L, p, I0) + # @kernel function lap!(L, p, I0) + # I = @index(Global, Cartesian) + # I = I + I0 + # lap = zero(eltype(p)) + # for α = 1:D + # # bc = boundary_conditions[α] + # if bc[1] isa PressureBC && I[α] == I0[α] + 1 + # lap += + # Ω[I] / Δ[α][I[α]] * + # ((p[I+δ(α)] - p[I]) / Δu[α][I[α]] - (p[I]) / Δu[α][I[α]-1]) + # elseif bc[2] isa PressureBC && I[α] == I0[α] + Np[α] + # lap += + # Ω[I] / Δ[α][I[α]] * + # ((-p[I]) / Δu[α][I[α]] - (p[I] - p[I-δ(α)]) / Δu[α][I[α]-1]) + # elseif bc[1] isa DirichletBC && I[α] == I0[α] + 1 + # lap += Ω[I] / Δ[α][I[α]] * ((p[I+δ(α)] - p[I]) / Δu[α][I[α]]) + # elseif bc[2] isa DirichletBC && I[α] == I0[α] + Np[α] + # lap += Ω[I] / Δ[α][I[α]] * (-(p[I] - p[I-δ(α)]) / Δu[α][I[α]-1]) + # else + # lap += + # Ω[I] / Δ[α][I[α]] * + # ((p[I+δ(α)] - p[I]) / Δu[α][I[α]] - (p[I] - p[I-δ(α)]) / Δu[α][I[α]-1]) + # end + # end + # L[I] = lap + # end + @kernel function lapα!(L, p, I0, ::Val{α}, bc) where {α} I = @index(Global, Cartesian) I = I + I0 - lap = zero(eltype(p)) - for α = 1:D - lap += + # bc = boundary_conditions[α] + if bc[1] isa PressureBC && I[α] == I0[α] + 1 + L[I] += + Ω[I] / Δ[α][I[α]] * + ((p[I+δ(α)] - p[I]) / Δu[α][I[α]] - (p[I]) / Δu[α][I[α]-1]) + elseif bc[2] isa PressureBC && I[α] == I0[α] + Np[α] + L[I] += + Ω[I] / Δ[α][I[α]] * + ((-p[I]) / Δu[α][I[α]] - (p[I] - p[I-δ(α)]) / Δu[α][I[α]-1]) + elseif bc[1] isa DirichletBC && I[α] == I0[α] + 1 + L[I] += Ω[I] / Δ[α][I[α]] * ((p[I+δ(α)] - p[I]) / Δu[α][I[α]]) + elseif bc[2] isa DirichletBC && I[α] == I0[α] + Np[α] + L[I] += Ω[I] / Δ[α][I[α]] * (-(p[I] - p[I-δ(α)]) / Δu[α][I[α]-1]) + else + L[I] += Ω[I] / Δ[α][I[α]] * ((p[I+δ(α)] - p[I]) / Δu[α][I[α]] - (p[I] - p[I-δ(α)]) / Δu[α][I[α]-1]) end - L[I] = lap + # L[I] = lap end # All volumes have a right velocity # All volumes have a left velocity except the first one @@ -340,7 +378,18 @@ function laplacian!(L, p, setup) ndrange = Np I0 = first(Ip) I0 -= oneunit(I0) - lap!(get_backend(L), workgroupsize)(L, p, I0; ndrange) + # lap!(get_backend(L), workgroupsize)(L, p, I0; ndrange) + L .= 0 + for α = 1:D + lapα!(get_backend(L), workgroupsize)( + L, + p, + I0, + Val(α), + boundary_conditions[α]; + ndrange, + ) + end L end @@ -441,88 +490,34 @@ function laplacian_mat(setup) # Ω isa CuArray ? cu(L) : L end -# @inline function ∂x(uα, I, ::Val{α}, ::Val{β}, Δβ, Δuβ) where {α,β} -# D = length(I) -# δ = Offset{D}() -# α == β ? (uα[I] - uα[I-δ(β)]) / Δβ[I[β]] : -# ( -# (uα[I+δ(β)] - uα[I]) / Δuβ[I[β]] + -# (uα[I-δ(α)+δ(β)] - uα[I-δ(α)]) / Δuβ[I[β]] + -# (uα[I] - uα[I-δ(β)]) / Δuβ[I[β]-1] + -# (uα[I-δ(α)] - uα[I-δ(α)-δ(β)]) / Δuβ[I[β]-1] -# ) / 4 -# end -# @inline ∇(::Val{2}, u, I, Δ, Δu) = SMatrix{2,2,eltype(u[1]),4}( -# ∂x(u[1], I, Val(1), Val(1), Δ[1], Δu[1]), -# ∂x(u[2], I, Val(2), Val(1), Δ[1], Δu[1]), -# ∂x(u[1], I, Val(1), Val(2), Δ[2], Δu[2]), -# ∂x(u[2], I, Val(2), Val(2), Δ[2], Δu[2]), -# ) -# @inline ∇(::Val{3}, u, I, Δ, Δu) = SMatrix{3,3,eltype(u[1]),9}( -# ∂x(u[1], I, Val(1), Val(1), Δ[1], Δu[1]), -# ∂x(u[2], I, Val(2), Val(1), Δ[1], Δu[1]), -# ∂x(u[3], I, Val(3), Val(1), Δ[1], Δu[1]), -# ∂x(u[1], I, Val(1), Val(2), Δ[2], Δu[2]), -# ∂x(u[2], I, Val(2), Val(2), Δ[2], Δu[2]), -# ∂x(u[3], I, Val(3), Val(2), Δ[2], Δu[2]), -# ∂x(u[1], I, Val(1), Val(3), Δ[3], Δu[3]), -# ∂x(u[2], I, Val(2), Val(3), Δ[3], Δu[3]), -# ∂x(u[3], I, Val(3), Val(3), Δ[3], Δu[3]), -# ) -# -# @inline function strain(valD, u, I, Δ, Δu) -# ∇u = ∇(valD, u, I, Δ, Δu) -# (∇u + ∇u') / 2 -# end -# -# @inline gridsize(::Val{2}, Δ, I) = sqrt(Δ[1][I[1]]^2 + Δ[2][I[2]]^2) -# @inline gridsize(::Val{3}, Δ, I) = sqrt(Δ[1][I[1]]^2 + Δ[2][I[2]]^2 + Δ[3][I[3]]^2) +@inline ∂x(uα, I::CartesianIndex{D}, α, β, Δβ, Δuβ; δ = Offset{D}()) where {D} = + α == β ? (uα[I] - uα[I-δ(β)]) / Δβ[I[β]] : + ( + (uα[I+δ(β)] - uα[I]) / Δuβ[I[β]] + + (uα[I-δ(α)+δ(β)] - uα[I-δ(α)]) / Δuβ[I[β]] + + (uα[I] - uα[I-δ(β)]) / Δuβ[I[β]-1] + + (uα[I-δ(α)] - uα[I-δ(α)-δ(β)]) / Δuβ[I[β]-1] + ) / 4 +@inline ∇(u, I::CartesianIndex{2}, Δ, Δu) = + @SMatrix [∂x(u[α], I, α, β, Δ[β], Δu[β]) for α = 1:2, β = 1:2] +∇(u, I::CartesianIndex{3}, Δ, Δu) = + @SMatrix [∂x(u[α], I, α, β, Δ[β], Δu[β]) for α = 1:3, β = 1:3] +@inline function strain(u, I, Δ, Δu) + ∇u = ∇(u, I, Δ, Δu) + (∇u + ∇u') / 2 +end +@inline gridsize(Δ, I::CartesianIndex{D}) where {D} = + sqrt(sum(ntuple(α -> Δ[α][I[α]]^2, D))) function smagtensor!(σ, u, θ, setup) # TODO: Combine with normal diffusion tensor (; boundary_conditions, grid, workgroupsize) = setup - (; dimension, Np, Ip, Δ, Δu) = grid - D = dimension() - δ = Offset{D}() - @assert D == 2 + (; Np, Ip, Δ, Δu) = grid @kernel function σ!(σ, u, I0) - ∂x(uα, I, ::Val{α}, ::Val{β}, Δβ, Δuβ) where {α,β} = - α == β ? (uα[I] - uα[I-δ(β)]) / Δβ[I[β]] : - ( - (uα[I+δ(β)] - uα[I]) / Δuβ[I[β]] + - (uα[I-δ(α)+δ(β)] - uα[I-δ(α)]) / Δuβ[I[β]] + - (uα[I] - uα[I-δ(β)]) / Δuβ[I[β]-1] + - (uα[I-δ(α)] - uα[I-δ(α)-δ(β)]) / Δuβ[I[β]-1] - ) / 4 I = @index(Global, Cartesian) I = I + I0 - if D == 2 - ∇u = SMatrix{2,2,eltype(u[1]),4}( - ∂x(u[1], I, Val(1), Val(1), Δ[1], Δu[1]), - ∂x(u[2], I, Val(2), Val(1), Δ[1], Δu[1]), - ∂x(u[1], I, Val(1), Val(2), Δ[2], Δu[2]), - ∂x(u[2], I, Val(2), Val(2), Δ[2], Δu[2]), - ) - d = sqrt(Δ[1][I[1]]^2 + Δ[2][I[2]]^2) - elseif D == 3 - # TODO: Figure out why KA doesn't like I[3] when D == 2 - # ∇u = SMatrix{3,3,eltype(u[1]),9}( - # ∂x(u[1], I, Val(1), Val(1), Δ[1], Δu[1]), - # ∂x(u[2], I, Val(2), Val(1), Δ[1], Δu[1]), - # ∂x(u[3], I, Val(3), Val(1), Δ[1], Δu[1]), - # ∂x(u[1], I, Val(1), Val(2), Δ[2], Δu[2]), - # ∂x(u[2], I, Val(2), Val(2), Δ[2], Δu[2]), - # ∂x(u[3], I, Val(3), Val(2), Δ[2], Δu[2]), - # ∂x(u[1], I, Val(1), Val(3), Δ[3], Δu[3]), - # ∂x(u[2], I, Val(2), Val(3), Δ[3], Δu[3]), - # ∂x(u[3], I, Val(3), Val(3), Δ[3], Δu[3]), - # ) - # d = sqrt(Δ[1][I[1]]^2 + Δ[2][I[2]]^2 + Δ[3][I[3]]^2) - end - # S = strain(Val(D), u, I, Δ, Δu) - # d = gridsize(Val(D), Δ, I) - S = (∇u + ∇u') / 2 - # νt = θ^2 * d^2 * sqrt(2 * dot(S, S)) + S = strain(u, I, Δ, Δu) + d = gridsize(Δ, I) νt = θ^2 * d^2 * sqrt(2 * sum(S .* S)) σ[I] = 2 * νt * S end @@ -540,7 +535,7 @@ function smagorinsky!(s, σ, setup) @kernel function s!(s, σ, ::Val{α}, ::Val{βrange}, I0) where {α,βrange} I = @index(Global, Cartesian) I = I + I0 - s[α][I] = 0 + s[α][I] = zero(eltype(s[1])) # for β = 1:D KernelAbstractions.Extras.LoopInfo.@unroll for β in βrange Δuαβ = α == β ? Δu[β] : Δ[β] @@ -578,12 +573,14 @@ end function smagorinsky_closure(setup) (; dimension, x, N) = setup.grid D = dimension() - backend = get_backend(x[1]) T = eltype(x[1]) σ = similar(x[1], SMatrix{D,D,T,D * D}, N) s = ntuple(α -> similar(x[1], N), D) + # σ = zero(similar(x[1], SMatrix{D,D,T,D * D}, N)) + # s = ntuple(α -> zero(similar(x[1], N)), D) function closure(u, θ) smagtensor!(σ, u, θ, setup) + apply_bc_p!(σ, zero(T), setup) smagorinsky!(s, σ, setup) end end @@ -776,33 +773,14 @@ Qfield(u, setup) = Qfield!(similar(u[1], setup.grid.N), u, setup) Compute the second eigenvalue of ``S^2 + \\Omega^2``. """ function eig2field!(λ, u, setup) - (; boundary_conditions, grid, workgroupsize) = setup + (; grid, workgroupsize) = setup (; dimension, Np, Ip, Δ, Δu) = grid D = dimension() - δ = Offset{D}() @assert D == 3 "eig2 only implemented in 3D" @kernel function λ!(λ, u, I0) I = @index(Global, Cartesian) I = I + I0 - ∂x(uα, I, ::Val{α}, ::Val{β}, Δβ, Δuβ) where {α,β} = - α == β ? (uα[I] - uα[I-δ(β)]) / Δβ[I[β]] : - ( - (uα[I+δ(β)] - uα[I]) / Δuβ[I[β]] + - (uα[I-δ(α)+δ(β)] - uα[I-δ(α)]) / Δuβ[I[β]] + - (uα[I] - uα[I-δ(β)]) / Δuβ[I[β]-1] + - (uα[I-δ(α)] - uα[I-δ(α)-δ(β)]) / Δuβ[I[β]-1] - ) / 4 - ∇u = SMatrix{3,3,eltype(λ),9}( - ∂x(u[1], I, Val(1), Val(1), Δ[1], Δu[1]), - ∂x(u[2], I, Val(2), Val(1), Δ[1], Δu[1]), - ∂x(u[3], I, Val(3), Val(1), Δ[1], Δu[1]), - ∂x(u[1], I, Val(1), Val(2), Δ[2], Δu[2]), - ∂x(u[2], I, Val(2), Val(2), Δ[2], Δu[2]), - ∂x(u[3], I, Val(3), Val(2), Δ[2], Δu[2]), - ∂x(u[1], I, Val(1), Val(3), Δ[3], Δu[3]), - ∂x(u[2], I, Val(2), Val(3), Δ[3], Δu[3]), - ∂x(u[3], I, Val(3), Val(3), Δ[3], Δu[3]), - ) + ∇u = ∇(u, I, Δ, Δu) S = @. (∇u + ∇u') / 2 Ω = @. (∇u - ∇u') / 2 λ[I] = eigvals(S^2 + Ω^2)[2] diff --git a/src/solvers/pressure/poisson.jl b/src/solvers/pressure/poisson.jl index 73d3a3c16..77e91861b 100644 --- a/src/solvers/pressure/poisson.jl +++ b/src/solvers/pressure/poisson.jl @@ -30,23 +30,41 @@ function poisson!(solver::DirectPressureSolver, p, f) T = eltype(p) # solver.f .= view(view(f, Ip), :) # copyto!(solver.f, view(view(f, Ip), :)) - copyto!(view(solver.f, 1:length(solver.f)-1), Array(view(view(f, Ip), :))) + pp = view(view(p, Ip), :) + if false # p isa CuArray + ldiv!(solver.p, fact, solver.f) + elseif false # p isa CuArray + copyto!(view(solver.f, 1:length(solver.f)-1), view(view(f, Ip), :)) + F = fact + a, b = solver.f, solver.p + # solver.p .= F.Q * (F.U \ (F.L \ (F.P * (F.Rs .* solver.f)))) + # copyto!(pp, view(solver.p, 1:length(solver.p)-1)) + a .*= F.Rs + mul!(b, F.P, a) + ldiv!(a, F.L, b) + ldiv!(b, F.U, a) + mul!(a, F.U, b) + copyto!(pp, view(a, 1:length(a)-1)) + else + copyto!(view(solver.f, 1:length(solver.f)-1), Array(view(view(f, Ip), :))) + solver.p .= fact \ solver.f + copyto!(pp, T.(view(solver.p, 1:length(solver.p)-1))) + end # @infiltrate - solver.p .= fact \ solver.f # ldiv!(solver.p, fact, solver.f) - pp = view(view(p, Ip), :) # pp .= solver.p # copyto!(pp, solver.p) - copyto!(pp, T.(view(solver.p, 1:length(solver.p)-1))) p end -# function poisson!(solver::CGPressureSolver, p, f) -# (; A, abstol, reltol, maxiter) = solver -# f = view(f, :) -# p = view(p, :) -# cg!(p, A, f; abstol, reltol, maxiter) -# end +function poisson!(solver::CGMatrixPressureSolver, p, f) + (; L, qin, qout, abstol, reltol, maxiter) = solver + copyto!(qin, view(view(f, Ip), :)) + p = view(p, :) + cg!(qout, L, qin; abstol, reltol, maxiter) + copyto!(view(view(p, Ip), :), qout) + p +end # Solve Lp = f # where Lp = Ω * div(pressurgrad(p)) @@ -125,6 +143,8 @@ function poisson!(solver::CGPressureSolver, p, f) iteration += 1 end + # @show iteration residual tolerance + p end diff --git a/src/solvers/pressure/solvers.jl b/src/solvers/pressure/solvers.jl index c9c0e43ad..c8b96e78c 100644 --- a/src/solvers/pressure/solvers.jl +++ b/src/solvers/pressure/solvers.jl @@ -16,19 +16,54 @@ struct DirectPressureSolver{T,S,F,A} <: AbstractPressureSolver{T} f::A p::A function DirectPressureSolver(setup) - (; x, Np) = setup.grid - # T = eltype(x[1]) - T = Float64 - backend = get_backend(x[1]) - # f = similar(x[1], prod(Np)) - # p = similar(x[1], prod(Np)) - f = zeros(T, prod(Np) + 1) - p = zeros(T, prod(Np) + 1) - L = laplacian_mat(setup) - e = ones(T, size(L, 2)) - L = [L e; e' 0] - # fact = lu(L) - fact = factorize(L) + (; grid, ArrayType) = setup + (; x, Np) = grid + if false #ArrayType == CuArray + # T = eltype(x[1]) + T = Float64 + f = similar(x[1], T, prod(Np) + 1) + p = similar(x[1], T, prod(Np) + 1) + L = laplacian_mat(setup) + e = ones(T, size(L, 2)) + L = [L e; e' 0] + L = L |> CuSparseMatrixCSR{Float64} + fact = CUSOLVERRF.RFLU(L; symbolic = :RF) + elseif false #ArrayType == CuArray + T = eltype(x[1]) + f = similar(x[1], prod(Np) + 1) + p = similar(x[1], prod(Np) + 1) + L = laplacian_mat(setup) + e = ones(Float64, size(L, 2)) + L = [L e; e' 0] + F = lu(L) + P = sparse(1:length(F.p), F.p, ones(length(F.p))) + Q = sparse(F.q, 1:length(F.q), ones(length(F.q))) + L = F.L |> CuSparseMatrixCSR{T} + U = F.U |> CuSparseMatrixCSR{T} + CUDA.@allowscalar L = L |> LowerTriangular + CUDA.@allowscalar U = U |> UpperTriangular + # CUDA.@allowscalar L = F.L |> CuSparseMatrixCSR{Float64} |> LowerTriangular + # CUDA.@allowscalar U = F.U |> CuSparseMatrixCSR{Float64} |> UpperTriangular + fact = (; + L, + U, + Rs = F.Rs |> CuArray{T}, + P = P |> CuSparseMatrixCSR{T}, + Q = Q |> CuSparseMatrixCSR{T}, + ) + else + # T = eltype(x[1]) + T = Float64 + backend = get_backend(x[1]) + f = zeros(T, prod(Np) + 1) + p = zeros(T, prod(Np) + 1) + L = laplacian_mat(setup) + e = ones(T, size(L, 2)) + L = [L e; e' 0] + # fact = lu(L) + # fact = ldlt(Symmetric(L)) + fact = factorize(L) + end new{T,typeof(setup),typeof(fact),typeof(f)}(setup, fact, f, p) end end @@ -72,6 +107,29 @@ Adapt.adapt_structure(to, s::DirectPressureSolver) = error( # adapt(to, s.maxiter), # ) +""" + CGMatrixPressureSolver(setup; [abstol], [reltol], [maxiter]) + +Conjugate gradients iterative pressure solver. +""" +struct CGMatrixPressureSolver{T,M} <: AbstractPressureSolver{T} + L::M + abstol::T + reltol::T + maxiter::Int +end + +function CGMatrixPressureSolver( + setup; + abstol = 0, + reltol = sqrt(eps(eltype(setup.grid.x[1]))), + maxiter = prod(setup.grid.Np), +) + L = laplacian_mat(setup) |> setup.device + L = L |> CuSparseMatrixCSR + CGPressureSolver{eltype(L),typeof(L)}(L, abstol, reltol, maxiter) +end + """ CGPressureSolver(setup; [abstol], [reltol], [maxiter]) From 67ab42f12b89f7ab5048c264c34c5304a459c232 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Wed, 13 Dec 2023 14:03:18 +0100 Subject: [PATCH 194/379] Rename function --- src/processors/real_time_plot.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/processors/real_time_plot.jl b/src/processors/real_time_plot.jl index 0799ae98b..2448bffb6 100644 --- a/src/processors/real_time_plot.jl +++ b/src/processors/real_time_plot.jl @@ -271,7 +271,7 @@ function energy_history_plot(state; setup) @assert state isa Observable "Energy history requires observable state." _points = Point2f[] points = lift(state) do (; u, p, t) - E = kinetic_energy(u, setup) + E = total_kinetic_energy(u, setup) push!(_points, Point2f(t, E)) end fig = lines(points; axis = (; xlabel = "t", ylabel = "Kinetic energy")) From 343e769fe95902218d07851f992b3a723945a141 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Wed, 13 Dec 2023 14:03:53 +0100 Subject: [PATCH 195/379] Add toggle option --- src/processors/real_time_plot.jl | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/processors/real_time_plot.jl b/src/processors/real_time_plot.jl index 2448bffb6..1ad756e62 100644 --- a/src/processors/real_time_plot.jl +++ b/src/processors/real_time_plot.jl @@ -291,7 +291,7 @@ The energy at a scalar wavenumber level ``\\kappa \\in \\mathbb{N}`` is defined as in San and Staples [San2012](@cite). """ -function energy_spectrum_plot(state; setup) +function energy_spectrum_plot(state; setup, doaverage = false) state isa Observable || (state = Observable(state)) (; dimension, xp, Ip) = setup.grid backend = get_backend(xp[1]) @@ -317,11 +317,10 @@ function energy_spectrum_plot(state; setup) ksort = k[ib] jprev = 2 # Do not include constant mode for ki = 1:kmax - j = findfirst(>(ki+1), ksort) + j = findfirst(>(ki + 1), ksort) isnothing(j) && (j = length(k) + 1) ia = [ia; fill!(similar(ia, j - jprev), ki)] - # val = doaverage ? T(1) / (j - jprev) : T(1) - val = T(π) * ((ki+1)^2 - ki^2) / (j - jprev) + val = doaverage ? T(1) / (j - jprev) : T(π) * ((ki + 1)^2 - ki^2) / (j - jprev) vals = [vals; fill!(similar(vals, j - jprev), val)] jprev = j end @@ -365,7 +364,7 @@ function energy_spectrum_plot(state; setup) limits = (extrema(kint)..., T(1e-8), T(1)), ) lines!(ax, kint, ehat; label = "Kinetic energy") - lines!(ax, krange, inertia; label = slopelabel, color = :red) + lines!(ax, krange, inertia; label = slopelabel, linestyle = :dash) axislegend(ax) # autolimits!(ax) on(e -> autolimits!(ax), ehat) From b126366ab546d2acea7026f8a791c5e50a4228b0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Wed, 13 Dec 2023 14:04:15 +0100 Subject: [PATCH 196/379] Fix wrong variable --- src/solvers/pressure/poisson.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/solvers/pressure/poisson.jl b/src/solvers/pressure/poisson.jl index 77e91861b..1217675fc 100644 --- a/src/solvers/pressure/poisson.jl +++ b/src/solvers/pressure/poisson.jl @@ -43,7 +43,7 @@ function poisson!(solver::DirectPressureSolver, p, f) mul!(b, F.P, a) ldiv!(a, F.L, b) ldiv!(b, F.U, a) - mul!(a, F.U, b) + mul!(a, F.Q, b) copyto!(pp, view(a, 1:length(a)-1)) else copyto!(view(solver.f, 1:length(solver.f)-1), Array(view(view(f, Ip), :))) From 7ba29e27ff91464db3fe5127418a23beffc35571 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Wed, 13 Dec 2023 14:37:15 +0100 Subject: [PATCH 197/379] Add kinetic energy --- docs/references.bib | 11 ++++++ scratch/multigrid.jl | 16 ++++----- src/operators.jl | 62 ++++++++++++++++++++++++++++++-- src/processors/real_time_plot.jl | 6 ++-- 4 files changed, 82 insertions(+), 13 deletions(-) diff --git a/docs/references.bib b/docs/references.bib index 19efae8a4..fb6f0f995 100644 --- a/docs/references.bib +++ b/docs/references.bib @@ -11,6 +11,17 @@ @article{San2012 month={jun}, pages={105–127} } +@article{Sanderse2023, + Author = {Benjamin Sanderse and Francesc Xavier Trias}, + Title = {Energy-consistent discretization of viscous dissipation with application + to natural convection flow}, + Eprint = {2307.10874v1}, + ArchivePrefix = {arXiv}, + PrimaryClass = {physics.flu-dyn}, + Year = {2023}, + Month = {Jul}, + Url = {http://arxiv.org/abs/2307.10874v1}, +} @article{Beck2021, author = {Beck, Andrea and Kurz, Marius}, doi = {10.1002/gamm.202100002}, diff --git a/scratch/multigrid.jl b/scratch/multigrid.jl index 1ce6d92db..3dad5c70c 100644 --- a/scratch/multigrid.jl +++ b/scratch/multigrid.jl @@ -237,8 +237,8 @@ fig = with_theme(; palette = (; color = ["#3366cc", "#cc0000", "#669900", "#ffcc ) for (i, it) in enumerate((1, length(sample.t))) u = device.(sample.u[ig][it]) - up = IncompressibleNavierStokes.interpolate_u_p(u, setup) - e = sum(up -> up[Ip] .^ 2, up) + ke = kinetic_energy(u, setup) + e = ke[Ip] e = fft(e)[ntuple(α -> kx[α] .+ 1, D)...] e = abs.(e) ./ size(e, 1) e = A * reshape(e, :) @@ -561,8 +561,8 @@ fig = with_theme(; palette = (; color = ["#3366cc", "#cc0000", "#669900", "#ffcc (state_cnn.u, "CNN (specialized)"), (uref, "Reference"), ) - up = IncompressibleNavierStokes.interpolate_u_p(u, setup) - e = sum(up -> up[Ip] .^ 2, up) + ke = kinetic_energy(u, setup) + e = ke[Ip] e = fft(e)[ntuple(α -> kx[α] .+ 1, D)...] e = abs.(e) ./ size(e, 1) e = A * reshape(e, :) @@ -635,8 +635,8 @@ fig = with_theme(; palette = (; color = ["#3366cc", "#cc0000", "#669900", "#ffcc yscale = log10, limits = (extrema(kint)..., T(1e-8), T(1)), ) - up = IncompressibleNavierStokes.interpolate_u_p(uref, setup) - e = sum(up -> up[Ip] .^ 2, up) + ke = kinetic_energy(uref, setup) + e = ke[Ip] e = fft(e)[ntuple(α -> kx[α] .+ 1, D)...] e = abs.(e) ./ size(e, 1) e = A * reshape(e, :) @@ -646,8 +646,8 @@ fig = with_theme(; palette = (; color = ["#3366cc", "#cc0000", "#669900", "#ffcc (state_smag.u, "Smagorinsky"), (state_cnn.u, "CNN (specialized)"), ) - up = IncompressibleNavierStokes.interpolate_u_p(u, setup) - e = sum(up -> up[Ip] .^ 2, up) + ke = kinetic_energy(u, setup) + e = ke[Ip] e = fft(e)[ntuple(α -> kx[α] .+ 1, D)...] e = abs.(e) ./ size(e, 1) e = A * reshape(e, :) diff --git a/src/operators.jl b/src/operators.jl index 6e987d531..ca91aa47b 100644 --- a/src/operators.jl +++ b/src/operators.jl @@ -799,12 +799,70 @@ Compute the second eigenvalue of ``S^2 + \\Omega^2``. eig2field(u, setup) = eig2field!(similar(u[1], setup.grid.N), u, setup) """ - kinetic_energy(setup, u) + kinetic_energy!(e, u, setup; interpolate_first = false) + +Compute kinetic energy field ``e`` (in-place version). +If `interpolate_first` is true, it is given by + +```math +e_I = \\frac{1}{8} \\sum_\\alpha (u^\\alpha_{I + \\delta(\\alpha) / 2} + u^\\alpha_{I - \\delta(\\alpha) / 2})^2. +``` + +Otherwise, it is given by + +```math +e_I = \\frac{1}{4} \\sum_\\alpha (u^\\alpha_{I + \\delta(\\alpha) / 2}^2 + u^\\alpha_{I - \\delta(\\alpha) / 2}^2), +``` + +as in [Sanderse2023](@cite). +""" +function kinetic_energy!(e, u, setup; interpolate_first = false) + (; grid, workgroupsize) = setup + (; dimension, Np, Ip) = grid + D = dimension() + δ = Offset{D}() + @kernel function efirst!(e, u, I0) + I = @index(Global, Cartesian) + I = I + I0 + k = zero(eltype(e)) + for α = 1:D + k += (u[α][I] + u[α][I-δ(α)])^2 + end + k = k / 8 + e[I] = k + end + @kernel function elast!(e, u, I0) + I = @index(Global, Cartesian) + I = I + I0 + k = zero(eltype(e)) + for α = 1:D + k += u[α][I]^2 + u[α][I-δ(α)]^2 + end + k = k / 4 + e[I] = k + end + e! = interpolate_first ? efirst! : elast! + I0 = first(Ip) + I0 -= oneunit(I0) + e!(get_backend(u[1]), workgroupsize)(e, u, I0; ndrange = Np) + e +end + +""" + kinetic_energy(u, setup; kwargs...) + +Compute kinetic energy field ``e`` (out-of-place version). +""" +kinetic_energy(u, setup; kwargs...) = + kinetic_energy!(similar(u[1], setup.grid.N), u, setup; kwargs...) + +""" + total_kinetic_energy(setup, u) Compute total kinetic energy. The velocity components are interpolated to the volume centers and squared. """ -function kinetic_energy(u, setup) +function total_kinetic_energy(u, setup) (; dimension, Ω, Ip) = setup.grid D = dimension() up = interpolate_u_p(u, setup) diff --git a/src/processors/real_time_plot.jl b/src/processors/real_time_plot.jl index 1ad756e62..4709dd5c1 100644 --- a/src/processors/real_time_plot.jl +++ b/src/processors/real_time_plot.jl @@ -328,10 +328,10 @@ function energy_spectrum_plot(state; setup, doaverage = false) A = sparse(ia, ib, vals, kmax, length(k)) # Energy - up = interpolate_u_p(state[].u, setup) + ke = kinetic_energy(state[].u, setup) ehat = lift(state) do (; u, p, t) - interpolate_u_p!(up, u, setup) - e = sum(up -> up[Ip] .^ 2, up) + kinetic_energy!(ke, u, setup) + e = ke[Ip] e = fft(e)[ntuple(α -> kx[α] .+ 1, D)...] e = abs.(e) ./ size(e, 1) e = A * reshape(e, :) From 7674d33177778a7d9a34994ca127a6c2d1ecbfd9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Wed, 13 Dec 2023 14:51:29 +0100 Subject: [PATCH 198/379] docs: Add docstrings --- src/operators.jl | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/src/operators.jl b/src/operators.jl index ca91aa47b..d8424b8f1 100644 --- a/src/operators.jl +++ b/src/operators.jl @@ -509,6 +509,12 @@ end @inline gridsize(Δ, I::CartesianIndex{D}) where {D} = sqrt(sum(ntuple(α -> Δ[α][I[α]]^2, D))) +""" + smagtensor!(σ, u, θ, setup) + +Compute Smagorinsky stress tensors `σ[I]`. +The Smagorinsky constant `θ` should be a scalar between `0` and `1`. +""" function smagtensor!(σ, u, θ, setup) # TODO: Combine with normal diffusion tensor (; boundary_conditions, grid, workgroupsize) = setup @@ -527,6 +533,12 @@ function smagtensor!(σ, u, θ, setup) σ end +""" + smagorinsky!(s, σ, setup) + +Compute the Smagorinsky closure term `s` (additional diffusive force). +The Smagorinsky stress tensors should be precomputed and stored in `σ`. +""" function smagorinsky!(s, σ, setup) (; boundary_conditions, grid, workgroupsize) = setup (; dimension, Nu, Iu, Δ, Δu, A) = grid @@ -570,6 +582,13 @@ function smagorinsky!(s, σ, setup) s end +""" + m = smagorinsky_closure(setup) + +Create Smagoinsky closure model `m`. +The model is called as `m(u, θ)`, where the Smagorinsky constant +`θ` should be a scalar between `0` and `1` (for example `θ = 0.1`). +""" function smagorinsky_closure(setup) (; dimension, x, N) = setup.grid D = dimension() From e5fe5c7c860c3349e67dd0018a2009436e3582a0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Wed, 13 Dec 2023 15:31:07 +0100 Subject: [PATCH 199/379] Siplify expression --- src/operators.jl | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/src/operators.jl b/src/operators.jl index d8424b8f1..7fd7add18 100644 --- a/src/operators.jl +++ b/src/operators.jl @@ -882,13 +882,8 @@ Compute total kinetic energy. The velocity components are interpolated to the volume centers and squared. """ function total_kinetic_energy(u, setup) - (; dimension, Ω, Ip) = setup.grid - D = dimension() - up = interpolate_u_p(u, setup) - E = zero(eltype(up[1])) - for α = 1:D - # E += sum(I -> Ω[I] * up[α][I]^2, Ip) - E += sum(Ω[Ip] .* up[α][Ip] .^ 2) - end - E + (; Ω, Ip) = setup.grid + e = kinetic_energy(u, setup) + e .*= Ω + sum(e[Ip]) end From bfe98c871b0d39320ff8b6cf7d4b18aeaee75b35 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Wed, 13 Dec 2023 20:29:14 +0100 Subject: [PATCH 200/379] Fill DOFs only in divergence --- src/operators.jl | 19 +++++++------------ 1 file changed, 7 insertions(+), 12 deletions(-) diff --git a/src/operators.jl b/src/operators.jl index 7fd7add18..cdae84a93 100644 --- a/src/operators.jl +++ b/src/operators.jl @@ -19,7 +19,7 @@ Compute divergence of velocity field (in-place version). """ function divergence!(div, u, setup) (; grid, workgroupsize) = setup - (; Δ, N, Ip) = grid + (; Δ, N, Ip, Np) = grid D = length(u) δ = Offset{D}() @kernel function div!(div, u, I0) @@ -31,15 +31,9 @@ function divergence!(div, u, setup) end div[I] = d end - # All volumes have a right velocity - # All volumes have a left velocity except the first one - # Start at second volume - ndrange = N .- 1 - I0 = 2 * oneunit(first(Ip)) - # ndrange = Np - # I0 = first(Ip) + I0 = first(Ip) I0 -= oneunit(I0) - div!(get_backend(div), workgroupsize)(div, u, I0; ndrange) + div!(get_backend(div), workgroupsize)(div, u, I0; ndrange = Np) div end @@ -787,11 +781,11 @@ Compute the ``Q``-field. Qfield(u, setup) = Qfield!(similar(u[1], setup.grid.N), u, setup) """ - eig2field!(λ, u, setup) + eig2field!(λ, u, setup; ϵ = eps(eltype(λ))) Compute the second eigenvalue of ``S^2 + \\Omega^2``. """ -function eig2field!(λ, u, setup) +function eig2field!(λ, u, setup; ϵ = eps(eltype(λ))) (; grid, workgroupsize) = setup (; dimension, Np, Ip, Δ, Δu) = grid D = dimension() @@ -802,7 +796,8 @@ function eig2field!(λ, u, setup) ∇u = ∇(u, I, Δ, Δu) S = @. (∇u + ∇u') / 2 Ω = @. (∇u - ∇u') / 2 - λ[I] = eigvals(S^2 + Ω^2)[2] + e2 = eigvals(S^2 + Ω^2)[2] + λ[I] = log10(max(ϵ, -e2)) end I0 = first(Ip) I0 -= oneunit(I0) From 83450d87951e8e747023b67907066e8f6e60e998 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Wed, 13 Dec 2023 20:30:06 +0100 Subject: [PATCH 201/379] Remove unused args --- src/operators.jl | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/operators.jl b/src/operators.jl index cdae84a93..726d18521 100644 --- a/src/operators.jl +++ b/src/operators.jl @@ -511,7 +511,7 @@ The Smagorinsky constant `θ` should be a scalar between `0` and `1`. """ function smagtensor!(σ, u, θ, setup) # TODO: Combine with normal diffusion tensor - (; boundary_conditions, grid, workgroupsize) = setup + (; grid, workgroupsize) = setup (; Np, Ip, Δ, Δu) = grid @kernel function σ!(σ, u, I0) I = @index(Global, Cartesian) @@ -534,7 +534,7 @@ Compute the Smagorinsky closure term `s` (additional diffusive force). The Smagorinsky stress tensors should be precomputed and stored in `σ`. """ function smagorinsky!(s, σ, setup) - (; boundary_conditions, grid, workgroupsize) = setup + (; grid, workgroupsize) = setup (; dimension, Nu, Iu, Δ, Δu, A) = grid D = dimension() δ = Offset{D}() @@ -579,7 +579,7 @@ end """ m = smagorinsky_closure(setup) -Create Smagoinsky closure model `m`. +Create Smagorinsky closure model `m`. The model is called as `m(u, θ)`, where the Smagorinsky constant `θ` should be a scalar between `0` and `1` (for example `θ = 0.1`). """ @@ -612,7 +612,7 @@ interpolate_u_p(u, setup) = Interpolate velocity to pressure points. """ function interpolate_u_p!(up, u, setup) - (; boundary_conditions, grid, workgroupsize, Re, bodyforce) = setup + (; grid, workgroupsize) = setup (; dimension, Np, Ip) = grid D = dimension() δ = Offset{D}() @@ -649,7 +649,7 @@ Interpolate vorticity to pressure points. interpolate_ω_p!(ωp, ω, setup) = interpolate_ω_p!(setup.grid.dimension, ωp, ω, setup) function interpolate_ω_p!(::Dimension{2}, ωp, ω, setup) - (; boundary_conditions, grid, workgroupsize, Re, bodyforce) = setup + (; grid, workgroupsize) = setup (; dimension, Np, Ip) = grid D = dimension() δ = Offset{D}() @@ -665,7 +665,7 @@ function interpolate_ω_p!(::Dimension{2}, ωp, ω, setup) end function interpolate_ω_p!(::Dimension{3}, ωp, ω, setup) - (; boundary_conditions, grid, workgroupsize, Re) = setup + (; grid, workgroupsize) = setup (; dimension, Np, Ip) = grid D = dimension() δ = Offset{D}() @@ -694,7 +694,7 @@ D = \\frac{2 | \\nabla p |}{\\nabla^2 p}. ``` """ function Dfield!(d, G, p, setup; ϵ = eps(eltype(p))) - (; boundary_conditions, grid, workgroupsize) = setup + (; grid, workgroupsize) = setup (; dimension, Np, Ip, Δ) = grid T = eltype(p) D = dimension() @@ -752,7 +752,7 @@ Q = - \\frac{1}{2} \\sum_{α, β} \\frac{\\partial u^α}{\\partial x^β} ``` """ function Qfield!(Q, u, setup; ϵ = eps(eltype(Q))) - (; boundary_conditions, grid, workgroupsize) = setup + (; grid, workgroupsize) = setup (; dimension, Np, Ip, Δ) = grid D = dimension() δ = Offset{D}() From 54cadc7804969a24bcdc29189618797b9f959b20 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Wed, 13 Dec 2023 20:39:16 +0100 Subject: [PATCH 202/379] Add log to plots --- src/operators.jl | 11 +++++------ src/processors/real_time_plot.jl | 9 ++++++++- 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/src/operators.jl b/src/operators.jl index 726d18521..18eb4dc2f 100644 --- a/src/operators.jl +++ b/src/operators.jl @@ -693,7 +693,7 @@ Compute the ``D``-field [LiJiajia2019](@cite) given by D = \\frac{2 | \\nabla p |}{\\nabla^2 p}. ``` """ -function Dfield!(d, G, p, setup; ϵ = eps(eltype(p))) +function Dfield!(d, G, p, setup) (; grid, workgroupsize) = setup (; dimension, Np, Ip, Δ) = grid T = eltype(p) @@ -742,7 +742,7 @@ Dfield(p, setup) = Dfield!( ) """ - Qfield!(Q, u, setup; ϵ = eps(eltype(Q))) + Qfield!(Q, u, setup) Compute ``Q``-field [Jeong1995](@cite) given by @@ -751,7 +751,7 @@ Q = - \\frac{1}{2} \\sum_{α, β} \\frac{\\partial u^α}{\\partial x^β} \\frac{\\partial u^β}{\\partial x^α}. ``` """ -function Qfield!(Q, u, setup; ϵ = eps(eltype(Q))) +function Qfield!(Q, u, setup) (; grid, workgroupsize) = setup (; dimension, Np, Ip, Δ) = grid D = dimension() @@ -785,7 +785,7 @@ Qfield(u, setup) = Qfield!(similar(u[1], setup.grid.N), u, setup) Compute the second eigenvalue of ``S^2 + \\Omega^2``. """ -function eig2field!(λ, u, setup; ϵ = eps(eltype(λ))) +function eig2field!(λ, u, setup) (; grid, workgroupsize) = setup (; dimension, Np, Ip, Δ, Δu) = grid D = dimension() @@ -796,8 +796,7 @@ function eig2field!(λ, u, setup; ϵ = eps(eltype(λ))) ∇u = ∇(u, I, Δ, Δu) S = @. (∇u + ∇u') / 2 Ω = @. (∇u - ∇u') / 2 - e2 = eigvals(S^2 + Ω^2)[2] - λ[I] = log10(max(ϵ, -e2)) + λ[I] = eigvals(S^2 + Ω^2)[2] end I0 = first(Ip) I0 -= oneunit(I0) diff --git a/src/processors/real_time_plot.jl b/src/processors/real_time_plot.jl index 4709dd5c1..267d86186 100644 --- a/src/processors/real_time_plot.jl +++ b/src/processors/real_time_plot.jl @@ -185,9 +185,10 @@ function fieldplot( alpha = convert(eltype(setup.grid.x[1]), 0.1), isorange = convert(eltype(setup.grid.x[1]), 0.5), equal_axis = true, - levels = 3, + levels = LinRange{eltype(setup.grid.x[1])}(-10, 5, 10), docolorbar = false, size = nothing, + logtol = eps(setup.T), kwargs..., ) (; boundary_conditions, grid) = setup @@ -223,12 +224,18 @@ function fieldplot( p elseif fieldname == :Dfield Dfield!(d, G, p, setup) + din = view(d, Ip) + @. din = log(max(logtol, din)) d elseif fieldname == :Qfield Qfield!(Q, u, setup) + Qin = view(Q, Ip) + @. Qin = log(max(logtol, Qin)) Q elseif fieldname == :eig2field eig2field!(λ, u, setup) + λin = view(λ, Ip) + @. λin .= log(max(logtol, -λin)) λ end Array(f)[Ip] From 78062a4918b6be90adee150543333b079b13d3bd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Wed, 13 Dec 2023 20:54:40 +0100 Subject: [PATCH 203/379] Add adjoint prototype --- Project.toml | 1 + scratch/train_model.jl | 1 + src/operators.jl | 63 ++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 65 insertions(+) diff --git a/Project.toml b/Project.toml index 9504135cc..e0ff272ca 100644 --- a/Project.toml +++ b/Project.toml @@ -6,6 +6,7 @@ version = "0.4.1" [deps] Adapt = "79e6a3ab-5dfb-504d-930d-738a2a938a0e" CUDA = "052768ef-5323-5732-b1bb-66c8b64840ba" +ChainRulesCore = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4" ComponentArrays = "b0b7db55-cfe3-40fc-9ded-d10e2dbeff66" FFTW = "7a1cc6ca-52ef-59f5-83cd-3a7055c09341" IterativeSolvers = "42fd0dbc-a981-5370-80f2-aaf504508153" diff --git a/scratch/train_model.jl b/scratch/train_model.jl index 3d915dd4d..3bc0aa891 100644 --- a/scratch/train_model.jl +++ b/scratch/train_model.jl @@ -133,6 +133,7 @@ closure, θ₀ = cnn(; channels = [5, 5, 5, params.D], activations = [leakyrelu, leakyrelu, leakyrelu, identity], use_bias = [true, true, true, false], + rng, ); closure.chain diff --git a/src/operators.jl b/src/operators.jl index 18eb4dc2f..0d603ff52 100644 --- a/src/operators.jl +++ b/src/operators.jl @@ -1,3 +1,40 @@ +# Note on implementation: +# This file contains various differential operators. +# +# Each operator comes with +# +# - an modifying in-place version, e.g. `divergence!(div, u, setup)`, +# - an allocating out-of-place version, e.g. `div = divergence(u, setup)`. +# +# The out-of-place versions can be used as building blocks in a +# Zygote-differentiable program, thanks to the `rrule` methods +# defined. +# +# The domain is divided into `N = (N[1], ..., N[D])` finite volumes. +# For a Cartesian index `I`, volume center fields are naturally in the center, +# but volume face fields are always to the _right_ of volume I. +# +# _All_ fields have the size `N`. These `N` components include +# +# - degrees of freedom +# - boundary values, which are still used, but are filled in separately +# - unused values, which are never used at all. These are still there so that +# we can guarantee that `ω[I]`, `u[1][I]`, `u[2][I]`, and `p[I]` etc. are +# at their canonical position in to the volume `I`. Otherwise we would +# need an offset for each BC type and each combination. Asymptotically +# speaking (for large `N`), the additional memory footprint of having these +# around is negligible. +# +# The operators are implemented as kernels. +# The kernels are called for each index in `ndrange`, typically set +# to the degrees of freedom of the output quantity. Boundary values for the +# output quantity are filled in separately, by calling `apply_bc_*` when needed. +# It is assumed that the appropriate boundary values for the input fields are +# already filled in. +# +# The adjoint kernels are written manually for now. +# In the future, Enzyme.jl might be able to do this automatically. + """ δ = Offset{D}() @@ -37,6 +74,23 @@ function divergence!(div, u, setup) div end +function divergence_adjoint!(u, div, setup) + (; grid, workgroupsize) = setup + (; Δ, N, Ip) = grid + D = length(u) + δ = Offset{D}() + @kernel function adj!(u, div) + I = @index(Global, Cartesian) + for α = 1:D + u[α][I] = zero(eltype(u[1])) + I ∈ Ip && (u[α][I] += div[I] / Δ[α][I[α]]) + I + δ(α) ∈ Ip && (u[α][I] -= div[I+δ(α)] / Δ[α][I[α]+1]) + end + end + adj!(get_backend(u[1]), workgroupsize)(u, div; ndrange = N) + u +end + """ divergence(u, setup) @@ -44,6 +98,15 @@ Compute divergence of velocity field. """ divergence(u, setup) = divergence!(similar(u[1], setup.grid.N), u, setup) +ChainRulesCore.rrule(::typeof(divergence), u, setup) = ( + divergence(u, setup), + divbar -> ( + NoTangent(), + divergence_adjoint!(similar.(u), divbar, setup), + NoTangent(), + ), +) + """ vorticity(u, setup) From ed335bc09177aebfe18fd789157e5853b425c9eb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Thu, 14 Dec 2023 19:34:00 +0100 Subject: [PATCH 204/379] Add methods --- src/operators.jl | 86 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 86 insertions(+) diff --git a/src/operators.jl b/src/operators.jl index 0d603ff52..7cb456e31 100644 --- a/src/operators.jl +++ b/src/operators.jl @@ -11,6 +11,7 @@ # defined. # # The domain is divided into `N = (N[1], ..., N[D])` finite volumes. +# These also include ghost volumes, possibly outside the domain. # For a Cartesian index `I`, volume center fields are naturally in the center, # but volume face fields are always to the _right_ of volume I. # @@ -198,8 +199,46 @@ function convection!(F, u, setup) end F end + +function convection_adjoint!(ubar, Fbar, u, setup) + (; grid, workgroupsize) = setup + (; dimension, Δ, Δu, N, Nu, Iu, A) = grid + D = dimension() + δ = Offset{D}() + @kernel function adj!(ubar, Fbar, u, F, ::Val{α}, ::Val{βrange}) where {α,βrange} + I = @index(Global, Cartesian) + # for β = 1:D + KernelAbstractions.Extras.LoopInfo.@unroll for β in βrange + # Δuαβ = α == β ? Δu[β] : Δ[β] + # uαβ1 = A[α][β][2][I[β]-1] * u[α][I-δ(β)] + A[α][β][1][I[β]] * u[α][I] + # uβα1 = + # A[β][α][2][I[α]-(α==β)] * u[β][I-δ(β)] + + # A[β][α][1][I[α]+(α!=β)] * u[β][I-δ(β)+δ(α)] + # uαβ2 = A[α][β][2][I[β]] * u[α][I] + A[α][β][1][I[β]+1] * u[α][I+δ(β)] + # uβα2 = A[β][α][2][I[α]] * u[β][I] + A[β][α][1][I[α]+1] * u[β][I+δ(α)] + # F[α][I] -= (uαβ2 * uβα2 - uαβ1 * uβα1) / Δuαβ[I[β]] + + ubar[α][I] += Fbar[β][I] + end + end + error() + for α = 1:D + adj!(get_backend(F[1]), workgroupsize)(ubar, Fbar, u, Val(α), Val(1:D); ndrange = N) + end + F +end + convection(u, setup) = convection!(zero.(u), u, setup) +ChainRulesCore.rrule(::typeof(convection), u, setup) = ( + convection(u, setup), + φ -> ( + NoTangent(), + convection_adjoint!(similar.(u), φ, setup), + NoTangent(), + ), +) + """ diffusion!(F, u, setup) @@ -231,8 +270,45 @@ function diffusion!(F, u, setup) end F end + +function diffusion_adjoint!(u, F, setup) + (; grid, workgroupsize, Re) = setup + (; dimension, N, Δ, Δu, Iu) = grid + D = dimension() + δ = Offset{D}() + ν = 1 / Re + @kernel function adj!(u, F, ::Val{α}, ::Val{βrange}) where {α,βrange} + I = @index(Global, Cartesian) + # for β = 1:D + KernelAbstractions.Extras.LoopInfo.@unroll for β in βrange + Δuαβ = (α == β ? Δu[β] : Δ[β]) + # F[α][I] += ν * u[α][I+δ(β)] / (β == α ? Δ[β][I[β]+1] : Δu[β][I[β]]) + # F[α][I] -= ν * u[α][I] / (β == α ? Δ[β][I[β]+1] : Δu[β][I[β]]) + # F[α][I] -= ν * u[α][I] / (β == α ? Δ[β][I[β]] : Δu[β][I[β]-1]) + # F[α][I] += ν * u[α][I-δ(β)] / (β == α ? Δ[β][I[β]] : Δu[β][I[β]-1]) + I - δ(β) ∈ Iu[α] && (u[α][I] += ν * F[α][I-δ(β)] / (β == α ? Δ[β][I[β]] : Δu[β][I[β]-1]) / Δuαβ[I[β] - 1]) + I ∈ Iu[α] && (u[α][I] -= ν * F[α][I] / (β == α ? Δ[β][I[β]+1] : Δu[β][I[β]]) / Δuαβ[I[β]]) + I ∈ Iu[α] && (u[α][I] -= ν * F[α][I] / (β == α ? Δ[β][I[β]] : Δu[β][I[β]-1]) / Δuαβ[I[β]]) + I + δ(β) ∈ Iu[α] && (u[α][I] += ν * F[α][I+δ(β)] / (β == α ? Δ[β][I[β] + 1] : Δu[β][I[β]]) / Δuαβ[I[β] + 1]) + end + end + for α = 1:D + adj!(get_backend(u[1]), workgroupsize)(u, F, Val(α), Val(1:D); ndrange = N) + end + u +end + diffusion(u, setup) = diffusion!(zero.(u), u, setup) +ChainRulesCore.rrule(::typeof(diffusion), u, setup) = ( + diffusion(u, setup), + φ -> ( + NoTangent(), + diffusion_adjoint!(zero.(u), φ, setup), + NoTangent(), + ), +) + function convectiondiffusion!(F, u, setup) (; grid, workgroupsize, Re) = setup (; dimension, Δ, Δu, Nu, Iu, A) = grid @@ -265,8 +341,18 @@ function convectiondiffusion!(F, u, setup) end F end + convectiondiffusion(u, setup) = convectiondiffusion!(zero.(u), u, setup) +ChainRulesCore.rrule(::typeof(convectiondiffusion), u, setup) = ( + convection(u, setup), + φ -> ( + NoTangent(), + convectiondiffusion_adjoint!(similar.(u), φ, setup), + NoTangent(), + ), +) + """ bodyforce!(F, u, t, setup) From 0ccd83c4d8c018dd6eccbf0c12e13538459fe8c5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Fri, 15 Dec 2023 21:53:57 +0100 Subject: [PATCH 205/379] Add adjoints --- src/boundary_conditions.jl | 125 ++++++++++++++++++++ src/operators.jl | 194 ++++++++++++++++++++++--------- src/solvers/pressure/poisson.jl | 4 + src/solvers/pressure/pressure.jl | 26 +++-- 4 files changed, 286 insertions(+), 63 deletions(-) diff --git a/src/boundary_conditions.jl b/src/boundary_conditions.jl index 7001d95c3..60688b0dd 100644 --- a/src/boundary_conditions.jl +++ b/src/boundary_conditions.jl @@ -105,6 +105,31 @@ offset_p(::PressureBC, atend) = 1 + !atend function apply_bc_u! end function apply_bc_p! end +apply_bc_u(u, t, setup; kwargs...) = apply_bc_u!(copy(u), t, setup; kwargs...) +apply_bc_p(u, t, setup; kwargs...) = apply_bc_p!(copy(p), t, setup; kwargs...) + +ChainRulesCore.rrule(::typeof(apply_bc_u), u, t, setup) = ( + apply_bc_u(u, t, setup), + # With respect to (apply_bc_u, u, t, setup) + φbar -> ( + NoTangent(), + apply_bc_u_pullback!(copy.(φbar), φbar, t, setup), + NoTangent(), + NoTangent(), + ), +) + +ChainRulesCore.rrule(::typeof(apply_bc_p), p, t, setup) = ( + apply_bc_p(p, t, setup), + # With respect to (apply_bc_p, p, t, setup) + φbar -> ( + NoTangent(), + apply_bc_p_pullback!(copy.(φbar), φbar, t, setup), + NoTangent(), + NoTangent(), + ), +) + function apply_bc_u!(u, t, setup; kwargs...) (; boundary_conditions) = setup D = length(u) @@ -115,6 +140,34 @@ function apply_bc_u!(u, t, setup; kwargs...) u end +function apply_bc_u_pullback!(ubar, φbar, t, setup; kwargs...) + (; boundary_conditions) = setup + D = length(u) + for β = 1:D + apply_bc_u_pullback!( + boundary_conditions[β][1], + ubar, + φbar, + β, + t, + setup; + atend = false, + kwargs... + ) + apply_bc_u_pullback!( + boundary_conditions[β][2], + ubar, + φbar, + β, + t, + setup; + atend = true, + kwargs... + ) + end + ubar +end + function apply_bc_p!(p, t, setup; kwargs...) (; boundary_conditions, grid) = setup (; dimension) = grid @@ -126,6 +179,32 @@ function apply_bc_p!(p, t, setup; kwargs...) p end +function apply_bc_p_pullback!(pbar, φbar, t, setup; kwargs...) + (; boundary_conditions) = setup + D = length(u) + for β = 1:D + apply_bc_p_pullback!( + boundary_conditions[β][1], + pbar, + φbar, + β, + t, + setup; + atend = false, + ) + apply_bc_p_pullback!( + boundary_conditions[β][2], + pbar, + φbar, + β, + t, + setup; + atend = true, + ) + end + pbar +end + function apply_bc_u!(::PeriodicBC, u, β, t, setup; atend, kwargs...) (; grid, workgroupsize) = setup (; dimension, N) = grid @@ -150,6 +229,30 @@ function apply_bc_u!(::PeriodicBC, u, β, t, setup; atend, kwargs...) u end +function apply_bc_u_pullback!(::PeriodicBC, ubar, φbar, β, t, setup; atend, kwargs...) + (; grid, workgroupsize) = setup + (; dimension, N) = grid + D = dimension() + δ = Offset{D}() + @kernel function adj_a!(u, φ, ::Val{α}, ::Val{β}) where {α,β} + I = @index(Global, Cartesian) + u[α][I+(N[β]-2)*δ(β)] += φ[α][I] + end + @kernel function adj_b!(u, φ, ::Val{α}, ::Val{β}) where {α,β} + I = @index(Global, Cartesian) + φ[α][I+δ(β)] += u[α][I+(N[β]-1)*δ(β)] + end + ndrange = ntuple(γ -> γ == β ? 1 : N[γ], D) + for α = 1:D + if atend + _bc_b!(get_backend(u[1]), workgroupsize)(ubar, φbar, Val(α), Val(β); ndrange) + else + _bc_a!(get_backend(u[1]), workgroupsize)(ubar, φbar, Val(α), Val(β); ndrange) + end + end + u +end + function apply_bc_p!(::PeriodicBC, p, β, t, setup; atend, kwargs...) (; grid, workgroupsize) = setup (; dimension, N) = grid @@ -172,6 +275,28 @@ function apply_bc_p!(::PeriodicBC, p, β, t, setup; atend, kwargs...) p end +function apply_bc_p_pullback!(::PeriodicBC, pbar, φbar, β, t, setup; atend, kwargs...) + (; grid, workgroupsize) = setup + (; dimension, N) = grid + D = dimension() + δ = Offset{D}() + @kernel function _bc_a(p, ::Val{β}) where {β} + I = @index(Global, Cartesian) + p[I+(N[β]-2)*δ(β)] += φ[I] + end + @kernel function _bc_b(p, ::Val{β}) where {β} + I = @index(Global, Cartesian) + p[I+δ(β)] += p[I+(N[β]-1)*δ(β)] + end + ndrange = ntuple(γ -> γ == β ? 1 : N[γ], D) + if atend + _bc_b(get_backend(p), workgroupsize)(pbar, φbar, Val(β); ndrange) + else + _bc_a(get_backend(p), workgroupsize)(pbar, φbar, Val(β); ndrange) + end + p +end + function apply_bc_u!(bc::DirichletBC, u, β, t, setup; atend, dudt = false, kwargs...) (; dimension, x, xp, N) = setup.grid D = dimension() diff --git a/src/operators.jl b/src/operators.jl index 7cb456e31..d1e7a12f0 100644 --- a/src/operators.jl +++ b/src/operators.jl @@ -75,20 +75,20 @@ function divergence!(div, u, setup) div end -function divergence_adjoint!(u, div, setup) +function divergence_adjoint!(u, φ, setup) (; grid, workgroupsize) = setup (; Δ, N, Ip) = grid D = length(u) δ = Offset{D}() - @kernel function adj!(u, div) + @kernel function adj!(u, φ) I = @index(Global, Cartesian) for α = 1:D u[α][I] = zero(eltype(u[1])) - I ∈ Ip && (u[α][I] += div[I] / Δ[α][I[α]]) - I + δ(α) ∈ Ip && (u[α][I] -= div[I+δ(α)] / Δ[α][I[α]+1]) + I ∈ Ip && (u[α][I] += φ[I] / Δ[α][I[α]]) + I + δ(α) ∈ Ip && (u[α][I] -= φ[I+δ(α)] / Δ[α][I[α]+1]) end end - adj!(get_backend(u[1]), workgroupsize)(u, div; ndrange = N) + adj!(get_backend(u[1]), workgroupsize)(u, φ; ndrange = N) u end @@ -101,11 +101,7 @@ divergence(u, setup) = divergence!(similar(u[1], setup.grid.N), u, setup) ChainRulesCore.rrule(::typeof(divergence), u, setup) = ( divergence(u, setup), - divbar -> ( - NoTangent(), - divergence_adjoint!(similar.(u), divbar, setup), - NoTangent(), - ), + φ -> (NoTangent(), divergence_adjoint!(similar.(u), φ, setup), NoTangent()), ) """ @@ -200,43 +196,110 @@ function convection!(F, u, setup) F end -function convection_adjoint!(ubar, Fbar, u, setup) +function convection_adjoint!(ubar, φbar, u, setup) (; grid, workgroupsize) = setup - (; dimension, Δ, Δu, N, Nu, Iu, A) = grid + (; dimension, Δ, Δu, N, Iu, A) = grid D = dimension() δ = Offset{D}() - @kernel function adj!(ubar, Fbar, u, F, ::Val{α}, ::Val{βrange}) where {α,βrange} - I = @index(Global, Cartesian) - # for β = 1:D - KernelAbstractions.Extras.LoopInfo.@unroll for β in βrange - # Δuαβ = α == β ? Δu[β] : Δ[β] - # uαβ1 = A[α][β][2][I[β]-1] * u[α][I-δ(β)] + A[α][β][1][I[β]] * u[α][I] - # uβα1 = - # A[β][α][2][I[α]-(α==β)] * u[β][I-δ(β)] + - # A[β][α][1][I[α]+(α!=β)] * u[β][I-δ(β)+δ(α)] - # uαβ2 = A[α][β][2][I[β]] * u[α][I] + A[α][β][1][I[β]+1] * u[α][I+δ(β)] - # uβα2 = A[β][α][2][I[α]] * u[β][I] + A[β][α][1][I[α]+1] * u[β][I+δ(α)] - # F[α][I] -= (uαβ2 * uβα2 - uαβ1 * uβα1) / Δuαβ[I[β]] - - ubar[α][I] += Fbar[β][I] + @kernel function adj!(ubar, φbar, u, ::Val{γ}, ::Val{looprange}) where {γ,looprange} + J = @index(Global, Cartesian) + KernelAbstractions.Extras.LoopInfo.@unroll for α in looprange + KernelAbstractions.Extras.LoopInfo.@unroll for β in looprange + Δuαβ = α == β ? Δu[β] : Δ[β] + Aαβ1 = A[α][β][1] + Aαβ2 = A[α][β][2] + Aβα1 = A[β][α][1] + Aβα2 = A[β][α][2] + + # 1 + I = J + if α == γ && I in Iu[α] + uαβ2 = Aαβ2[I[β]] + uβα2 = Aβα2[I[α]] * u[β][I] + Aβα1[I[α]+1] * u[β][I+δ(α)] + dφdu = -uαβ2 * uβα2 / Δuαβ[I[β]] + ubar[γ][J] += φbar[α][I] * dφdu + end + + # 2 + I = J - δ(β) + if α == γ && I in Iu[α] + uαβ2 = Aαβ1[I[β]+1] + uβα2 = Aβα2[I[α]] * u[β][I] + Aβα1[I[α]+1] * u[β][I+δ(α)] + dφdu = -uαβ2 * uβα2 / Δuαβ[I[β]] + ubar[γ][J] += φbar[α][I] * dφdu + end + + # 3 + I = J + if β == γ && I in Iu[α] + uαβ2 = Aαβ2[I[β]] * u[α][I] + Aαβ1[I[β]+1] * u[α][I+δ(β)] + uβα2 = Aβα2[I[α]] + dφdu = -uαβ2 * uβα2 / Δuαβ[I[β]] + ubar[γ][J] += φbar[α][I] * dφdu + end + + # 4 + I = J - δ(α) + if β == γ && I in Iu[α] + uαβ2 = Aαβ2[I[β]] * u[α][I] + Aαβ1[I[β]+1] * u[α][I+δ(β)] + uβα2 = Aβα1[I[α]+1] + dφdu = -uαβ2 * uβα2 / Δuαβ[I[β]] + ubar[γ][J] += φbar[α][I] * dφdu + end + + # 5 + I = J + δ(β) + if α == γ && I in Iu[α] + uαβ1 = Aαβ2[I[β]-1] + uβα1 = + Aβα2[I[α]-(α==β)] * u[β][I-δ(β)] + + Aβα1[I[α]+(α!=β)] * u[β][I-δ(β)+δ(α)] + dφdu = uαβ1 * uβα1 / Δuαβ[I[β]] + ubar[γ][J] += φbar[α][I] * dφdu + end + + # 6 + I = J + if α == γ && I in Iu[α] + uαβ1 = Aαβ1[I[β]] + uβα1 = + Aβα2[I[α]-(α==β)] * u[β][I-δ(β)] + + Aβα1[I[α]+(α!=β)] * u[β][I-δ(β)+δ(α)] + dφdu = uαβ1 * uβα1 / Δuαβ[I[β]] + ubar[γ][J] += φbar[α][I] * dφdu + end + + # 7 + I = J + δ(β) + if β == γ && I in Iu[α] + uαβ1 = Aαβ2[I[β]-1] * u[α][I-δ(β)] + Aαβ1[I[β]] * u[α][I] + uβα1 = Aβα2[I[α]-(α==β)] + dφdu = uαβ1 * uβα1 / Δuαβ[I[β]] + ubar[γ][J] += φbar[α][I] * dφdu + end + + # 8 + I = J + δ(β) - δ(α) + if β == γ && I in Iu[α] + uαβ1 = Aαβ2[I[β]-1] * u[α][I-δ(β)] + Aαβ1[I[β]] * u[α][I] + uβα1 = Aβα1[I[α]+(α!=β)] + dφdu = uαβ1 * uβα1 / Δuαβ[I[β]] + ubar[γ][J] += φbar[α][I] * dφdu + end + end end end - error() - for α = 1:D - adj!(get_backend(F[1]), workgroupsize)(ubar, Fbar, u, Val(α), Val(1:D); ndrange = N) + for γ = 1:D + adj!(get_backend(u[1]), workgroupsize)(ubar, φbar, u, Val(γ), Val(1:D); ndrange = N) end - F + ubar end convection(u, setup) = convection!(zero.(u), u, setup) ChainRulesCore.rrule(::typeof(convection), u, setup) = ( convection(u, setup), - φ -> ( - NoTangent(), - convection_adjoint!(similar.(u), φ, setup), - NoTangent(), - ), + φ -> (NoTangent(), convection_adjoint!(similar.(u), (φ...,), u, setup), NoTangent()), ) """ @@ -271,13 +334,13 @@ function diffusion!(F, u, setup) F end -function diffusion_adjoint!(u, F, setup) +function diffusion_adjoint!(u, φ, setup) (; grid, workgroupsize, Re) = setup (; dimension, N, Δ, Δu, Iu) = grid D = dimension() δ = Offset{D}() ν = 1 / Re - @kernel function adj!(u, F, ::Val{α}, ::Val{βrange}) where {α,βrange} + @kernel function adj!(u, φ, ::Val{α}, ::Val{βrange}) where {α,βrange} I = @index(Global, Cartesian) # for β = 1:D KernelAbstractions.Extras.LoopInfo.@unroll for β in βrange @@ -286,14 +349,26 @@ function diffusion_adjoint!(u, F, setup) # F[α][I] -= ν * u[α][I] / (β == α ? Δ[β][I[β]+1] : Δu[β][I[β]]) # F[α][I] -= ν * u[α][I] / (β == α ? Δ[β][I[β]] : Δu[β][I[β]-1]) # F[α][I] += ν * u[α][I-δ(β)] / (β == α ? Δ[β][I[β]] : Δu[β][I[β]-1]) - I - δ(β) ∈ Iu[α] && (u[α][I] += ν * F[α][I-δ(β)] / (β == α ? Δ[β][I[β]] : Δu[β][I[β]-1]) / Δuαβ[I[β] - 1]) - I ∈ Iu[α] && (u[α][I] -= ν * F[α][I] / (β == α ? Δ[β][I[β]+1] : Δu[β][I[β]]) / Δuαβ[I[β]]) - I ∈ Iu[α] && (u[α][I] -= ν * F[α][I] / (β == α ? Δ[β][I[β]] : Δu[β][I[β]-1]) / Δuαβ[I[β]]) - I + δ(β) ∈ Iu[α] && (u[α][I] += ν * F[α][I+δ(β)] / (β == α ? Δ[β][I[β] + 1] : Δu[β][I[β]]) / Δuαβ[I[β] + 1]) + I - δ(β) ∈ Iu[α] && ( + u[α][I] += + ν * φ[α][I-δ(β)] / (β == α ? Δ[β][I[β]] : Δu[β][I[β]-1]) / Δuαβ[I[β]-1] + ) + I ∈ Iu[α] && ( + u[α][I] -= + ν * φ[α][I] / (β == α ? Δ[β][I[β]+1] : Δu[β][I[β]]) / Δuαβ[I[β]] + ) + I ∈ Iu[α] && ( + u[α][I] -= + ν * φ[α][I] / (β == α ? Δ[β][I[β]] : Δu[β][I[β]-1]) / Δuαβ[I[β]] + ) + I + δ(β) ∈ Iu[α] && ( + u[α][I] += + ν * φ[α][I+δ(β)] / (β == α ? Δ[β][I[β]+1] : Δu[β][I[β]]) / Δuαβ[I[β]+1] + ) end end for α = 1:D - adj!(get_backend(u[1]), workgroupsize)(u, F, Val(α), Val(1:D); ndrange = N) + adj!(get_backend(u[1]), workgroupsize)(u, φ, Val(α), Val(1:D); ndrange = N) end u end @@ -302,11 +377,7 @@ diffusion(u, setup) = diffusion!(zero.(u), u, setup) ChainRulesCore.rrule(::typeof(diffusion), u, setup) = ( diffusion(u, setup), - φ -> ( - NoTangent(), - diffusion_adjoint!(zero.(u), φ, setup), - NoTangent(), - ), + φ -> (NoTangent(), diffusion_adjoint!(zero.(u), (φ...,), setup), NoTangent()), ) function convectiondiffusion!(F, u, setup) @@ -346,11 +417,8 @@ convectiondiffusion(u, setup) = convectiondiffusion!(zero.(u), u, setup) ChainRulesCore.rrule(::typeof(convectiondiffusion), u, setup) = ( convection(u, setup), - φ -> ( - NoTangent(), - convectiondiffusion_adjoint!(similar.(u), φ, setup), - NoTangent(), - ), + φ -> + (NoTangent(), convectiondiffusion_adjoint!(similar.(u), φ, setup), NoTangent()), ) """ @@ -445,6 +513,23 @@ function pressuregradient!(G, p, setup) G end +function pressuregradient_adjoint!(pbar, φ, setup) + (; grid, workgroupsize) = setup + (; dimension, Δu, N, Iu) = grid + D = dimension() + δ = Offset{D}() + @kernel function adj!(p, φ) + I = @index(Global, Cartesian) + p[I] = zero(eltype(p)) + for α = 1:D + I - δ(α) ∈ Iu[α] && (p[I] += φ[α][I-δ(α)] / Δu[α][I[α] - 1]) + I ∈ Iu[α] && (p[I] -= φ[α][I] / Δu[α][I[α]]) + end + end + adj!(get_backend(pbar), workgroupsize)(pbar, φ; ndrange = N) + pbar +end + """ pressuregradient(p, setup) @@ -456,6 +541,11 @@ pressuregradient(p, setup) = pressuregradient!( setup, ) +ChainRulesCore.rrule(::typeof(pressuregradient), p, setup) = ( + pressuregradient(p, setup), + φ -> (NoTangent(), pressuregradient_adjoint!(similar(p), (φ...,), setup), NoTangent()), +) + """ laplacian!(L, p, setup) diff --git a/src/solvers/pressure/poisson.jl b/src/solvers/pressure/poisson.jl index 1217675fc..4f8c664c4 100644 --- a/src/solvers/pressure/poisson.jl +++ b/src/solvers/pressure/poisson.jl @@ -12,6 +12,10 @@ function poisson end poisson(solver, f) = poisson!(solver, zero(f), f) +# Laplacian is auto-adjoint +ChainRulesCore.rrule(::typeof(poisson), f, setup) = + (poisson(p, setup), φ -> (NoTangent(), poisson(φ, setup), NoTangent())) + """ poisson!(solver, p, f) diff --git a/src/solvers/pressure/pressure.jl b/src/solvers/pressure/pressure.jl index f76293384..543b7f2ff 100644 --- a/src/solvers/pressure/pressure.jl +++ b/src/solvers/pressure/pressure.jl @@ -1,10 +1,10 @@ """ - pressure!(pressure_solver, u, p, t, setup, F, f, M) + pressure!(solver, u, p, t, setup, F, f, M) Compute pressure from velocity field. This makes the pressure compatible with the velocity field, resulting in same order pressure as velocity. """ -function pressure!(pressure_solver, u, p, t, setup, F, G, M) +function pressure!(solver, u, p, t, setup, F, G, M) (; grid) = setup (; dimension, Iu, Ip, Ω) = grid D = dimension() @@ -12,22 +12,26 @@ function pressure!(pressure_solver, u, p, t, setup, F, G, M) apply_bc_u!(F, t, setup; dudt = true) divergence!(M, F, setup) @. M *= Ω - poisson!(pressure_solver, p, M) + poisson!(solver, p, M) apply_bc_p!(p, t, setup) p end """ - pressure(pressure_solver, u, t, setup) + pressure(solver, u, t, setup) Do additional pressure solve. This makes the pressure compatible with the velocity field, resulting in same order pressure as velocity. """ -function pressure(pressure_solver, u, t, setup) - D = setup.grid.dimension() - p = similar(u[1], setup.grid.N) - F = similar.(u) - G = similar.(u) - M = similar(u[1], setup.grid.N) - pressure!(pressure_solver, u, p, t, setup, F, G, M) +function pressure(solver, u, t, setup) + (; grid) = setup + (; dimension, Iu, Ip, Ω) = grid + D = dimension() + F = momentum(u, t, setup) + F = apply_bc_u(F, t, setup; dudt = true) + M = divergence(F, setup) + M = @. M * Ω + p = poisson(solver, M) + p = apply_bc_p(p, t, setup) + p end From 3b66c65f0781df765b1c23c665bd431d19f5b761 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Fri, 15 Dec 2023 21:54:49 +0100 Subject: [PATCH 206/379] Add projector --- src/IncompressibleNavierStokes.jl | 1 + src/solvers/pressure/project.jl | 39 +++++++++++++++++++++++++++++++ 2 files changed, 40 insertions(+) create mode 100644 src/solvers/pressure/project.jl diff --git a/src/IncompressibleNavierStokes.jl b/src/IncompressibleNavierStokes.jl index fa43751f4..54ef2faf3 100644 --- a/src/IncompressibleNavierStokes.jl +++ b/src/IncompressibleNavierStokes.jl @@ -52,6 +52,7 @@ include("setup.jl") include("solvers/pressure/solvers.jl") include("solvers/pressure/poisson.jl") include("solvers/pressure/pressure.jl") +include("solvers/pressure/project.jl") # Time steppers include("time_steppers/methods.jl") diff --git a/src/solvers/pressure/project.jl b/src/solvers/pressure/project.jl new file mode 100644 index 000000000..beeae8091 --- /dev/null +++ b/src/solvers/pressure/project.jl @@ -0,0 +1,39 @@ +function project(solver, u, setup) + (; Ω) = setup.grid + T = eltype(u[1]) + + # Divergence of tentative velocity field + M = divergence(v, setup) + M = @. M * Ω + + # Solve the Poisson equation + p = poisson(pressure_solver, M) + p = apply_bc_p(p, T(0), setup) + + # Compute pressure correction term + G = pressuregradient(p, setup) + + # Update velocity, which is now divergence free + v .- G +end + +function project!(solver, u, setup; M, p, G) + (; Ω) = setup.grid + T = eltype(u[1]) + + # Divergence of tentative velocity field + divergence!(M, u, setup) + @. M *= Ω + + # Solve the Poisson equation + poisson!(pressure_solver, p, M) + apply_bc_p!(p, T(0), setup) + + # Compute pressure correction term + pressuregradient!(G, p, setup) + + # Update velocity, which is now divergence free + ntuple(length(u)) do α + @. u[α] -= G[α] + end +end From 3eee0ab35159216fb6e785e3f931ddac22ac138e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Fri, 15 Dec 2023 21:55:02 +0100 Subject: [PATCH 207/379] Add missing import --- src/IncompressibleNavierStokes.jl | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/IncompressibleNavierStokes.jl b/src/IncompressibleNavierStokes.jl index 54ef2faf3..dcd3828cc 100644 --- a/src/IncompressibleNavierStokes.jl +++ b/src/IncompressibleNavierStokes.jl @@ -6,6 +6,7 @@ Energy-conserving solvers for the incompressible Navier-Stokes equations. module IncompressibleNavierStokes using Adapt +using ChainRulesCore using ComponentArrays: ComponentArray using FFTW using IterativeSolvers @@ -27,7 +28,10 @@ using Zygote # Must be loaded inside for Tullio to work correctly using CUDA using CUDA.CUSPARSE -using CUSOLVERRF +# using CUSOLVERRF + +# Must be loaded inside for debugging with breakpoints +using Infiltrator # # Easily retrieve value from Val # (::Val{x})() where {x} = x From b4a6c460cbed23241424aed08b7db06ad9cbeff6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Fri, 15 Dec 2023 21:59:57 +0100 Subject: [PATCH 208/379] Add differentiable stepper --- .../step_explicit_runge_kutta.jl | 57 ++++++++++++++++++- 1 file changed, 56 insertions(+), 1 deletion(-) diff --git a/src/time_steppers/step_explicit_runge_kutta.jl b/src/time_steppers/step_explicit_runge_kutta.jl index 390cd44e9..18718bdbb 100644 --- a/src/time_steppers/step_explicit_runge_kutta.jl +++ b/src/time_steppers/step_explicit_runge_kutta.jl @@ -3,7 +3,7 @@ create_stepper(::ExplicitRungeKuttaMethod; setup, pressure_solver, u, p, t, n = function timestep!(method::ExplicitRungeKuttaMethod, stepper, Δt; cache) (; setup, pressure_solver, u, p, t, n) = stepper - (; grid, boundary_conditions) = setup + (; grid) = setup (; dimension, Iu, Ip, Ω) = grid (; A, b, c, p_add_solve) = method (; u₀, ku, v, F, M, G) = cache @@ -77,3 +77,58 @@ function timestep!(method::ExplicitRungeKuttaMethod, stepper, Δt; cache) create_stepper(method; setup, pressure_solver, u, p, t, n = n + 1) end + +function timestep(method::ExplicitRungeKuttaMethod, stepper, Δt) + (; setup, pressure_solver, u, p, t, n) = stepper + (; grid) = setup + (; dimension) = grid + (; A, b, c) = method + + D = dimension() + + # Update current solution (does not depend on previous step size) + t₀ = t + u₀ = u + + # Number of stages + nstage = length(b) + + ku = () + + ## Start looping over stages + + # At i = 1 we calculate F₁ = F(u₀), p₁ and u₁ + # ⋮ + # At i = s we calculate Fₛ = F(uₛ₋₁), pₛ, and uₛ + for i = 1:nstage + # Right-hand side for tᵢ₋₁ based on current velocity field uᵢ₋₁, vᵢ₋₁ at + # level i-1. This includes force evaluation at tᵢ₋₁. + F = momentum(u, t, setup) + + # Store right-hand side of stage i + ku = (ku..., F) + + # Intermediate time step + t = t₀ + c[i] * Δt + + # Update velocity current stage by sum of Fᵢ's until this stage, weighted + # with Butcher tableau coefficients. This gives vᵢ + u = ntuple(D) do α + uα = u₀[α] + for j = 1:i + uα = @. uα + Δt * A[i, j] * ku[j][α] + end + uα + end + + # Boundary conditions at tᵢ + u = apply_bc_u(u, t, setup) + u = project(pressure_solver, u, setup) + u = apply_bc_u(u, t, setup) + end + + # Complete time step + t = t₀ + Δt + + create_stepper(method; setup, pressure_solver, u, p, t, n = n + 1) +end From 44bfd0da63777c3b9e9e7232fca80297aee42778 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Fri, 15 Dec 2023 23:24:04 +0100 Subject: [PATCH 209/379] Remove debugger --- src/IncompressibleNavierStokes.jl | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/IncompressibleNavierStokes.jl b/src/IncompressibleNavierStokes.jl index dcd3828cc..908079a0c 100644 --- a/src/IncompressibleNavierStokes.jl +++ b/src/IncompressibleNavierStokes.jl @@ -30,9 +30,6 @@ using CUDA using CUDA.CUSPARSE # using CUSOLVERRF -# Must be loaded inside for debugging with breakpoints -using Infiltrator - # # Easily retrieve value from Val # (::Val{x})() where {x} = x From f3d1a4fcb1eac744360c069ad7687c9e5e80067c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Sat, 16 Dec 2023 00:11:38 +0100 Subject: [PATCH 210/379] Minor changes --- src/boundary_conditions.jl | 14 +++++++------- src/operators.jl | 26 ++++++++++++++++++++++++-- 2 files changed, 31 insertions(+), 9 deletions(-) diff --git a/src/boundary_conditions.jl b/src/boundary_conditions.jl index 60688b0dd..7ed7591ce 100644 --- a/src/boundary_conditions.jl +++ b/src/boundary_conditions.jl @@ -105,8 +105,8 @@ offset_p(::PressureBC, atend) = 1 + !atend function apply_bc_u! end function apply_bc_p! end -apply_bc_u(u, t, setup; kwargs...) = apply_bc_u!(copy(u), t, setup; kwargs...) -apply_bc_p(u, t, setup; kwargs...) = apply_bc_p!(copy(p), t, setup; kwargs...) +apply_bc_u(u, t, setup; kwargs...) = apply_bc_u!(copy.(u), t, setup; kwargs...) +apply_bc_p(p, t, setup; kwargs...) = apply_bc_p!(copy(p), t, setup; kwargs...) ChainRulesCore.rrule(::typeof(apply_bc_u), u, t, setup) = ( apply_bc_u(u, t, setup), @@ -142,7 +142,7 @@ end function apply_bc_u_pullback!(ubar, φbar, t, setup; kwargs...) (; boundary_conditions) = setup - D = length(u) + D = length(ubar) for β = 1:D apply_bc_u_pullback!( boundary_conditions[β][1], @@ -240,17 +240,17 @@ function apply_bc_u_pullback!(::PeriodicBC, ubar, φbar, β, t, setup; atend, kw end @kernel function adj_b!(u, φ, ::Val{α}, ::Val{β}) where {α,β} I = @index(Global, Cartesian) - φ[α][I+δ(β)] += u[α][I+(N[β]-1)*δ(β)] + u[α][I+δ(β)] += φ[α][I+(N[β]-1)*δ(β)] end ndrange = ntuple(γ -> γ == β ? 1 : N[γ], D) for α = 1:D if atend - _bc_b!(get_backend(u[1]), workgroupsize)(ubar, φbar, Val(α), Val(β); ndrange) + adj_b!(get_backend(ubar[1]), workgroupsize)(ubar, φbar, Val(α), Val(β); ndrange) else - _bc_a!(get_backend(u[1]), workgroupsize)(ubar, φbar, Val(α), Val(β); ndrange) + adj_a!(get_backend(ubar[1]), workgroupsize)(ubar, φbar, Val(α), Val(β); ndrange) end end - u + ubar end function apply_bc_p!(::PeriodicBC, p, β, t, setup; atend, kwargs...) diff --git a/src/operators.jl b/src/operators.jl index d1e7a12f0..4fb042fda 100644 --- a/src/operators.jl +++ b/src/operators.jl @@ -455,6 +455,12 @@ function bodyforce!(F, u, t, setup) end F end +bodyforce(u, t, setup) = bodyforce!(zero.(u), u, t, setup) + +ChainRulesCore.rrule(::typeof(bodyforce), u, t, setup) = ( + bodyforce(u, t, setup), + φ -> (NoTangent(), ZeroTangent(), NoTangent(), NoTangent()), +) """ momentum!(F, u, t, setup) @@ -487,7 +493,23 @@ end Right hand side of momentum equations, excluding pressure gradient. """ -momentum(u, t, setup) = momentum!(zero.(u), u, t, setup) +# momentum(u, t, setup) = momentum!(zero.(u), u, t, setup) +function momentum(u, t, setup) + (; grid, closure_model) = setup + (; dimension) = grid + D = dimension() + d = diffusion(u, setup) + c = convection(u, setup) + f = bodyforce(u, t, setup) + F = ntuple(D) do α + d[α] .+ c[α] .+ f[α] + end + if !isnothing(closure_model) + m = closure_model(u) + F = F .+ m + end + F +end """ pressuregradient!(G, p, setup) @@ -536,7 +558,7 @@ end Compute pressure gradient. """ pressuregradient(p, setup) = pressuregradient!( - ntuple(α -> similar(p, setup.grid.N), setup.grid.dimension()), + ntuple(α -> similar(p), setup.grid.dimension()), p, setup, ) From 8bb30bb7ea8cdd88403ad1671930396c0f99e876 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Mon, 18 Dec 2023 15:56:42 +0100 Subject: [PATCH 211/379] Update pullbacks --- scratch/testgrad.jl | 241 ++++++++++++++++++ src/boundary_conditions.jl | 19 +- src/operators.jl | 47 ++-- src/solvers/pressure/poisson.jl | 8 +- src/solvers/pressure/project.jl | 8 +- .../step_explicit_runge_kutta.jl | 16 +- 6 files changed, 299 insertions(+), 40 deletions(-) create mode 100644 scratch/testgrad.jl diff --git a/scratch/testgrad.jl b/scratch/testgrad.jl new file mode 100644 index 000000000..6dc3d08af --- /dev/null +++ b/scratch/testgrad.jl @@ -0,0 +1,241 @@ +# Little LSP hack to get function signatures, go #src +# to definition etc. #src +if isdefined(@__MODULE__, :LanguageServer) #src + include("../src/IncompressibleNavierStokes.jl") #src + using .IncompressibleNavierStokes #src +end #src + +# # Decaying Homogeneous Isotropic Turbulence - 2D +# +# In this example we consider decaying homogeneous isotropic turbulence, +# similar to the cases considered in [Kochkov2021](@cite) and +# [Kurz2022](@cite). The initial velocity field is created randomly, but with a +# specific energy spectrum. Due to viscous dissipation, the turbulent features +# eventually group to form larger visible eddies. + +# We start by loading packages. +# A [Makie](https://github.com/JuliaPlots/Makie.jl) plotting backend is needed +# for plotting. `GLMakie` creates an interactive window (useful for real-time +# plotting), but does not work when building this example on GitHub. +# `CairoMakie` makes high-quality static vector-graphics plots. + +#md using CairoMakie +using GLMakie #!md +using IncompressibleNavierStokes + +set_theme!(; GLMakie = (; scalefactor = 1.5)) + +# Output directory +output = "output/DecayingTurbulence2D" + +# Floating point precision +T = Float64 + +# Array type +ArrayType = Array +## using CUDA; ArrayType = CuArray +## using AMDGPU; ArrayType = ROCArray +## using oneAPI; ArrayType = oneArray +## using Metal; ArrayType = MtlArray + +using CUDA; +T = Float32; +ArrayType = CuArray; +CUDA.allowscalar(false); + +# Viscosity model +Re = T(10_000) + +# A 2D grid is a Cartesian product of two vectors +n = 8 +# n = 32 +# n = 128 +# n = 1024 +# n = 2056 +lims = T(0), T(1) +x = LinRange(lims..., n + 1), LinRange(lims..., n + 1) +# plotgrid(x...) + +# Build setup and assemble operators +setup = Setup(x...; Re, ArrayType); + +# Since the grid is uniform and identical for x and y, we may use a specialized +# spectral pressure solver +pressure_solver = SpectralPressureSolver(setup); + +u₀, p₀ = random_field(setup, T(0); pressure_solver); +u, p = u₀, p₀ + +using KernelAbstractions +using Zygote +using LinearAlgebra +using Random +(; Iu, Ip) = setup.grid + +IncompressibleNavierStokes.diffusion(u, setup)[1] +IncompressibleNavierStokes.diffusion_adjoint!(zero.(u), u, setup)[1] + +gradient(u -> sum(IncompressibleNavierStokes.diffusion(u, setup)[1]), u)[1][1] + +# Divergence +ur = randn!.(similar.(u)) +φ = IncompressibleNavierStokes.divergence!(zero(p), ur, setup) +φbar = randn!(similar(φ)) +dot(φbar, φ) +dot(IncompressibleNavierStokes.divergence_adjoint!(zero.(ur), φbar, setup), ur) + +# Diffusion +φ = IncompressibleNavierStokes.diffusion(u, setup) +φbar = randn!.(similar.(φ)) +ubar = IncompressibleNavierStokes.diffusion_adjoint!(zero.(u), φbar, setup) +dot(φbar, φ) +dot(ubar, u) + +# Convection +ur = randn!.(similar.(u)) +# ur = zero.(u) +# for α = 1:length(ur) +# ur[α][Iu[α]] .= randn.() +# end +# ur = u +φ = IncompressibleNavierStokes.convection!(zero.(ur), ur, setup) +φbar = randn!.(similar.(φ)) +ubar = IncompressibleNavierStokes.convection_adjoint!(zero.(ur), φbar, ur, setup) +dot(φbar, φ) +dot(ubar, ur) / 2 + +I = CartesianIndex(14, 17) +g1 = gradient(p) do p + φ = IncompressibleNavierStokes.pressuregradient(p, setup) + dot(φ, φ) + # sum(φ[1]) +end +g2 = begin + p1 = copy(p) + CUDA.@allowscalar p1[I] -= sqrt(eps(T)) / 2 + φ = IncompressibleNavierStokes.pressuregradient(p1, setup) + r1 = dot(φ, φ) + p2 = copy(p) + CUDA.@allowscalar p2[I] += sqrt(eps(T)) / 2 + φ = IncompressibleNavierStokes.pressuregradient(p2, setup) + r2 = dot(φ, φ) + (r2 - r1) / sqrt(eps(T)) +end +CUDA.@allowscalar g1[1][I] +g2 + +g1 = gradient(u) do u + # φ = IncompressibleNavierStokes.convection(u, setup) + dot(φ, φ) + # sum(φ[1]) +end + +IncompressibleNavierStokes.momentum(u, T(0), setup) +IncompressibleNavierStokes.apply_bc_u(u, T(0), setup)[1] +u[1] + +function f(u, setup) + (; Iu) = setup.grid + u = IncompressibleNavierStokes.apply_bc_u(u, T(0), setup) + φ = IncompressibleNavierStokes.momentum(u, T(0), setup) + # φ = IncompressibleNavierStokes.diffusion(u, setup) + # φ = IncompressibleNavierStokes.convection(u, setup) + # dot(φ, φ) + dot(getindex.(φ, Iu), getindex.(φ, Iu)) + # sum(abs2, getindex.(φ, Iu)) +end + +solver = SpectralPressureSolver(setup) +function f(u, setup) + (; Ω, Iu) = setup.grid + φ = u + φ = IncompressibleNavierStokes.apply_bc_u(u, T(0), setup) + φ = IncompressibleNavierStokes.momentum(φ, T(0), setup) + φ = IncompressibleNavierStokes.apply_bc_u(φ, T(0), setup) + φ = IncompressibleNavierStokes.project(solver, φ, setup) + # dot(φ, φ) + dot(getindex.(φ, Iu), getindex.(φ, Iu)) + # sum(abs2, getindex.(φ, Iu)) +end + +pressure_solver = SpectralPressureSolver(setup) +# method = +function f(u, setup) + (; Ω, Iu) = setup.grid + method = RK44(; T) + stepper = IncompressibleNavierStokes.create_stepper( + method; + setup, + pressure_solver, + u, + p, + t = T(0), + ) + stepper = IncompressibleNavierStokes.timestep(method, stepper, T(1e-4)) + φ = stepper.u + # dot(φ, φ) + dot(getindex.(φ, Iu), getindex.(φ, Iu)) + # sum(abs2, getindex.(φ, Iu)) +end + +function f(u, setup) + (; Iu) = setup.grid + φ = IncompressibleNavierStokes.tupleadd(u, u) + dot(getindex.(φ, Iu), getindex.(φ, Iu)) +end + +IncompressibleNavierStokes.tupleadd(u, u) + +(g1 = gradient(u -> f(u, setup), u)[1]; KernelAbstractions.synchronize(get_backend(u[1])); g1[1][Iu[1]]) +(g1 = gradient(u -> f(u, setup), u)[1]; KernelAbstractions.synchronize(get_backend(u[1])); g1[1][Iu[1]]) +(g1 = gradient(u -> f(u, setup), u)[1]; KernelAbstractions.synchronize(get_backend(u[1])); g1[1][Iu[1]]) +(g1 = gradient(u -> f(u, setup), u)[1]; KernelAbstractions.synchronize(get_backend(u[1])); g1[1][Iu[1]]) +(g1 = gradient(u -> f(u, setup), u)[1]; KernelAbstractions.synchronize(get_backend(u[1])); g1[1][Iu[1]]) +(g1 = gradient(u -> f(u, setup), u)[1]; KernelAbstractions.synchronize(get_backend(u[1])); g1[1][Iu[1]]) +(g1 = gradient(u -> f(u, setup), u)[1]; KernelAbstractions.synchronize(get_backend(u[1])); g1[1][Iu[1]]) +(g1 = gradient(u -> f(u, setup), u)[1]; KernelAbstractions.synchronize(get_backend(u[1])); g1[1][Iu[1]]) +(g1 = gradient(u -> f(u, setup), u)[1]; KernelAbstractions.synchronize(get_backend(u[1])); g1[1][Iu[1]]) +(g1 = gradient(u -> f(u, setup), u)[1]; KernelAbstractions.synchronize(get_backend(u[1])); g1[1][Iu[1]]) +(g1 = gradient(u -> f(u, setup), u)[1]; KernelAbstractions.synchronize(get_backend(u[1])); g1[1][Iu[1]]) + +I = CartesianIndex(2, 3) +g1 = gradient(u -> f(u, setup), u)[1] +g2 = begin + u1 = copy.(u) + CUDA.@allowscalar u1[1][I] -= sqrt(eps(T)) / 2 + r1 = f(u1, setup) + u2 = copy.(u) + CUDA.@allowscalar u2[1][I] += sqrt(eps(T)) / 2 + r2 = f(u2, setup) + (r2 - r1) / sqrt(eps(T)) +end +CUDA.@allowscalar g1[1][I] +g2 + +function fp(p, setup) + (; Ip) = setup.grid + p = IncompressibleNavierStokes.apply_bc_p(p, T(0), setup) + φ = IncompressibleNavierStokes.pressuregradient(p, setup) + dot(φ, φ) + # dot(getindex.(φ, Iu), getindex.(φ, Iu)) + # sum(abs2, getindex.(φ, Iu)) +end + +I = CartesianIndex(2, 2) +g1 = gradient(p -> fp(p, setup), p)[1] +g2 = begin + p1 = copy(p) + CUDA.@allowscalar p1[I] -= sqrt(eps(T)) / 2 + r1 = fp(p1, setup) + p2 = copy(p) + CUDA.@allowscalar p2[I] += sqrt(eps(T)) / 2 + r2 = fp(p2, setup) + (r2 - r1) / sqrt(eps(T)) +end +CUDA.@allowscalar g1[I] +g2 + +φ[1] +φbar[1] +ur[1] +ubar[1] diff --git a/src/boundary_conditions.jl b/src/boundary_conditions.jl index 7ed7591ce..936db0002 100644 --- a/src/boundary_conditions.jl +++ b/src/boundary_conditions.jl @@ -124,7 +124,7 @@ ChainRulesCore.rrule(::typeof(apply_bc_p), p, t, setup) = ( # With respect to (apply_bc_p, p, t, setup) φbar -> ( NoTangent(), - apply_bc_p_pullback!(copy.(φbar), φbar, t, setup), + apply_bc_p_pullback!(copy(φbar), φbar, t, setup), NoTangent(), NoTangent(), ), @@ -180,8 +180,9 @@ function apply_bc_p!(p, t, setup; kwargs...) end function apply_bc_p_pullback!(pbar, φbar, t, setup; kwargs...) - (; boundary_conditions) = setup - D = length(u) + (; grid, boundary_conditions) = setup + (; dimension) = grid + D = dimension() for β = 1:D apply_bc_p_pullback!( boundary_conditions[β][1], @@ -280,21 +281,21 @@ function apply_bc_p_pullback!(::PeriodicBC, pbar, φbar, β, t, setup; atend, kw (; dimension, N) = grid D = dimension() δ = Offset{D}() - @kernel function _bc_a(p, ::Val{β}) where {β} + @kernel function adj_a!(p, φ, ::Val{β}) where {β} I = @index(Global, Cartesian) p[I+(N[β]-2)*δ(β)] += φ[I] end - @kernel function _bc_b(p, ::Val{β}) where {β} + @kernel function adj_b!(p, φ, ::Val{β}) where {β} I = @index(Global, Cartesian) - p[I+δ(β)] += p[I+(N[β]-1)*δ(β)] + p[I+δ(β)] += φ[I+(N[β]-1)*δ(β)] end ndrange = ntuple(γ -> γ == β ? 1 : N[γ], D) if atend - _bc_b(get_backend(p), workgroupsize)(pbar, φbar, Val(β); ndrange) + adj_b!(get_backend(pbar), workgroupsize)(pbar, φbar, Val(β); ndrange) else - _bc_a(get_backend(p), workgroupsize)(pbar, φbar, Val(β); ndrange) + adj_a!(get_backend(pbar), workgroupsize)(pbar, φbar, Val(β); ndrange) end - p + pbar end function apply_bc_u!(bc::DirichletBC, u, β, t, setup; atend, dudt = false, kwargs...) diff --git a/src/operators.jl b/src/operators.jl index 4fb042fda..e50f989c9 100644 --- a/src/operators.jl +++ b/src/operators.jl @@ -299,7 +299,8 @@ convection(u, setup) = convection!(zero.(u), u, setup) ChainRulesCore.rrule(::typeof(convection), u, setup) = ( convection(u, setup), - φ -> (NoTangent(), convection_adjoint!(similar.(u), (φ...,), u, setup), NoTangent()), + φ -> + (NoTangent(), convection_adjoint!(zero.(u), (φ...,), u, setup), NoTangent()), ) """ @@ -457,10 +458,8 @@ function bodyforce!(F, u, t, setup) end bodyforce(u, t, setup) = bodyforce!(zero.(u), u, t, setup) -ChainRulesCore.rrule(::typeof(bodyforce), u, t, setup) = ( - bodyforce(u, t, setup), - φ -> (NoTangent(), ZeroTangent(), NoTangent(), NoTangent()), -) +ChainRulesCore.rrule(::typeof(bodyforce), u, t, setup) = + (bodyforce(u, t, setup), φ -> (NoTangent(), ZeroTangent(), NoTangent(), NoTangent())) """ momentum!(F, u, t, setup) @@ -488,6 +487,14 @@ function momentum!(F, u, t, setup) F end +monitor(u) = (@info("Forward", typeof(u)); u) +ChainRulesCore.rrule(::typeof(monitor), u) = + (monitor(u), φ -> (@info("Reverse", typeof(φ)); (NoTangent(), φ))) + +tupleadd(u...) = ntuple(α -> sum(u -> u[α], u), length(u[1])) +ChainRulesCore.rrule(::typeof(tupleadd), u...) = + (tupleadd(u...), φ -> (NoTangent(), map(u -> φ, u)...)) + """ momentum(u, t, setup) @@ -501,16 +508,29 @@ function momentum(u, t, setup) d = diffusion(u, setup) c = convection(u, setup) f = bodyforce(u, t, setup) - F = ntuple(D) do α - d[α] .+ c[α] .+ f[α] - end + # F = ntuple(D) do α + # d[α] .+ c[α] .+ f[α] + # end + # F = @. d + c + f + F = tupleadd(d, c, f) if !isnothing(closure_model) m = closure_model(u) - F = F .+ m + # F = F .+ m + F = tupleadd(F, m) end F end +# ChainRulesCore.rrule(::typeof(momentum), u, t, setup) = ( +# (error(); momentum(u, t, setup)), +# φ -> ( +# NoTangent(), +# momentum_pullback!(zero.(φ), φ, u, t, setup), +# NoTangent(), +# NoTangent(), +# ), +# ) + """ pressuregradient!(G, p, setup) @@ -544,7 +564,7 @@ function pressuregradient_adjoint!(pbar, φ, setup) I = @index(Global, Cartesian) p[I] = zero(eltype(p)) for α = 1:D - I - δ(α) ∈ Iu[α] && (p[I] += φ[α][I-δ(α)] / Δu[α][I[α] - 1]) + I - δ(α) ∈ Iu[α] && (p[I] += φ[α][I-δ(α)] / Δu[α][I[α]-1]) I ∈ Iu[α] && (p[I] -= φ[α][I] / Δu[α][I[α]]) end end @@ -557,11 +577,8 @@ end Compute pressure gradient. """ -pressuregradient(p, setup) = pressuregradient!( - ntuple(α -> similar(p), setup.grid.dimension()), - p, - setup, -) +pressuregradient(p, setup) = + pressuregradient!(ntuple(α -> zero(p), setup.grid.dimension()), p, setup) ChainRulesCore.rrule(::typeof(pressuregradient), p, setup) = ( pressuregradient(p, setup), diff --git a/src/solvers/pressure/poisson.jl b/src/solvers/pressure/poisson.jl index 4f8c664c4..b2e692ea1 100644 --- a/src/solvers/pressure/poisson.jl +++ b/src/solvers/pressure/poisson.jl @@ -8,13 +8,13 @@ Non-mutating/allocating/out-of-place version. See also [`poisson!`](@ref). """ -function poisson end - poisson(solver, f) = poisson!(solver, zero(f), f) # Laplacian is auto-adjoint -ChainRulesCore.rrule(::typeof(poisson), f, setup) = - (poisson(p, setup), φ -> (NoTangent(), poisson(φ, setup), NoTangent())) +ChainRulesCore.rrule(::typeof(poisson), solver, f) = ( + poisson(solver, f), + φ -> (NoTangent(), NoTangent(), poisson(solver, φ)), +) """ poisson!(solver, p, f) diff --git a/src/solvers/pressure/project.jl b/src/solvers/pressure/project.jl index beeae8091..a51a29a64 100644 --- a/src/solvers/pressure/project.jl +++ b/src/solvers/pressure/project.jl @@ -3,18 +3,18 @@ function project(solver, u, setup) T = eltype(u[1]) # Divergence of tentative velocity field - M = divergence(v, setup) + M = divergence(u, setup) M = @. M * Ω # Solve the Poisson equation - p = poisson(pressure_solver, M) + p = poisson(solver, M) p = apply_bc_p(p, T(0), setup) # Compute pressure correction term G = pressuregradient(p, setup) # Update velocity, which is now divergence free - v .- G + u .- G end function project!(solver, u, setup; M, p, G) @@ -26,7 +26,7 @@ function project!(solver, u, setup; M, p, G) @. M *= Ω # Solve the Poisson equation - poisson!(pressure_solver, p, M) + poisson!(solver, p, M) apply_bc_p!(p, T(0), setup) # Compute pressure correction term diff --git a/src/time_steppers/step_explicit_runge_kutta.jl b/src/time_steppers/step_explicit_runge_kutta.jl index 18718bdbb..f2527bbd7 100644 --- a/src/time_steppers/step_explicit_runge_kutta.jl +++ b/src/time_steppers/step_explicit_runge_kutta.jl @@ -103,28 +103,28 @@ function timestep(method::ExplicitRungeKuttaMethod, stepper, Δt) for i = 1:nstage # Right-hand side for tᵢ₋₁ based on current velocity field uᵢ₋₁, vᵢ₋₁ at # level i-1. This includes force evaluation at tᵢ₋₁. + u = apply_bc_u(u, t, setup) F = momentum(u, t, setup) # Store right-hand side of stage i - ku = (ku..., F) + ku = (ku..., F) # Intermediate time step t = t₀ + c[i] * Δt # Update velocity current stage by sum of Fᵢ's until this stage, weighted # with Butcher tableau coefficients. This gives vᵢ - u = ntuple(D) do α - uα = u₀[α] - for j = 1:i - uα = @. uα + Δt * A[i, j] * ku[j][α] - end - uα + u = u₀ + for j = 1:i + # u = @. u + Δt * A[i, j] * ku[j] + u = tupleadd(u, @.(Δt * A[i, j] * ku[j])) end # Boundary conditions at tᵢ u = apply_bc_u(u, t, setup) + + # Make divergence free u = project(pressure_solver, u, setup) - u = apply_bc_u(u, t, setup) end # Complete time step From a8a91f4aeece5be3e72bbfa0db95f1dc8f62a761 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Mon, 18 Dec 2023 16:27:37 +0100 Subject: [PATCH 212/379] Enforce tuple --- scratch/testgrad.jl | 82 ++++++------------- src/boundary_conditions.jl | 2 +- src/operators.jl | 14 ++-- .../step_explicit_runge_kutta.jl | 4 +- 4 files changed, 37 insertions(+), 65 deletions(-) diff --git a/scratch/testgrad.jl b/scratch/testgrad.jl index 6dc3d08af..18f4d6017 100644 --- a/scratch/testgrad.jl +++ b/scratch/testgrad.jl @@ -72,6 +72,26 @@ using LinearAlgebra using Random (; Iu, Ip) = setup.grid +function finitediff(f, u::Tuple, I; h = sqrt(eps(eltype(u[1])))) + u1 = copy.(u) + CUDA.@allowscalar u1[1][I] -= h / 2 + r1 = f(u1) + u2 = copy.(u) + CUDA.@allowscalar u2[1][I] += h / 2 + r2 = f(u2) + (r2 - r1) / h +end + +function finitediff(f, p, I; h = sqrt(eps(eltype(p)))) + p1 = copy(p) + CUDA.@allowscalar p1[I] -= h / 2 + r1 = f(p1) + p2 = copy(p) + CUDA.@allowscalar p2[I] += h / 2 + r2 = f(p2) + (r2 - r1) / h +end + IncompressibleNavierStokes.diffusion(u, setup)[1] IncompressibleNavierStokes.diffusion_adjoint!(zero.(u), u, setup)[1] @@ -104,36 +124,6 @@ ubar = IncompressibleNavierStokes.convection_adjoint!(zero.(ur), φbar, ur, setu dot(φbar, φ) dot(ubar, ur) / 2 -I = CartesianIndex(14, 17) -g1 = gradient(p) do p - φ = IncompressibleNavierStokes.pressuregradient(p, setup) - dot(φ, φ) - # sum(φ[1]) -end -g2 = begin - p1 = copy(p) - CUDA.@allowscalar p1[I] -= sqrt(eps(T)) / 2 - φ = IncompressibleNavierStokes.pressuregradient(p1, setup) - r1 = dot(φ, φ) - p2 = copy(p) - CUDA.@allowscalar p2[I] += sqrt(eps(T)) / 2 - φ = IncompressibleNavierStokes.pressuregradient(p2, setup) - r2 = dot(φ, φ) - (r2 - r1) / sqrt(eps(T)) -end -CUDA.@allowscalar g1[1][I] -g2 - -g1 = gradient(u) do u - # φ = IncompressibleNavierStokes.convection(u, setup) - dot(φ, φ) - # sum(φ[1]) -end - -IncompressibleNavierStokes.momentum(u, T(0), setup) -IncompressibleNavierStokes.apply_bc_u(u, T(0), setup)[1] -u[1] - function f(u, setup) (; Iu) = setup.grid u = IncompressibleNavierStokes.apply_bc_u(u, T(0), setup) @@ -186,6 +176,8 @@ end IncompressibleNavierStokes.tupleadd(u, u) +f(u, setup) + (g1 = gradient(u -> f(u, setup), u)[1]; KernelAbstractions.synchronize(get_backend(u[1])); g1[1][Iu[1]]) (g1 = gradient(u -> f(u, setup), u)[1]; KernelAbstractions.synchronize(get_backend(u[1])); g1[1][Iu[1]]) (g1 = gradient(u -> f(u, setup), u)[1]; KernelAbstractions.synchronize(get_backend(u[1])); g1[1][Iu[1]]) @@ -198,19 +190,9 @@ IncompressibleNavierStokes.tupleadd(u, u) (g1 = gradient(u -> f(u, setup), u)[1]; KernelAbstractions.synchronize(get_backend(u[1])); g1[1][Iu[1]]) (g1 = gradient(u -> f(u, setup), u)[1]; KernelAbstractions.synchronize(get_backend(u[1])); g1[1][Iu[1]]) -I = CartesianIndex(2, 3) -g1 = gradient(u -> f(u, setup), u)[1] -g2 = begin - u1 = copy.(u) - CUDA.@allowscalar u1[1][I] -= sqrt(eps(T)) / 2 - r1 = f(u1, setup) - u2 = copy.(u) - CUDA.@allowscalar u2[1][I] += sqrt(eps(T)) / 2 - r2 = f(u2, setup) - (r2 - r1) / sqrt(eps(T)) -end -CUDA.@allowscalar g1[1][I] -g2 +I = CartesianIndex(2, 2) +CUDA.@allowscalar gradient(u -> f(u, setup), u)[1][1][I] +finitediff(u -> f(u, setup), u, I) function fp(p, setup) (; Ip) = setup.grid @@ -222,18 +204,8 @@ function fp(p, setup) end I = CartesianIndex(2, 2) -g1 = gradient(p -> fp(p, setup), p)[1] -g2 = begin - p1 = copy(p) - CUDA.@allowscalar p1[I] -= sqrt(eps(T)) / 2 - r1 = fp(p1, setup) - p2 = copy(p) - CUDA.@allowscalar p2[I] += sqrt(eps(T)) / 2 - r2 = fp(p2, setup) - (r2 - r1) / sqrt(eps(T)) -end -CUDA.@allowscalar g1[I] -g2 +CUDA.@allowscalar gradient(p -> fp(p, setup), p)[1][I] +finitediff(p -> fp(p, setup), p, I) φ[1] φbar[1] diff --git a/src/boundary_conditions.jl b/src/boundary_conditions.jl index 936db0002..3f1b55366 100644 --- a/src/boundary_conditions.jl +++ b/src/boundary_conditions.jl @@ -113,7 +113,7 @@ ChainRulesCore.rrule(::typeof(apply_bc_u), u, t, setup) = ( # With respect to (apply_bc_u, u, t, setup) φbar -> ( NoTangent(), - apply_bc_u_pullback!(copy.(φbar), φbar, t, setup), + apply_bc_u_pullback!(copy.((φbar...,)), (φbar...,), t, setup), NoTangent(), NoTangent(), ), diff --git a/src/operators.jl b/src/operators.jl index e50f989c9..dca5ee06d 100644 --- a/src/operators.jl +++ b/src/operators.jl @@ -491,9 +491,9 @@ monitor(u) = (@info("Forward", typeof(u)); u) ChainRulesCore.rrule(::typeof(monitor), u) = (monitor(u), φ -> (@info("Reverse", typeof(φ)); (NoTangent(), φ))) -tupleadd(u...) = ntuple(α -> sum(u -> u[α], u), length(u[1])) -ChainRulesCore.rrule(::typeof(tupleadd), u...) = - (tupleadd(u...), φ -> (NoTangent(), map(u -> φ, u)...)) +# tupleadd(u...) = ntuple(α -> sum(u -> u[α], u), length(u[1])) +# ChainRulesCore.rrule(::typeof(tupleadd), u...) = +# (tupleadd(u...), φ -> (NoTangent(), map(u -> φ, u)...)) """ momentum(u, t, setup) @@ -511,12 +511,12 @@ function momentum(u, t, setup) # F = ntuple(D) do α # d[α] .+ c[α] .+ f[α] # end - # F = @. d + c + f - F = tupleadd(d, c, f) + F = @. d + c + f + # F = tupleadd(d, c, f) if !isnothing(closure_model) m = closure_model(u) - # F = F .+ m - F = tupleadd(F, m) + F = F .+ m + # F = tupleadd(F, m) end F end diff --git a/src/time_steppers/step_explicit_runge_kutta.jl b/src/time_steppers/step_explicit_runge_kutta.jl index f2527bbd7..c40cbd7ad 100644 --- a/src/time_steppers/step_explicit_runge_kutta.jl +++ b/src/time_steppers/step_explicit_runge_kutta.jl @@ -116,8 +116,8 @@ function timestep(method::ExplicitRungeKuttaMethod, stepper, Δt) # with Butcher tableau coefficients. This gives vᵢ u = u₀ for j = 1:i - # u = @. u + Δt * A[i, j] * ku[j] - u = tupleadd(u, @.(Δt * A[i, j] * ku[j])) + u = @. u + Δt * A[i, j] * ku[j] + # u = tupleadd(u, @.(Δt * A[i, j] * ku[j])) end # Boundary conditions at tᵢ From 85ae27c471967c1ef32a44625386867c9518dd72 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Tue, 19 Dec 2023 10:55:22 +0100 Subject: [PATCH 213/379] Add trajectory loss --- scratch/train_model.jl | 56 ++++++++++++++----- src/closures/closure.jl | 36 ++++++++---- src/closures/create_les_data.jl | 1 - src/closures/training.jl | 37 +++++++++++- src/operators.jl | 13 +++-- src/solvers/solve_unsteady.jl | 5 +- .../step_explicit_runge_kutta.jl | 8 +-- 7 files changed, 119 insertions(+), 37 deletions(-) diff --git a/scratch/train_model.jl b/scratch/train_model.jl index 3bc0aa891..4655e050f 100644 --- a/scratch/train_model.jl +++ b/scratch/train_model.jl @@ -10,6 +10,7 @@ end #src # Here, we consider a periodic box ``[0, 1]^2``. It is discretized with a # uniform Cartesian grid with square cells. +using Adapt using GLMakie using IncompressibleNavierStokes using JLD2 @@ -40,22 +41,25 @@ device = identity using LuxCUDA using CUDA; T = Float32; +# T = Float64; ArrayType = CuArray; CUDA.allowscalar(false); -device = cu +# device = cu +device = x -> adapt(CuArray{T}, x) # Parameters # nles = 50 -nles = 128 +# nles = 64 +# nles = 128 # ndns = 200 -# nles = 256 +nles = 256 params = (; D = 2, Re = T(6_000), lims = (T(0), T(1)), nles = [nles], - ndns = 512, - # ndns = 1024, + # ndns = 512, + ndns = 1024, tburn = T(0.05), tsim = T(0.5), Δt = T(1e-4), @@ -109,8 +113,9 @@ fieldplot( o; setup, # fieldname = :velocity, - fieldname = 2, + # fieldname = 2, ) +# energy_spectrum_plot(o; setup) for i = 1:length(field) o[] = (; o[]..., u = device(field[i])) sleep(0.001) @@ -169,28 +174,53 @@ closure(sample, θ₀) |> size # θ = 2 * device(θ₀); opt = Optimisers.setup(Adam(T(1.0e-3)), θ); callbackstate = Point2f[]; +it = rand(1:size(io_valid[1].u, 4), 50); +validset = map(v -> v[:, :, :, it], io_valid[1]); + +# A-priori loss loss = createloss(mean_squared_error, closure); dataloader = createdataloader(io_train[1]; batchsize = 50, device); +dataloaders = [dataloader] dataloader() -loss(dataloader(), θ) -it = rand(1:size(io_valid[1].u, 4), 50); -validset = map(v -> v[:, :, :, it], io_valid[1]); + +# A-posteriori loss +loss = IncompressibleNavierStokes.create_trajectory_loss(; setup, pressure_solver, closure); +dataloaders = [ + IncompressibleNavierStokes.createtrajectoryloader(data_train; device, nunroll = 20) + for _ = 1:4 +]; +loss(dataloaders[1](), device(θ₀)) # Warm-up -loss(dataloader(), θ) -@time loss(dataloader(), θ); -b = dataloader() +loss(dataloaders[1](), θ) +@time loss(dataloaders[1](), θ); +b = dataloaders[1](); first(gradient(θ -> loss(b, θ), θ)); @time first(gradient(θ -> loss(b, θ), θ)); GC.gc() CUDA.reclaim() +map() do + i = 3 + # h = 1000 * sqrt(eps(T)) + h = cbrt(eps(T)) + θ1 = copy(θ) + θ2 = copy(θ) + CUDA.@allowscalar θ1[i] -= h / 2 + CUDA.@allowscalar θ2[i] += h / 2 + b = dataloaders[1]() + @show loss(b, θ2) loss(b, θ1) + a = (loss(b, θ2) - loss(b, θ1)) / h + b = CUDA.@allowscalar first(gradient(θ -> loss(b, θ), θ))[i] + [a; b] +end + # Training # Note: The states `opt`, `θ`, and `callbackstate` # will not be overwritten until training is finished. # This allows for cancelling with "Control-C" should errors explode. (; opt, θ, callbackstate) = train( - dataloader, + dataloaders, loss, opt, θ; diff --git a/src/closures/closure.jl b/src/closures/closure.jl index ce887e33e..3b4a09eea 100644 --- a/src/closures/closure.jl +++ b/src/closures/closure.jl @@ -1,19 +1,35 @@ """ - wrappedclosure(m, θ, setup) + wrappedclosure(m, setup) Wrap closure model and parameters so that it can be used in the solver. """ -function wrappedclosure(m, θ, setup) +function wrappedclosure(m, setup) (; dimension, Iu) = setup.grid D = dimension() - function neuralclosure(u) - u = stack(ntuple(α -> u[α][Iu[α]], D)) - u = reshape(u, size(u)..., 1) # One sample - mu = m(u, θ) - mu = pad_circular(mu, 1) - sz..., _ = size(mu) - i = ntuple(Returns(:), D) - mu = ntuple(α -> mu[i..., α, 1], D) + # function neuralclosure(u) + # u = stack(ntuple(α -> u[α][Iu[α]], D)) + # u = reshape(u, size(u)..., 1) # One sample + # mu = m(u, θ) + # mu = pad_circular(mu, 1) + # sz..., _ = size(mu) + # i = ntuple(Returns(:), D) + # mu = ntuple(α -> mu[i..., α, 1], D) + # end + function neuralclosure(u, θ) + if D == 2 + u = cat(u[1][Iu[1]], u[2][Iu[2]]; dims = 3) + u = reshape(u, size(u)..., 1) # One sample + mu = m(u, θ) + mu = pad_circular(mu, 1) + mu = (mu[:, :, 1, 1], mu[:, :, 2, 1]) + elseif D == 3 + u = cat(u[1][Iu[1]], u[2][Iu[2]], u[3][Iu[3]]; dims = 4) + u = reshape(u, size(u)..., 1) # One sample + mu = m(u, θ) + mu = pad_circular(mu, 1) + mu = (mu[:, :, :, 1, 1], mu[:, :, :, 2, 1], mu[:, :, :, 3, 1]) + end + mu end end diff --git a/src/closures/create_les_data.jl b/src/closures/create_les_data.jl index 35fcf2b2b..39993ccf7 100644 --- a/src/closures/create_les_data.jl +++ b/src/closures/create_les_data.jl @@ -107,7 +107,6 @@ filtersaver(dns, les, compression, pressure_solver; nupdate = 1) = ArrayType = Array, ) - Create filtered DNS data. """ function create_les_data( diff --git a/src/closures/training.jl b/src/closures/training.jl index b57cf084f..6c7f3f20c 100644 --- a/src/closures/training.jl +++ b/src/closures/training.jl @@ -15,6 +15,16 @@ createdataloader(data; batchsize = 50, device = identity) = function dataloader( xuse, yuse end +createtrajectoryloader(trajectories; nunroll = 10, device = identity) = + function dataloader() + (; u, t) = rand(trajectories) + nt = length(t) + @assert nt ≥ nunroll + istart = rand(1:nt-nunroll) + it = istart:istart+nunroll + (; u = device.(u[1][it]), t = t[it]) + end + """ train( dataloaders, @@ -107,6 +117,31 @@ relerr_trajectory(uref, setup; nupdate = 1) = e end +function create_trajectory_loss(; + setup, + method = RK44(; T = eltype(setup.grid.x[1])), + pressure_solver = DirectPressureSolver(setup), + closure, +) + closure_model = wrappedclosure(closure, setup) + setup = (; setup..., closure_model) + function trajectory_loss(traj, θ) + (; u, t) = traj + v = u[1] + stepper = + create_stepper(method; setup, pressure_solver, u = v, p = zero(v[1]), t = t[1]) + loss = zero(eltype(v[1])) + for it = 2:length(t) + Δt = t[it] - t[it-1] + stepper = timestep(method, stepper, Δt; θ) + for α = 1:length(u[1]) + loss += sum(abs2, stepper.u[α] - u[it][α]) / sum(abs2, u[it][α]) + end + end + loss / (length(t) - 1) + end +end + """ create_callback( f, @@ -136,7 +171,7 @@ function create_callback(f, x, y; state = Point2f[], display_each_iteration = fa @info "Iteration $i \trelative error: $e" state = push!(copy(state), Point2f(istart + i, e)) obs[] = state - autolimits!(fig.axis) + i < 10 || autolimits!(fig.axis) display_each_iteration && display(fig) state end diff --git a/src/operators.jl b/src/operators.jl index dca5ee06d..4f5958b2c 100644 --- a/src/operators.jl +++ b/src/operators.jl @@ -467,7 +467,7 @@ ChainRulesCore.rrule(::typeof(bodyforce), u, t, setup) = Right hand side of momentum equations, excluding pressure gradient. Put the result in ``F``. """ -function momentum!(F, u, t, setup) +function momentum!(F, u, t, setup; θ = nothing) (; grid, closure_model) = setup (; dimension) = grid D = dimension() @@ -479,9 +479,10 @@ function momentum!(F, u, t, setup) convectiondiffusion!(F, u, setup) bodyforce!(F, u, t, setup) if !isnothing(closure_model) - m = closure_model(u) + m = closure_model + mu = m(u, θ) for α = 1:D - F[α] .+= m[α] + F[α] .+= mu[α] end end F @@ -501,7 +502,7 @@ ChainRulesCore.rrule(::typeof(monitor), u) = Right hand side of momentum equations, excluding pressure gradient. """ # momentum(u, t, setup) = momentum!(zero.(u), u, t, setup) -function momentum(u, t, setup) +function momentum(u, t, setup; θ = nothing) (; grid, closure_model) = setup (; dimension) = grid D = dimension() @@ -514,8 +515,8 @@ function momentum(u, t, setup) F = @. d + c + f # F = tupleadd(d, c, f) if !isnothing(closure_model) - m = closure_model(u) - F = F .+ m + m = closure_model + F = F .+ m(u, θ) # F = tupleadd(F, m) end F diff --git a/src/solvers/solve_unsteady.jl b/src/solvers/solve_unsteady.jl index 601fb9941..d550ea494 100644 --- a/src/solvers/solve_unsteady.jl +++ b/src/solvers/solve_unsteady.jl @@ -41,6 +41,7 @@ function solve_unsteady( n_adapt_Δt = 1, docopy = true, processors = (;), + θ = nothing, ) if docopy u₀ = copy.(u₀) @@ -71,7 +72,7 @@ function solve_unsteady( Δt = min(Δt, t_end - stepper.t) # Perform a single time step with the time integration method - stepper = timestep!(method, stepper, Δt; cache) + stepper = timestep!(method, stepper, Δt; cache, θ) # Process iteration results with each processor state[] = get_state(stepper) @@ -81,7 +82,7 @@ function solve_unsteady( Δt = (t_end - t_start) / nstep for it = 1:nstep # Perform a single time step with the time integration method - stepper = timestep!(method, stepper, Δt; cache) + stepper = timestep!(method, stepper, Δt; cache, θ) # Process iteration results with each processor state[] = get_state(stepper) diff --git a/src/time_steppers/step_explicit_runge_kutta.jl b/src/time_steppers/step_explicit_runge_kutta.jl index c40cbd7ad..fa49bace2 100644 --- a/src/time_steppers/step_explicit_runge_kutta.jl +++ b/src/time_steppers/step_explicit_runge_kutta.jl @@ -1,7 +1,7 @@ create_stepper(::ExplicitRungeKuttaMethod; setup, pressure_solver, u, p, t, n = 0) = (; setup, pressure_solver, u, p, t, n) -function timestep!(method::ExplicitRungeKuttaMethod, stepper, Δt; cache) +function timestep!(method::ExplicitRungeKuttaMethod, stepper, Δt; cache, θ = nothing) (; setup, pressure_solver, u, p, t, n) = stepper (; grid) = setup (; dimension, Iu, Ip, Ω) = grid @@ -27,7 +27,7 @@ function timestep!(method::ExplicitRungeKuttaMethod, stepper, Δt; cache) for i = 1:nstage # Right-hand side for tᵢ₋₁ based on current velocity field uᵢ₋₁, vᵢ₋₁ at # level i-1. This includes force evaluation at tᵢ₋₁. - momentum!(F, u, t, setup) + momentum!(F, u, t, setup; θ) # Store right-hand side of stage i for α = 1:D @@ -78,7 +78,7 @@ function timestep!(method::ExplicitRungeKuttaMethod, stepper, Δt; cache) create_stepper(method; setup, pressure_solver, u, p, t, n = n + 1) end -function timestep(method::ExplicitRungeKuttaMethod, stepper, Δt) +function timestep(method::ExplicitRungeKuttaMethod, stepper, Δt; θ = nothing) (; setup, pressure_solver, u, p, t, n) = stepper (; grid) = setup (; dimension) = grid @@ -104,7 +104,7 @@ function timestep(method::ExplicitRungeKuttaMethod, stepper, Δt) # Right-hand side for tᵢ₋₁ based on current velocity field uᵢ₋₁, vᵢ₋₁ at # level i-1. This includes force evaluation at tᵢ₋₁. u = apply_bc_u(u, t, setup) - F = momentum(u, t, setup) + F = momentum(u, t, setup; θ) # Store right-hand side of stage i ku = (ku..., F) From 919a770a7a97046889b00a79bfbca6ab7b507204 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Wed, 20 Dec 2023 13:54:43 +0100 Subject: [PATCH 214/379] Rename solver --- examples/BackwardFacingStep2D.jl | 7 +-- examples/DecayingTurbulence2D.jl | 6 +- examples/DecayingTurbulence3D.jl | 6 +- examples/LidDrivenCavity2D.jl | 7 +-- examples/PlaneJets2D.jl | 11 ++-- examples/ShearLayer2D.jl | 6 +- examples/TaylorGreenVortex2D.jl | 8 +-- examples/TaylorGreenVortex3D.jl | 6 +- scratch/multigrid.jl | 18 +++--- scratch/testgrad.jl | 12 ---- src/boundary_conditions.jl | 4 +- src/closures/create_les_data.jl | 32 ++++------ src/closures/training.jl | 5 +- src/create_initial_conditions.jl | 16 ++--- src/operators.jl | 3 +- src/solvers/pressure/poisson.jl | 6 +- src/solvers/pressure/pressure.jl | 10 +-- src/solvers/pressure/project.jl | 2 +- src/solvers/solve_unsteady.jl | 6 +- src/time_steppers/step_ab_cn.jl | 39 +++++------- .../step_explicit_runge_kutta.jl | 18 +++--- .../step_implicit_runge_kutta.jl | 43 ++++--------- src/time_steppers/step_one_leg.jl | 63 +++++-------------- test/postprocess2D.jl | 7 +-- test/postprocess3D.jl | 7 +-- test/solvers.jl | 38 ++++------- 26 files changed, 138 insertions(+), 248 deletions(-) diff --git a/examples/BackwardFacingStep2D.jl b/examples/BackwardFacingStep2D.jl index 00638e6c8..d12d894e3 100644 --- a/examples/BackwardFacingStep2D.jl +++ b/examples/BackwardFacingStep2D.jl @@ -59,11 +59,10 @@ plotgrid(x, y) # Build setup and assemble operators setup = Setup(x, y; Re, boundary_conditions, ArrayType); -pressure_solver = DirectPressureSolver(setup); +psolver = DirectPressureSolver(setup); # Initial conditions (extend inflow) -u₀, p₀ = - create_initial_conditions(setup, (dim, x, y) -> U(dim, x, y, zero(x)); pressure_solver); +u₀, p₀ = create_initial_conditions(setup, (dim, x, y) -> U(dim, x, y, zero(x)); psolver); # Solve steady state problem ## u, p = solve_steady_state(setup, u₀, p₀); @@ -75,7 +74,7 @@ state, outputs = solve_unsteady( p₀, (T(0), T(7)); Δt = T(0.002), - pressure_solver, + psolver, processors = ( rtp = realtimeplotter(; setup, diff --git a/examples/DecayingTurbulence2D.jl b/examples/DecayingTurbulence2D.jl index 9a0e28d14..e4ab68572 100644 --- a/examples/DecayingTurbulence2D.jl +++ b/examples/DecayingTurbulence2D.jl @@ -43,10 +43,10 @@ setup = Setup(x...; Re, ArrayType); # Since the grid is uniform and identical for x and y, we may use a specialized # spectral pressure solver -pressure_solver = SpectralPressureSolver(setup); +psolver = SpectralPressureSolver(setup); # Create random initial conditions -u₀, p₀ = random_field(setup, T(0); pressure_solver); +u₀, p₀ = random_field(setup, T(0); psolver); # Solve unsteady problem state, outputs = solve_unsteady( @@ -55,7 +55,7 @@ state, outputs = solve_unsteady( p₀, (T(0), T(1)); Δt = T(1e-3), - pressure_solver, + psolver, processors = ( ## rtp = realtimeplotter(; setup, nupdate = 1), ehist = realtimeplotter(; diff --git a/examples/DecayingTurbulence3D.jl b/examples/DecayingTurbulence3D.jl index 80e158dde..8a13ddb14 100644 --- a/examples/DecayingTurbulence3D.jl +++ b/examples/DecayingTurbulence3D.jl @@ -45,10 +45,10 @@ setup = Setup(x, y, z; Re, ArrayType); # Since the grid is uniform and identical for x, y, and z, we may use a # specialized spectral pressure solver -pressure_solver = SpectralPressureSolver(setup); +psolver = SpectralPressureSolver(setup); # Initial conditions -u₀, p₀ = random_field(setup; pressure_solver); +u₀, p₀ = random_field(setup; psolver); # Solve unsteady problem (; u, p, t), outputs = solve_unsteady( @@ -57,7 +57,7 @@ u₀, p₀ = random_field(setup; pressure_solver); p₀, (T(0), T(1)); Δt = T(1e-3), - pressure_solver, + psolver, processors = ( ## rtp = realtimeplotter(; setup, plot = fieldplot, nupdate = 10), ehist = realtimeplotter(; diff --git a/examples/LidDrivenCavity2D.jl b/examples/LidDrivenCavity2D.jl index d9cb30deb..159ec441d 100644 --- a/examples/LidDrivenCavity2D.jl +++ b/examples/LidDrivenCavity2D.jl @@ -93,11 +93,11 @@ setup = Setup(x, y; boundary_conditions, Re, ArrayType); # - [`SpectralPressureSolver`](@ref) (only for periodic boundary conditions and # uniform grids) -pressure_solver = DirectPressureSolver(setup); +psolver = DirectPressureSolver(setup); # The initial conditions are provided in function. The value `dim()` determines # the velocity component. -u₀, p₀ = create_initial_conditions(setup, (dim, x, y) -> zero(x); pressure_solver); +u₀, p₀ = create_initial_conditions(setup, (dim, x, y) -> zero(x); psolver); # ## Solve problems # @@ -128,8 +128,7 @@ processors = ( # By default, a standard fourth order Runge-Kutta method is used. If we don't # provide the time step explicitly, an adaptive time step is used. tlims = (T(0), T(10)) -state, outputs = - solve_unsteady(setup, u₀, p₀, tlims; Δt = T(1e-3), pressure_solver, processors); +state, outputs = solve_unsteady(setup, u₀, p₀, tlims; Δt = T(1e-3), psolver, processors); # ## Post-process # diff --git a/examples/PlaneJets2D.jl b/examples/PlaneJets2D.jl index 7b3e72ebb..cae0dc89f 100644 --- a/examples/PlaneJets2D.jl +++ b/examples/PlaneJets2D.jl @@ -95,14 +95,11 @@ setup = Setup(x, y; Re, ArrayType); # Since the grid is uniform and identical for x and y, we may use a specialized # spectral pressure solver -pressure_solver = SpectralPressureSolver(setup) +psolver = SpectralPressureSolver(setup) # Initial conditions -u₀, p₀ = create_initial_conditions( - setup, - (dim, x, y) -> dim() == 1 ? U(x, y) : zero(x); - pressure_solver, -); +u₀, p₀ = + create_initial_conditions(setup, (dim, x, y) -> dim() == 1 ? U(x, y) : zero(x); psolver); # Real time plot: Streamwise average and spectrum function meanplot(state; setup) @@ -167,7 +164,7 @@ state, outputs = solve_unsteady( (T(0), T(1)); method = RK44P2(), Δt = 0.001, - pressure_solver, + psolver, processors = ( rtp = realtimeplotter(; setup, diff --git a/examples/ShearLayer2D.jl b/examples/ShearLayer2D.jl index 6a49a17db..452d2078b 100644 --- a/examples/ShearLayer2D.jl +++ b/examples/ShearLayer2D.jl @@ -45,7 +45,7 @@ plotgrid(x, y) # Build setup and assemble operators setup = Setup(x, y; Re, ArrayType); -pressure_solver = SpectralPressureSolver(setup) +psolver = SpectralPressureSolver(setup) # Initial conditions: We add 1 to u in order to make global momentum # conservation less trivial @@ -56,7 +56,7 @@ U1(y) = y ≤ π ? tanh((y - T(π / 2)) / d) : tanh((T(3π / 2) - y) / d) u₀, p₀ = create_initial_conditions( setup, (dim, x, y) -> dim() == 1 ? U1(y) : e * sin(x); - pressure_solver, + psolver, ); # Solve unsteady problem @@ -66,7 +66,7 @@ state, outputs = solve_unsteady( p₀, (T(0), T(8)); Δt = T(0.01), - pressure_solver, + psolver, processors = ( rtp = realtimeplotter(; setup, diff --git a/examples/TaylorGreenVortex2D.jl b/examples/TaylorGreenVortex2D.jl index 949f04f5f..0e63819d3 100644 --- a/examples/TaylorGreenVortex2D.jl +++ b/examples/TaylorGreenVortex2D.jl @@ -34,21 +34,21 @@ function compute_convergence(; D, nlist, lims, Re, tlims, Δt, uref, ArrayType = @info "Computing error for n = $n" x = ntuple(α -> LinRange(lims..., n + 1), D) setup = Setup(x...; Re, ArrayType) - pressure_solver = SpectralPressureSolver(setup) + psolver = SpectralPressureSolver(setup) u₀, p₀ = create_initial_conditions( setup, (dim, x...) -> uref(dim, x..., tlims[1]), tlims[1]; - pressure_solver, + psolver, ) ut, pt = create_initial_conditions( setup, (dim, x...) -> uref(dim, x..., tlims[2]), tlims[2]; - pressure_solver, + psolver, project = false, ) - (; u, p, t), outputs = solve_unsteady(setup, u₀, p₀, tlims; Δt, pressure_solver) + (; u, p, t), outputs = solve_unsteady(setup, u₀, p₀, tlims; Δt, psolver) (; Ip) = setup.grid a, b = T(0), T(0) for α = 1:D diff --git a/examples/TaylorGreenVortex3D.jl b/examples/TaylorGreenVortex3D.jl index d91cb49f6..d01ac0ad6 100644 --- a/examples/TaylorGreenVortex3D.jl +++ b/examples/TaylorGreenVortex3D.jl @@ -41,7 +41,7 @@ setup = Setup(x, y, z; Re, ArrayType); # Since the grid is uniform and identical for x, y, and z, we may use a # specialized spectral pressure solver -pressure_solver = SpectralPressureSolver(setup); +psolver = SpectralPressureSolver(setup); # Initial conditions u₀, p₀ = create_initial_conditions( @@ -49,7 +49,7 @@ u₀, p₀ = create_initial_conditions( (dim, x, y, z) -> dim() == 1 ? sinpi(2x) * cospi(2y) * sinpi(2z) / 2 : dim() == 2 ? -cospi(2x) * sinpi(2y) * sinpi(2z) / 2 : zero(x); - pressure_solver, + psolver, ); # Solve unsteady problem @@ -73,7 +73,7 @@ u₀, p₀ = create_initial_conditions( ## field = fieldsaver(; setup, nupdate = 10), log = timelogger(; nupdate = 100), ), - pressure_solver, + psolver, ); # ## Post-process diff --git a/scratch/multigrid.jl b/scratch/multigrid.jl index 3dad5c70c..3960bbe74 100644 --- a/scratch/multigrid.jl +++ b/scratch/multigrid.jl @@ -197,18 +197,18 @@ fig = with_theme(; palette = (; color = ["#3366cc", "#cc0000", "#669900", "#ffcc ksort = k[ib] jprev = 2 # Do not include constant mode for ki = 1:kmax - j = findfirst(>(ki+1), ksort) + j = findfirst(>(ki + 1), ksort) isnothing(j) && (j = length(k) + 1) ia = [ia; fill!(similar(ia, j - jprev), ki)] # val = doaverage ? T(1) / (j - jprev) : T(1) - val = T(π) * ((ki+1)^2 - ki^2) / (j - jprev) + val = T(π) * ((ki + 1)^2 - ki^2) / (j - jprev) vals = [vals; fill!(similar(vals, j - jprev), val)] jprev = j end ib = ib[2:jprev-1] A = sparse(ia, ib, vals, kmax, length(k)) # Build inertial slope above energy - krange = [T(kmax)^(T(2)/3), T(kmax)] + krange = [T(kmax)^(T(2) / 3), T(kmax)] # slope, slopelabel = D == 2 ? (-T(3), L"k^{-3}") : (-T(5 / 3), L"k^{-5/3}") # slope, slopelabel = D == 2 ? (-T(3), "||k||₂⁻³") : (-T(5 / 3), "k⁻⁵³") slope, slopelabel = D == 2 ? (-T(3), "|k|⁻³") : (-T(5 / 3), "|k|⁻⁵³") @@ -521,11 +521,11 @@ fig = with_theme(; palette = (; color = ["#3366cc", "#cc0000", "#669900", "#ffcc ksort = k[ib] jprev = 2 # Do not include constant mode for ki = 1:kmax - j = findfirst(>(ki+1), ksort) + j = findfirst(>(ki + 1), ksort) isnothing(j) && (j = length(k) + 1) ia = [ia; fill!(similar(ia, j - jprev), ki)] # val = doaverage ? T(1) / (j - jprev) : T(1) - val = T(π) * ((ki+1)^2 - ki^2) / (j - jprev) + val = T(π) * ((ki + 1)^2 - ki^2) / (j - jprev) # val = T(1) / (j - jprev) vals = [vals; fill!(similar(vals, j - jprev), val)] jprev = j @@ -534,7 +534,7 @@ fig = with_theme(; palette = (; color = ["#3366cc", "#cc0000", "#669900", "#ffcc A = sparse(ia, ib, vals, kmax, length(k)) # Build inertial slope above energy # krange = [cbrt(T(kmax)), T(kmax)] - krange = [T(kmax)^(T(2)/3), T(kmax)] + krange = [T(kmax)^(T(2) / 3), T(kmax)] # slope, slopelabel = D == 2 ? (-T(3), L"k^{-3}") : (-T(5 / 3), L"k^{-5/3}") slope, slopelabel = D == 2 ? (-T(3), "|k|⁻³") : (-T(5 / 3), "|k|⁻⁵³") slopeconst = T(0) @@ -603,11 +603,11 @@ fig = with_theme(; palette = (; color = ["#3366cc", "#cc0000", "#669900", "#ffcc ksort = k[ib] jprev = 2 # Do not include constant mode for ki = 1:kmax - j = findfirst(>(ki+1), ksort) + j = findfirst(>(ki + 1), ksort) isnothing(j) && (j = length(k) + 1) ia = [ia; fill!(similar(ia, j - jprev), ki)] # val = doaverage ? T(1) / (j - jprev) : T(1) - val = T(π) * ((ki+1)^2 - ki^2) / (j - jprev) + val = T(π) * ((ki + 1)^2 - ki^2) / (j - jprev) vals = [vals; fill!(similar(vals, j - jprev), val)] jprev = j end @@ -615,7 +615,7 @@ fig = with_theme(; palette = (; color = ["#3366cc", "#cc0000", "#669900", "#ffcc A = sparse(ia, ib, vals, kmax, length(k)) # Build inertial slope above energy # krange = [cbrt(T(kmax)), T(kmax)] - krange = [T(kmax)^(T(2)/3), T(kmax)] + krange = [T(kmax)^(T(2) / 3), T(kmax)] # slope, slopelabel = D == 2 ? (-T(3), L"k^{-3}") : (-T(5 / 3), L"k^{-5/3}") slope, slopelabel = D == 2 ? (-T(3), "|k|⁻³") : (-T(5 / 3), "|k|⁻⁵³") slopeconst = T(0) diff --git a/scratch/testgrad.jl b/scratch/testgrad.jl index 18f4d6017..c3821c29f 100644 --- a/scratch/testgrad.jl +++ b/scratch/testgrad.jl @@ -178,18 +178,6 @@ IncompressibleNavierStokes.tupleadd(u, u) f(u, setup) -(g1 = gradient(u -> f(u, setup), u)[1]; KernelAbstractions.synchronize(get_backend(u[1])); g1[1][Iu[1]]) -(g1 = gradient(u -> f(u, setup), u)[1]; KernelAbstractions.synchronize(get_backend(u[1])); g1[1][Iu[1]]) -(g1 = gradient(u -> f(u, setup), u)[1]; KernelAbstractions.synchronize(get_backend(u[1])); g1[1][Iu[1]]) -(g1 = gradient(u -> f(u, setup), u)[1]; KernelAbstractions.synchronize(get_backend(u[1])); g1[1][Iu[1]]) -(g1 = gradient(u -> f(u, setup), u)[1]; KernelAbstractions.synchronize(get_backend(u[1])); g1[1][Iu[1]]) -(g1 = gradient(u -> f(u, setup), u)[1]; KernelAbstractions.synchronize(get_backend(u[1])); g1[1][Iu[1]]) -(g1 = gradient(u -> f(u, setup), u)[1]; KernelAbstractions.synchronize(get_backend(u[1])); g1[1][Iu[1]]) -(g1 = gradient(u -> f(u, setup), u)[1]; KernelAbstractions.synchronize(get_backend(u[1])); g1[1][Iu[1]]) -(g1 = gradient(u -> f(u, setup), u)[1]; KernelAbstractions.synchronize(get_backend(u[1])); g1[1][Iu[1]]) -(g1 = gradient(u -> f(u, setup), u)[1]; KernelAbstractions.synchronize(get_backend(u[1])); g1[1][Iu[1]]) -(g1 = gradient(u -> f(u, setup), u)[1]; KernelAbstractions.synchronize(get_backend(u[1])); g1[1][Iu[1]]) - I = CartesianIndex(2, 2) CUDA.@allowscalar gradient(u -> f(u, setup), u)[1][1][I] finitediff(u -> f(u, setup), u, I) diff --git a/src/boundary_conditions.jl b/src/boundary_conditions.jl index 3f1b55366..568e3e624 100644 --- a/src/boundary_conditions.jl +++ b/src/boundary_conditions.jl @@ -152,7 +152,7 @@ function apply_bc_u_pullback!(ubar, φbar, t, setup; kwargs...) t, setup; atend = false, - kwargs... + kwargs..., ) apply_bc_u_pullback!( boundary_conditions[β][2], @@ -162,7 +162,7 @@ function apply_bc_u_pullback!(ubar, φbar, t, setup; kwargs...) t, setup; atend = true, - kwargs... + kwargs..., ) end ubar diff --git a/src/closures/create_les_data.jl b/src/closures/create_les_data.jl index 39993ccf7..195f2e31f 100644 --- a/src/closures/create_les_data.jl +++ b/src/closures/create_les_data.jl @@ -32,9 +32,9 @@ function gaussian_force( force end -function lesdatagen(dnsobs, les, compression, pressure_solver) +function lesdatagen(dnsobs, les, compression, psolver) Φu = zero.(face_average(dnsobs[].u, les, compression)) - q = zero(pressure(pressure_solver, Φu, dnsobs[].t, les)) + q = zero(pressure(psolver, Φu, dnsobs[].t, les)) M = zero(q) ΦF = zero.(Φu) FΦ = zero.(Φu) @@ -49,7 +49,7 @@ function lesdatagen(dnsobs, les, compression, pressure_solver) apply_bc_u!(FΦ, t, les; dudt = true) divergence!(M, FΦ, les) @. M *= les.grid.Ω - poisson!(pressure_solver, q, M) + poisson!(psolver, q, M) apply_bc_p!(q, t, les) pressuregradient!(GΦ, q, les) for α = 1:length(u) @@ -62,7 +62,7 @@ function lesdatagen(dnsobs, les, compression, pressure_solver) results end -filtersaver(dns, les, compression, pressure_solver; nupdate = 1) = +filtersaver(dns, les, compression, psolver; nupdate = 1) = processor() do state (; dimension, x) = dns.grid T = eltype(x[1]) @@ -70,10 +70,8 @@ filtersaver(dns, les, compression, pressure_solver; nupdate = 1) = F = zero.(state[].u) G = zero.(state[].u) dnsobs = Observable((; state[].u, F, state[].t)) - data = [ - lesdatagen(dnsobs, les[i], compression[i], pressure_solver[i]) for - i = 1:length(les) - ] + data = + [lesdatagen(dnsobs, les[i], compression[i], psolver[i]) for i = 1:length(les)] results = (; t = fill(zero(eltype(x[1])), 0), u = [d.u for d in data], @@ -135,8 +133,8 @@ function create_les_data( # Since the grid is uniform and identical for x and y, we may use a specialized # spectral pressure solver - pressure_solver = SpectralPressureSolver(dns) - pressure_solver_les = SpectralPressureSolver.(les) + psolver = SpectralPressureSolver(dns) + psolver_les = SpectralPressureSolver.(les) # Number of time steps to save nt = round(Int, tsim / Δt) @@ -148,7 +146,7 @@ function create_les_data( @info "Generating $datasize Mb of LES data" # Initial conditions - u₀, p₀ = random_field(dns, T(0); pressure_solver, ic_params...) + u₀, p₀ = random_field(dns, T(0); psolver, ic_params...) # Random body force # force_dns = @@ -166,7 +164,7 @@ function create_les_data( # _les = (; les..., bodyforce = force_les) # Solve burn-in DNS - (; u, p, t), outputs = solve_unsteady(_dns, u₀, p₀, (T(0), tburn); Δt, pressure_solver) + (; u, p, t), outputs = solve_unsteady(_dns, u₀, p₀, (T(0), tburn); Δt, psolver) # Solve DNS and store filtered quantities (; u, p, t), outputs = solve_unsteady( @@ -176,15 +174,9 @@ function create_les_data( (T(0), tsim); Δt, processors = (; - f = filtersaver( - _dns, - _les, - compression, - pressure_solver_les; - nupdate = savefreq, - ), + f = filtersaver(_dns, _les, compression, psolver_les; nupdate = savefreq), ), - pressure_solver, + psolver, ) # Store result for current IC diff --git a/src/closures/training.jl b/src/closures/training.jl index 6c7f3f20c..2fa75f620 100644 --- a/src/closures/training.jl +++ b/src/closures/training.jl @@ -120,7 +120,7 @@ relerr_trajectory(uref, setup; nupdate = 1) = function create_trajectory_loss(; setup, method = RK44(; T = eltype(setup.grid.x[1])), - pressure_solver = DirectPressureSolver(setup), + psolver = DirectPressureSolver(setup), closure, ) closure_model = wrappedclosure(closure, setup) @@ -128,8 +128,7 @@ function create_trajectory_loss(; function trajectory_loss(traj, θ) (; u, t) = traj v = u[1] - stepper = - create_stepper(method; setup, pressure_solver, u = v, p = zero(v[1]), t = t[1]) + stepper = create_stepper(method; setup, psolver, u = v, p = zero(v[1]), t = t[1]) loss = zero(eltype(v[1])) for it = 2:length(t) Δt = t[it] - t[it-1] diff --git a/src/create_initial_conditions.jl b/src/create_initial_conditions.jl index 9b5b9e08a..2b1104a0c 100644 --- a/src/create_initial_conditions.jl +++ b/src/create_initial_conditions.jl @@ -3,7 +3,7 @@ setup, initial_velocity, t = 0; - pressure_solver = DirectPressureSolver(setup), + psolver = DirectPressureSolver(setup), project = true, ) @@ -15,7 +15,7 @@ function create_initial_conditions( setup, initial_velocity, t = convert(eltype(setup.grid.x[1]), 0); - pressure_solver = DirectPressureSolver(setup), + psolver = DirectPressureSolver(setup), project = true, ) (; grid) = setup @@ -41,7 +41,7 @@ function create_initial_conditions( if project f = divergence(u, setup) @. f *= Ω - p = poisson(pressure_solver, f) + p = poisson(psolver, f) apply_bc_p!(p, t, setup) G = pressuregradient(p, setup) for α = 1:D @@ -50,7 +50,7 @@ function create_initial_conditions( end apply_bc_u!(u, t, setup) - p = pressure(pressure_solver, u, t, setup) + p = pressure(psolver, u, t, setup) apply_bc_p!(p, t, setup) # Initial conditions, including initial boundary condititions @@ -86,7 +86,7 @@ end A = 10, σ = 30, s = 5, - pressure_solver = DirectPressureSolver(setup), + psolver = DirectPressureSolver(setup), ) Create random field. @@ -102,7 +102,7 @@ function random_field( A = convert(eltype(setup.grid.x[1]), 10), σ = convert(eltype(setup.grid.x[1]), 30), s = convert(eltype(setup.grid.x[1]), 5), - pressure_solver = DirectPressureSolver(setup), + psolver = DirectPressureSolver(setup), ) (; dimension, x, Ip, Ω) = setup.grid D = dimension() @@ -116,7 +116,7 @@ function random_field( # Make velocity field divergence free M = divergence(u, setup) @. M *= Ω - p = poisson(pressure_solver, M) + p = poisson(psolver, M) apply_bc_p!(p, t, setup) G = pressuregradient(p, setup) for α = 1:D @@ -125,7 +125,7 @@ function random_field( apply_bc_u!(u, t, setup) # Compute pressure - p = pressure(pressure_solver, u, t, setup) + p = pressure(psolver, u, t, setup) apply_bc_p!(p, t, setup) u, p diff --git a/src/operators.jl b/src/operators.jl index 4f5958b2c..9310218c9 100644 --- a/src/operators.jl +++ b/src/operators.jl @@ -299,8 +299,7 @@ convection(u, setup) = convection!(zero.(u), u, setup) ChainRulesCore.rrule(::typeof(convection), u, setup) = ( convection(u, setup), - φ -> - (NoTangent(), convection_adjoint!(zero.(u), (φ...,), u, setup), NoTangent()), + φ -> (NoTangent(), convection_adjoint!(zero.(u), (φ...,), u, setup), NoTangent()), ) """ diff --git a/src/solvers/pressure/poisson.jl b/src/solvers/pressure/poisson.jl index b2e692ea1..c46371a66 100644 --- a/src/solvers/pressure/poisson.jl +++ b/src/solvers/pressure/poisson.jl @@ -11,10 +11,8 @@ See also [`poisson!`](@ref). poisson(solver, f) = poisson!(solver, zero(f), f) # Laplacian is auto-adjoint -ChainRulesCore.rrule(::typeof(poisson), solver, f) = ( - poisson(solver, f), - φ -> (NoTangent(), NoTangent(), poisson(solver, φ)), -) +ChainRulesCore.rrule(::typeof(poisson), solver, f) = + (poisson(solver, f), φ -> (NoTangent(), NoTangent(), poisson(solver, φ))) """ poisson!(solver, p, f) diff --git a/src/solvers/pressure/pressure.jl b/src/solvers/pressure/pressure.jl index 543b7f2ff..23c85b60e 100644 --- a/src/solvers/pressure/pressure.jl +++ b/src/solvers/pressure/pressure.jl @@ -1,10 +1,10 @@ """ - pressure!(solver, u, p, t, setup, F, f, M) + pressure!(psolver, u, p, t, setup, F, f, M) Compute pressure from velocity field. This makes the pressure compatible with the velocity field, resulting in same order pressure as velocity. """ -function pressure!(solver, u, p, t, setup, F, G, M) +function pressure!(psolver, u, p, t, setup, F, G, M) (; grid) = setup (; dimension, Iu, Ip, Ω) = grid D = dimension() @@ -12,13 +12,13 @@ function pressure!(solver, u, p, t, setup, F, G, M) apply_bc_u!(F, t, setup; dudt = true) divergence!(M, F, setup) @. M *= Ω - poisson!(solver, p, M) + poisson!(psolver, p, M) apply_bc_p!(p, t, setup) p end """ - pressure(solver, u, t, setup) + pressure(psolver, u, t, setup) Do additional pressure solve. This makes the pressure compatible with the velocity field, resulting in same order pressure as velocity. @@ -31,7 +31,7 @@ function pressure(solver, u, t, setup) F = apply_bc_u(F, t, setup; dudt = true) M = divergence(F, setup) M = @. M * Ω - p = poisson(solver, M) + p = poisson(psolver, M) p = apply_bc_p(p, t, setup) p end diff --git a/src/solvers/pressure/project.jl b/src/solvers/pressure/project.jl index a51a29a64..67e6f0755 100644 --- a/src/solvers/pressure/project.jl +++ b/src/solvers/pressure/project.jl @@ -4,7 +4,7 @@ function project(solver, u, setup) # Divergence of tentative velocity field M = divergence(u, setup) - M = @. M * Ω + M = @. M * Ω # Solve the Poisson equation p = poisson(solver, M) diff --git a/src/solvers/solve_unsteady.jl b/src/solvers/solve_unsteady.jl index d550ea494..24ce98666 100644 --- a/src/solvers/solve_unsteady.jl +++ b/src/solvers/solve_unsteady.jl @@ -5,7 +5,7 @@ p₀, tlims; method = RK44(; T = eltype(u₀[1])), - pressure_solver = DirectPressureSolver(setup), + psolver = DirectPressureSolver(setup), Δt = zero(eltype(u₀[1])), cfl = 1, n_adapt_Δt = 1, @@ -35,7 +35,7 @@ function solve_unsteady( p₀, tlims; method = RK44(; T = eltype(u₀[1])), - pressure_solver = DirectPressureSolver(setup), + psolver = DirectPressureSolver(setup), Δt = zero(eltype(u₀[1])), cfl = 1, n_adapt_Δt = 1, @@ -55,7 +55,7 @@ function solve_unsteady( cache = ode_method_cache(method, setup, u₀, p₀) # Time stepper - stepper = create_stepper(method; setup, pressure_solver, u = u₀, p = p₀, t = t_start) + stepper = create_stepper(method; setup, psolver, u = u₀, p = p₀, t = t_start) # Initialize processors for iteration results state = Observable(get_state(stepper)) diff --git a/src/time_steppers/step_ab_cn.jl b/src/time_steppers/step_ab_cn.jl index 9d1e4e605..c9df17c48 100644 --- a/src/time_steppers/step_ab_cn.jl +++ b/src/time_steppers/step_ab_cn.jl @@ -1,7 +1,7 @@ create_stepper( method::AdamsBashforthCrankNicolsonMethod; setup, - pressure_solver, + psolver, bc_vectors, V, p, @@ -14,10 +14,10 @@ create_stepper( cₙ = copy(V), tₙ = t, Diff_fact = spzeros(eltype(V), 0, 0), -) = (; setup, pressure_solver, bc_vectors, V, p, t, n, Vₙ, pₙ, cₙ, tₙ, Diff_fact) +) = (; setup, psolver, bc_vectors, V, p, t, n, Vₙ, pₙ, cₙ, tₙ, Diff_fact) function timestep(method::AdamsBashforthCrankNicolsonMethod, stepper, Δt) - (; setup, pressure_solver, bc_vectors, V, p, t, n, Vₙ, pₙ, cₙ, tₙ, Diff_fact) = stepper + (; setup, psolver, bc_vectors, V, p, t, n, Vₙ, pₙ, cₙ, tₙ, Diff_fact) = stepper (; convection_model, viscosity_model, Re, force, grid, operators, boundary_conditions) = setup (; bc_unsteady) = boundary_conditions @@ -32,7 +32,7 @@ function timestep(method::AdamsBashforthCrankNicolsonMethod, stepper, Δt) # the first iteration. Do one startup step instead if n == 0 stepper_startup = - create_stepper(method_startup; setup, pressure_solver, bc_vectors, V, p, t) + create_stepper(method_startup; setup, psolver, bc_vectors, V, p, t) n += 1 Vₙ = V pₙ = p @@ -51,7 +51,7 @@ function timestep(method::AdamsBashforthCrankNicolsonMethod, stepper, Δt) return create_stepper( method; setup, - pressure_solver, + psolver, bc_vectors, V, p, @@ -129,7 +129,7 @@ function timestep(method::AdamsBashforthCrankNicolsonMethod, stepper, Δt) f = (M * V + yM) / Δt - M * y_Δp # Solve the Poisson equation for the pressure - Δp = poisson(pressure_solver, f) + Δp = poisson(psolver, f) # Update velocity field V -= Δt ./ Ω .* (G * Δp .+ y_Δp) @@ -138,7 +138,7 @@ function timestep(method::AdamsBashforthCrankNicolsonMethod, stepper, Δt) p = pₙ .+ Δp if p_add_solve - p = pressure(pressure_solver, V, p, tₙ + Δt, setup; bc_vectors) + p = pressure(psolver, V, p, tₙ + Δt, setup; bc_vectors) end t = tₙ + Δtₙ @@ -146,7 +146,7 @@ function timestep(method::AdamsBashforthCrankNicolsonMethod, stepper, Δt) create_stepper( method; setup, - pressure_solver, + psolver, bc_vectors, V, p, @@ -167,7 +167,7 @@ function timestep!( cache, momentum_cache, ) - (; setup, pressure_solver, bc_vectors, V, p, t, n, Vₙ, pₙ, cₙ, tₙ, Diff_fact) = stepper + (; setup, psolver, bc_vectors, V, p, t, n, Vₙ, pₙ, cₙ, tₙ, Diff_fact) = stepper (; convection_model, viscosity_model, Re, force, grid, operators, boundary_conditions) = setup (; bc_unsteady) = boundary_conditions @@ -184,7 +184,7 @@ function timestep!( # the first iteration. Do one startup step instead if n == 0 stepper_startup = - create_stepper(method_startup; setup, pressure_solver, bc_vectors, V, p, t) + create_stepper(method_startup; setup, psolver, bc_vectors, V, p, t) n += 1 Vₙ = V pₙ = p @@ -202,7 +202,7 @@ function timestep!( return create_stepper( method; setup, - pressure_solver, + psolver, bc_vectors, V, p, @@ -283,7 +283,7 @@ function timestep!( f = (M * V + yM) / Δt - M * y_Δp # Solve the Poisson equation for the pressure - poisson!(pressure_solver, Δp, f) + poisson!(psolver, Δp, f) # Update velocity field V .-= Δt ./ Ω .* (G * Δp .+ y_Δp) @@ -292,18 +292,7 @@ function timestep!( p .= pₙ .+ Δp if p_add_solve - pressure!( - pressure_solver, - V, - p, - tₙ + Δt, - setup, - momentum_cache, - F, - f, - Δp; - bc_vectors, - ) + pressure!(psolver, V, p, tₙ + Δt, setup, momentum_cache, F, f, Δp; bc_vectors) end t = tₙ + Δtₙ @@ -311,7 +300,7 @@ function timestep!( create_stepper( method; setup, - pressure_solver, + psolver, bc_vectors, V, p, diff --git a/src/time_steppers/step_explicit_runge_kutta.jl b/src/time_steppers/step_explicit_runge_kutta.jl index fa49bace2..1209d63f2 100644 --- a/src/time_steppers/step_explicit_runge_kutta.jl +++ b/src/time_steppers/step_explicit_runge_kutta.jl @@ -1,8 +1,8 @@ -create_stepper(::ExplicitRungeKuttaMethod; setup, pressure_solver, u, p, t, n = 0) = - (; setup, pressure_solver, u, p, t, n) +create_stepper(::ExplicitRungeKuttaMethod; setup, psolver, u, p, t, n = 0) = + (; setup, psolver, u, p, t, n) function timestep!(method::ExplicitRungeKuttaMethod, stepper, Δt; cache, θ = nothing) - (; setup, pressure_solver, u, p, t, n) = stepper + (; setup, psolver, u, p, t, n) = stepper (; grid) = setup (; dimension, Iu, Ip, Ω) = grid (; A, b, c, p_add_solve) = method @@ -55,7 +55,7 @@ function timestep!(method::ExplicitRungeKuttaMethod, stepper, Δt; cache, θ = n @. M *= Ω / (c[i] * Δt) # Solve the Poisson equation - poisson!(pressure_solver, p, M) + poisson!(psolver, p, M) apply_bc_p!(p, t, setup) # Compute pressure correction term @@ -73,13 +73,13 @@ function timestep!(method::ExplicitRungeKuttaMethod, stepper, Δt; cache, θ = n t = t₀ + Δt # Do additional pressure solve to avoid first order pressure - p_add_solve && pressure!(pressure_solver, u, p, t, setup, F, G, M) + p_add_solve && pressure!(psolver, u, p, t, setup, F, G, M) - create_stepper(method; setup, pressure_solver, u, p, t, n = n + 1) + create_stepper(method; setup, psolver, u, p, t, n = n + 1) end function timestep(method::ExplicitRungeKuttaMethod, stepper, Δt; θ = nothing) - (; setup, pressure_solver, u, p, t, n) = stepper + (; setup, psolver, u, p, t, n) = stepper (; grid) = setup (; dimension) = grid (; A, b, c) = method @@ -124,11 +124,11 @@ function timestep(method::ExplicitRungeKuttaMethod, stepper, Δt; θ = nothing) u = apply_bc_u(u, t, setup) # Make divergence free - u = project(pressure_solver, u, setup) + u = project(psolver, u, setup) end # Complete time step t = t₀ + Δt - create_stepper(method; setup, pressure_solver, u, p, t, n = n + 1) + create_stepper(method; setup, psolver, u, p, t, n = n + 1) end diff --git a/src/time_steppers/step_implicit_runge_kutta.jl b/src/time_steppers/step_implicit_runge_kutta.jl index 5f12f3a14..ec08d148d 100644 --- a/src/time_steppers/step_implicit_runge_kutta.jl +++ b/src/time_steppers/step_implicit_runge_kutta.jl @@ -1,19 +1,11 @@ -create_stepper( - ::ImplicitRungeKuttaMethod; - setup, - pressure_solver, - bc_vectors, - V, - p, - t, - n = 0, -) = (; setup, pressure_solver, bc_vectors, V, p, t, n) +create_stepper(::ImplicitRungeKuttaMethod; setup, psolver, bc_vectors, V, p, t, n = 0) = + (; setup, psolver, bc_vectors, V, p, t, n) function timestep(method::ImplicitRungeKuttaMethod, stepper, Δt) # TODO: Implement out-of-place IRK error() - (; setup, pressure_solver, bc_vectors, V, p, t, n) = stepper + (; setup, psolver, bc_vectors, V, p, t, n) = stepper (; grid, operators, boundary_conditions) = setup (; bc_unsteady) = boundary_conditions (; NV, Np, Ω) = grid @@ -142,13 +134,13 @@ function timestep(method::ImplicitRungeKuttaMethod, stepper, Δt) (; yM) = bc_vectors f = 1 / Δtₙ .* (M * V .+ yM) - p = poisson!(pressure_solver, f) + p = poisson!(psolver, f) mul!(Gp, G, p) V = @. V - Δtₙ / Ω * Gp if p_add_solve - p = pressure(pressure_solver, V, p, tₙ + Δtₙ, setup; bc_vectors) + p = pressure(psolver, V, p, tₙ + Δtₙ, setup; bc_vectors) else # Standard method; take last pressure p = pⱼ[(end-Np+1):end] @@ -156,7 +148,7 @@ function timestep(method::ImplicitRungeKuttaMethod, stepper, Δt) else # For steady BC we do an additional pressure solve # That saves a pressure solve for iter = 1 in the next time step - # pressure!(pressure_solver, V, p, tₙ + Δtₙ, setup, momentum_cache, F, f, Δp) + # pressure!(psolver, V, p, tₙ + Δtₙ, setup, momentum_cache, F, f, Δp) # Standard method; take pressure of last stage p = pⱼ[(end-Np+1):end] @@ -164,11 +156,11 @@ function timestep(method::ImplicitRungeKuttaMethod, stepper, Δt) t = tₙ + Δtₙ - create_stepper(method; setup, pressure_solver, bc_vectors, V, p, t, n) + create_stepper(method; setup, psolver, bc_vectors, V, p, t, n) end function timestep!(method::ImplicitRungeKuttaMethod, stepper, Δt; cache, momentum_cache) - (; setup, pressure_solver, bc_vectors, V, p, t, n) = stepper + (; setup, psolver, bc_vectors, V, p, t, n) = stepper (; grid, operators, boundary_conditions) = setup (; bc_unsteady) = boundary_conditions (; NV, Np, Ω) = grid @@ -365,24 +357,13 @@ function timestep!(method::ImplicitRungeKuttaMethod, stepper, Δt; cache, moment f .= yM mul!(f, M, V, 1 / Δtₙ, 1 / Δtₙ) # f .= 1 / Δtₙ .* (M * V .+ yM) - poisson!(pressure_solver, p, f) + poisson!(psolver, p, f) mul!(Gp, G, p) @. V -= Δtₙ / Ω * Gp if p_add_solve - pressure!( - pressure_solver, - V, - p, - tₙ + Δtₙ, - setup, - momentum_cache, - F, - f, - Δp; - bc_vectors, - ) + pressure!(psolver, V, p, tₙ + Δtₙ, setup, momentum_cache, F, f, Δp; bc_vectors) else # Standard method; take last pressure p .= pⱼ[(end-Np+1):end] @@ -390,7 +371,7 @@ function timestep!(method::ImplicitRungeKuttaMethod, stepper, Δt; cache, moment else # For steady BC we do an additional pressure solve # That saves a pressure solve for iter = 1 in the next time step - # pressure!(pressure_solver, V, p, tₙ + Δtₙ, setup, momentum_cache, F, f, Δp) + # pressure!(psolver, V, p, tₙ + Δtₙ, setup, momentum_cache, F, f, Δp) # Standard method; take pressure of last stage p .= pⱼ[(end-Np+1):end] @@ -398,7 +379,7 @@ function timestep!(method::ImplicitRungeKuttaMethod, stepper, Δt; cache, moment t = tₙ + Δtₙ - create_stepper(method; setup, pressure_solver, bc_vectors, V, p, t, n) + create_stepper(method; setup, psolver, bc_vectors, V, p, t, n) end # Call momentum for multiple `(Vⱼ, pⱼ)` pairs, as required in implicit RK methods. diff --git a/src/time_steppers/step_one_leg.jl b/src/time_steppers/step_one_leg.jl index 81c313b76..77eef22c0 100644 --- a/src/time_steppers/step_one_leg.jl +++ b/src/time_steppers/step_one_leg.jl @@ -1,7 +1,7 @@ create_stepper( method::OneLegMethod; setup, - pressure_solver, + psolver, bc_vectors, V, p, @@ -12,10 +12,10 @@ create_stepper( Vₙ = copy(V), pₙ = copy(p), tₙ = t, -) = (; setup, pressure_solver, bc_vectors, V, p, t, n, Vₙ, pₙ, tₙ) +) = (; setup, psolver, bc_vectors, V, p, t, n, Vₙ, pₙ, tₙ) function timestep(method::OneLegMethod, stepper, Δt) - (; setup, pressure_solver, bc_vectors, V, p, t, n, Vₙ, pₙ, tₙ) = stepper + (; setup, psolver, bc_vectors, V, p, t, n, Vₙ, pₙ, tₙ) = stepper (; p_add_solve, β, method_startup) = method (; grid, operators, boundary_conditions) = setup (; bc_unsteady) = boundary_conditions @@ -26,25 +26,13 @@ function timestep(method::OneLegMethod, stepper, Δt) # the first iteration. Do one startup step instead if n == 0 stepper_startup = - create_stepper(method_startup; setup, pressure_solver, bc_vectors, V, p, t) + create_stepper(method_startup; setup, psolver, bc_vectors, V, p, t) n += 1 Vₙ = V pₙ = p tₙ = t (; V, p, t) = timestep(method_startup, stepper_startup, Δt) - return create_stepper( - method; - setup, - pressure_solver, - bc_vectors, - V, - p, - t, - n, - Vₙ, - pₙ, - tₙ, - ) + return create_stepper(method; setup, psolver, bc_vectors, V, p, t, n, Vₙ, pₙ, tₙ) end # Update current solution @@ -86,7 +74,7 @@ function timestep(method::OneLegMethod, stepper, Δt) f = (M * V + yM) / Δtᵦ # Solve the Poisson equation for the pressure - Δp = poisson(pressure_solver, f) + Δp = poisson(psolver, f) GΔp = G * Δp # Update velocity field @@ -97,16 +85,16 @@ function timestep(method::OneLegMethod, stepper, Δt) # Alternatively, do an additional Poisson solve if p_add_solve - p = pressure(pressure_solver, V, p, tₙ + Δtₙ, setup; bc_vectors) + p = pressure(psolver, V, p, tₙ + Δtₙ, setup; bc_vectors) end t = tₙ + Δtₙ - create_stepper(method; setup, pressure_solver, bc_vectors, V, p, t, n, Vₙ, pₙ, tₙ) + create_stepper(method; setup, psolver, bc_vectors, V, p, t, n, Vₙ, pₙ, tₙ) end function timestep!(method::OneLegMethod, stepper, Δt; cache, momentum_cache) - (; setup, pressure_solver, bc_vectors, n, V, p, t, Vₙ, pₙ, tₙ) = stepper + (; setup, psolver, bc_vectors, n, V, p, t, Vₙ, pₙ, tₙ) = stepper (; p_add_solve, β, method_startup) = method (; grid, operators, boundary_conditions) = setup (; bc_unsteady) = boundary_conditions @@ -118,7 +106,7 @@ function timestep!(method::OneLegMethod, stepper, Δt; cache, momentum_cache) # the first iteration. Do one startup step instead if n == 0 stepper_startup = - create_stepper(method_startup; setup, pressure_solver, bc_vectors, V, p, t) + create_stepper(method_startup; setup, psolver, bc_vectors, V, p, t) n += 1 Vₙ = V pₙ = p @@ -126,19 +114,7 @@ function timestep!(method::OneLegMethod, stepper, Δt; cache, momentum_cache) # Note: We do one out-of-place step here, with a few allocations (; V, p, t) = timestep(method_startup, stepper_startup, Δt) - return create_stepper( - method; - setup, - pressure_solver, - bc_vectors, - V, - p, - t, - n, - Vₙ, - pₙ, - tₙ, - ) + return create_stepper(method; setup, psolver, bc_vectors, V, p, t, n, Vₙ, pₙ, tₙ) end # Update current solution @@ -182,7 +158,7 @@ function timestep!(method::OneLegMethod, stepper, Δt; cache, momentum_cache) # f .= (M * V + yM) / Δtᵦ # Solve the Poisson equation for the pressure - poisson!(pressure_solver, Δp, f) + poisson!(psolver, Δp, f) mul!(GΔp, G, Δp) # Update velocity field @@ -193,21 +169,10 @@ function timestep!(method::OneLegMethod, stepper, Δt; cache, momentum_cache) # Alternatively, do an additional Poisson solve if p_add_solve - pressure!( - pressure_solver, - V, - p, - tₙ + Δtₙ, - setup, - momentum_cache, - F, - f, - Δp; - bc_vectors, - ) + pressure!(psolver, V, p, tₙ + Δtₙ, setup, momentum_cache, F, f, Δp; bc_vectors) end t = tₙ + Δtₙ - create_stepper(method; setup, pressure_solver, bc_vectors, V, p, t, n, Vₙ, pₙ, tₙ) + create_stepper(method; setup, psolver, bc_vectors, V, p, t, n, Vₙ, pₙ, tₙ) end diff --git a/test/postprocess2D.jl b/test/postprocess2D.jl index 10a2e8eed..153eb756f 100644 --- a/test/postprocess2D.jl +++ b/test/postprocess2D.jl @@ -9,7 +9,7 @@ @test plotgrid(x, y) isa Makie.FigureAxisPlot - pressure_solver = SpectralPressureSolver(setup) + psolver = SpectralPressureSolver(setup) t_start, t_end = tlims = (0.0, 1.0) @@ -23,7 +23,7 @@ initial_velocity_v, t_start; initial_pressure, - pressure_solver, + psolver, ) # Iteration processors @@ -40,8 +40,7 @@ ) # Solve unsteady problem - state, outputs = - solve_unsteady(setup, V₀, p₀, tlims; Δt = 0.01, processors, pressure_solver) + state, outputs = solve_unsteady(setup, V₀, p₀, tlims; Δt = 0.01, processors, psolver) @testset "VTK files" begin @info "Testing 2D processors: VTK files" diff --git a/test/postprocess3D.jl b/test/postprocess3D.jl index f24b87ce0..c27ac1b4e 100644 --- a/test/postprocess3D.jl +++ b/test/postprocess3D.jl @@ -12,7 +12,7 @@ @test plotgrid(x, y, z) isa Makie.Figure - pressure_solver = SpectralPressureSolver(setup) + psolver = SpectralPressureSolver(setup) t_start, t_end = tlims = (T(0), T(1)) @@ -27,7 +27,7 @@ initial_velocity_w, t_start; initial_pressure, - pressure_solver, + psolver, ) # Iteration processors @@ -44,8 +44,7 @@ ) # Solve unsteady problem - state, outputs = - solve_unsteady(setup, u₀, p₀, tlims; Δt = T(0.01), processors, pressure_solver) + state, outputs = solve_unsteady(setup, u₀, p₀, tlims; Δt = T(0.01), processors, psolver) @testset "VTK files" begin @info "Testing 3D processors: VTK files" diff --git a/test/solvers.jl b/test/solvers.jl index 4bc6ebe29..74b7a2258 100644 --- a/test/solvers.jl +++ b/test/solvers.jl @@ -7,7 +7,7 @@ y = LinRange(0, 2π, n + 1) setup = Setup(x, y; Re) - pressure_solver = SpectralPressureSolver(setup) + psolver = SpectralPressureSolver(setup) t_start, t_end = tlims = (0.0, 5.0) @@ -20,7 +20,7 @@ initial_velocity_v, t_start; initial_pressure, - pressure_solver, + psolver, ) @testset "Steady state" begin @@ -44,26 +44,12 @@ @testset "Unsteady solvers" begin @testset "Explicit Runge Kutta" begin @info "Testing explicit Runge-Kutta, out-of-place version" - state, outputs = solve_unsteady( - setup, - V₀, - p₀, - tlims; - Δt = 0.01, - pressure_solver, - inplace = false, - ) + state, outputs = + solve_unsteady(setup, V₀, p₀, tlims; Δt = 0.01, psolver, inplace = false) @test norm(state.u - u_exact) / norm(u_exact) < 1e-4 @info "Testing explicit Runge-Kutta, in-place version" - stateip, outputsip = solve_unsteady( - setup, - V₀, - p₀, - tlims; - Δt = 0.01, - pressure_solver, - inplace = true, - ) + stateip, outputsip = + solve_unsteady(setup, V₀, p₀, tlims; Δt = 0.01, psolver, inplace = true) @test stateip.u ≈ state.u @test stateip.p ≈ state.p end @@ -77,7 +63,7 @@ tlims; method = RIA2(), Δt = 0.01, - pressure_solver, + psolver, inplace = false, ) isa Tuple @info "Testing implicit Runge-Kutta, in-place version" @@ -88,7 +74,7 @@ tlims; method = RIA2(), Δt = 0.01, - pressure_solver, + psolver, inplace = true, processors = (timelogger(),), ) @@ -104,7 +90,7 @@ tlims; method = OneLegMethod(T), Δt = 0.01, - pressure_solver, + psolver, inplace = false, ) @test norm(state.u - u_exact) / norm(u_exact) < 1e-4 @@ -116,7 +102,7 @@ tlims; method = OneLegMethod(T), Δt = 0.01, - pressure_solver, + psolver, inplace = true, ) @test stateip.u ≈ state.u @@ -132,7 +118,7 @@ tlims; method = AdamsBashforthCrankNicolsonMethod(T), Δt = 0.01, - pressure_solver, + psolver, inplace = false, ) @test norm(state.u - u_exact) / norm(u_exact) < 1e-4 @@ -144,7 +130,7 @@ tlims; method = AdamsBashforthCrankNicolsonMethod(T), Δt = 0.01, - pressure_solver, + psolver, inplace = true, ) @test stateip.u ≈ state.u From f3e235a8e77b59ff7229c42a4fec7d9294207e40 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Wed, 20 Dec 2023 16:10:47 +0100 Subject: [PATCH 215/379] Hide pressure --- README.md | 4 +- examples/Actuator2D.jl | 5 +- examples/Actuator3D.jl | 5 +- examples/BackwardFacingStep2D.jl | 3 +- examples/BackwardFacingStep3D.jl | 3 +- examples/DecayingTurbulence2D.jl | 3 +- examples/DecayingTurbulence3D.jl | 5 +- examples/LidDrivenCavity2D.jl | 4 +- examples/LidDrivenCavity3D.jl | 5 +- examples/PlanarMixing2D.jl | 3 +- examples/PlaneJets2D.jl | 3 +- examples/ShearLayer2D.jl | 3 +- examples/TaylorGreenVortex2D.jl | 8 +- examples/TaylorGreenVortex3D.jl | 5 +- src/closures/create_les_data.jl | 46 +++++----- src/closures/training.jl | 2 +- src/create_initial_conditions.jl | 39 ++------- src/processors/processors.jl | 32 ++++--- src/processors/real_time_plot.jl | 51 +++++++---- src/solvers/pressure/pressure.jl | 20 ++--- src/solvers/pressure/project.jl | 34 +++----- src/solvers/solve_unsteady.jl | 27 +++--- .../step_explicit_runge_kutta.jl | 84 ++++--------------- src/time_steppers/time_stepper_caches.jl | 9 +- test/models.jl | 4 +- test/postprocess2D.jl | 4 +- test/postprocess3D.jl | 4 +- test/simulation2D.jl | 5 +- test/simulation3D.jl | 4 +- test/solvers.jl | 14 +--- 30 files changed, 177 insertions(+), 261 deletions(-) diff --git a/README.md b/README.md index 8a8a99fe3..dd55ea26b 100644 --- a/README.md +++ b/README.md @@ -93,11 +93,11 @@ bodyforce(dim, x, y, t) = dim() == 1 ? -cₜ * inside(x, y) : 0.0 setup = Setup(x, y; Re = 100.0, boundary_conditions, bodyforce); # Initial conditions (extend inflow) -u₀, p₀ = create_initial_conditions(setup, (dim, x, y) -> dim() == 1 ? 1.0 : 0.0); +u₀ = create_initial_conditions(setup, (dim, x, y) -> dim() == 1 ? 1.0 : 0.0); # Solve unsteady Navier-Stokes equations solve_unsteady( - setup, u₀, p₀, (0.0, 12.0); + setup, u₀, (0.0, 12.0); Δt = 0.05, processors = ( anim = animator(; setup, path ="vorticity.mp4", nupdate = 4), diff --git a/examples/Actuator2D.jl b/examples/Actuator2D.jl index f67b8570f..b0a722efa 100644 --- a/examples/Actuator2D.jl +++ b/examples/Actuator2D.jl @@ -62,14 +62,13 @@ bodyforce(dim, x, y, t) = dim() == 1 ? -cₜ * inside(x, y) : 0.0 setup = Setup(x, y; Re = 100.0, boundary_conditions, bodyforce); # Initial conditions (extend inflow) -u₀, p₀ = create_initial_conditions(setup, (dim, x, y) -> dim() == 1 ? 1.0 : 0.0); -u, p = u₀, p₀ +u₀ = create_initial_conditions(setup, (dim, x, y) -> dim() == 1 ? 1.0 : 0.0); +u = u₀ # Solve unsteady problem state, outputs = solve_unsteady( setup, u₀, - p₀, (0.0, 12.0); method = RK44P2(), Δt = 0.05, diff --git a/examples/Actuator3D.jl b/examples/Actuator3D.jl index fb902a284..32f3a13bd 100644 --- a/examples/Actuator3D.jl +++ b/examples/Actuator3D.jl @@ -79,13 +79,12 @@ bodyforce(dim, x, y, z) = dim() == 1 ? -cₜ * inside(x, y, z) : zero(x) setup = Setup(x, y, z; Re, boundary_conditions, bodyforce, ArrayType); # Initial conditions (extend inflow) -u₀, p₀ = create_initial_conditions(setup, (dim, x, y, z) -> dim() == 1 ? one(x) : zero(x)); +u₀ = create_initial_conditions(setup, (dim, x, y, z) -> dim() == 1 ? one(x) : zero(x)); # Solve unsteady problem -(; u, p, t), outputs = solve_unsteady( +(; u, t), outputs = solve_unsteady( setup, u₀, - p₀, (T(0), T(3)); method = RK44P2(), Δt = T(0.05), diff --git a/examples/BackwardFacingStep2D.jl b/examples/BackwardFacingStep2D.jl index d12d894e3..6c0f545fc 100644 --- a/examples/BackwardFacingStep2D.jl +++ b/examples/BackwardFacingStep2D.jl @@ -62,7 +62,7 @@ setup = Setup(x, y; Re, boundary_conditions, ArrayType); psolver = DirectPressureSolver(setup); # Initial conditions (extend inflow) -u₀, p₀ = create_initial_conditions(setup, (dim, x, y) -> U(dim, x, y, zero(x)); psolver); +u₀ = create_initial_conditions(setup, (dim, x, y) -> U(dim, x, y, zero(x)); psolver); # Solve steady state problem ## u, p = solve_steady_state(setup, u₀, p₀); @@ -71,7 +71,6 @@ u₀, p₀ = create_initial_conditions(setup, (dim, x, y) -> U(dim, x, y, zero(x state, outputs = solve_unsteady( setup, u₀, - p₀, (T(0), T(7)); Δt = T(0.002), psolver, diff --git a/examples/BackwardFacingStep3D.jl b/examples/BackwardFacingStep3D.jl index 4899a49b6..3cf2376d2 100644 --- a/examples/BackwardFacingStep3D.jl +++ b/examples/BackwardFacingStep3D.jl @@ -62,7 +62,7 @@ boundary_conditions = ( setup = Setup(x, y, z; Re, boundary_conditions, ArrayType); # Initial conditions (extend inflow) -u₀, p₀ = create_initial_conditions(setup, (dim, x, y, z) -> U(dim, x, y, z, zero(x))); +u₀ = create_initial_conditions(setup, (dim, x, y, z) -> U(dim, x, y, z, zero(x))); # Solve steady state problem ## u, p = solve_steady_state(setup, u₀, p₀); @@ -72,7 +72,6 @@ nothing state, outputs = solve_unsteady( setup, u₀, - p₀, (T(0), T(7)); Δt = T(0.01), processors = ( diff --git a/examples/DecayingTurbulence2D.jl b/examples/DecayingTurbulence2D.jl index e4ab68572..d242819f3 100644 --- a/examples/DecayingTurbulence2D.jl +++ b/examples/DecayingTurbulence2D.jl @@ -46,13 +46,12 @@ setup = Setup(x...; Re, ArrayType); psolver = SpectralPressureSolver(setup); # Create random initial conditions -u₀, p₀ = random_field(setup, T(0); psolver); +u₀ = random_field(setup, T(0); psolver); # Solve unsteady problem state, outputs = solve_unsteady( setup, u₀, - p₀, (T(0), T(1)); Δt = T(1e-3), psolver, diff --git a/examples/DecayingTurbulence3D.jl b/examples/DecayingTurbulence3D.jl index 8a13ddb14..e24155f42 100644 --- a/examples/DecayingTurbulence3D.jl +++ b/examples/DecayingTurbulence3D.jl @@ -48,13 +48,12 @@ setup = Setup(x, y, z; Re, ArrayType); psolver = SpectralPressureSolver(setup); # Initial conditions -u₀, p₀ = random_field(setup; psolver); +u₀ = random_field(setup; psolver); # Solve unsteady problem -(; u, p, t), outputs = solve_unsteady( +(; u, t), outputs = solve_unsteady( setup, u₀, - p₀, (T(0), T(1)); Δt = T(1e-3), psolver, diff --git a/examples/LidDrivenCavity2D.jl b/examples/LidDrivenCavity2D.jl index 159ec441d..54bd31f73 100644 --- a/examples/LidDrivenCavity2D.jl +++ b/examples/LidDrivenCavity2D.jl @@ -97,7 +97,7 @@ psolver = DirectPressureSolver(setup); # The initial conditions are provided in function. The value `dim()` determines # the velocity component. -u₀, p₀ = create_initial_conditions(setup, (dim, x, y) -> zero(x); psolver); +u₀ = create_initial_conditions(setup, (dim, x, y) -> zero(x); psolver); # ## Solve problems # @@ -128,7 +128,7 @@ processors = ( # By default, a standard fourth order Runge-Kutta method is used. If we don't # provide the time step explicitly, an adaptive time step is used. tlims = (T(0), T(10)) -state, outputs = solve_unsteady(setup, u₀, p₀, tlims; Δt = T(1e-3), psolver, processors); +state, outputs = solve_unsteady(setup, u₀, tlims; Δt = T(1e-3), psolver, processors); # ## Post-process # diff --git a/examples/LidDrivenCavity3D.jl b/examples/LidDrivenCavity3D.jl index 81227b338..af444f929 100644 --- a/examples/LidDrivenCavity3D.jl +++ b/examples/LidDrivenCavity3D.jl @@ -56,13 +56,12 @@ boundary_conditions = ( setup = Setup(x, y, z; Re, boundary_conditions, ArrayType); # Initial conditions -u₀, p₀ = create_initial_conditions(setup, (dim, x, y, z) -> zero(x)) +u₀ = create_initial_conditions(setup, (dim, x, y, z) -> zero(x)) # Solve unsteady problem -(; u, p, t), outputs = solve_unsteady( +(; u, t), outputs = solve_unsteady( setup, u₀, - p₀, (T(0), T(0.2)); Δt = T(1e-3), processors = ( diff --git a/examples/PlanarMixing2D.jl b/examples/PlanarMixing2D.jl index 289cb4c97..26b2ea446 100644 --- a/examples/PlanarMixing2D.jl +++ b/examples/PlanarMixing2D.jl @@ -56,13 +56,12 @@ plotgrid(x, y) setup = Setup(x, y; Re, boundary_conditions); # Initial conditions (extend inflow) -u₀, p₀ = create_initial_conditions(setup, (dim, x, y) -> U(dim, x, y, 0.0)); +u₀ = create_initial_conditions(setup, (dim, x, y) -> U(dim, x, y, 0.0)); # Solve unsteady problem state, outputs = solve_unsteady( setup, u₀, - p₀, (0.0, 100.0); method = RK44P2(), Δt = 0.1, diff --git a/examples/PlaneJets2D.jl b/examples/PlaneJets2D.jl index cae0dc89f..d547b98fd 100644 --- a/examples/PlaneJets2D.jl +++ b/examples/PlaneJets2D.jl @@ -98,7 +98,7 @@ setup = Setup(x, y; Re, ArrayType); psolver = SpectralPressureSolver(setup) # Initial conditions -u₀, p₀ = +u₀ = create_initial_conditions(setup, (dim, x, y) -> dim() == 1 ? U(x, y) : zero(x); psolver); # Real time plot: Streamwise average and spectrum @@ -160,7 +160,6 @@ end state, outputs = solve_unsteady( setup, u₀, - p₀, (T(0), T(1)); method = RK44P2(), Δt = 0.001, diff --git a/examples/ShearLayer2D.jl b/examples/ShearLayer2D.jl index 452d2078b..cde432ee2 100644 --- a/examples/ShearLayer2D.jl +++ b/examples/ShearLayer2D.jl @@ -53,7 +53,7 @@ d = T(π / 15) e = T(0.05) U1(y) = y ≤ π ? tanh((y - T(π / 2)) / d) : tanh((T(3π / 2) - y) / d) ## U1(y) = T(1) + (y ≤ π ? tanh((y - T(π / 2)) / d) : tanh((T(3π / 2) - y) / d)) -u₀, p₀ = create_initial_conditions( +u₀ = create_initial_conditions( setup, (dim, x, y) -> dim() == 1 ? U1(y) : e * sin(x); psolver, @@ -63,7 +63,6 @@ u₀, p₀ = create_initial_conditions( state, outputs = solve_unsteady( setup, u₀, - p₀, (T(0), T(8)); Δt = T(0.01), psolver, diff --git a/examples/TaylorGreenVortex2D.jl b/examples/TaylorGreenVortex2D.jl index 0e63819d3..41153fd3c 100644 --- a/examples/TaylorGreenVortex2D.jl +++ b/examples/TaylorGreenVortex2D.jl @@ -35,20 +35,20 @@ function compute_convergence(; D, nlist, lims, Re, tlims, Δt, uref, ArrayType = x = ntuple(α -> LinRange(lims..., n + 1), D) setup = Setup(x...; Re, ArrayType) psolver = SpectralPressureSolver(setup) - u₀, p₀ = create_initial_conditions( + u = create_initial_conditions( setup, (dim, x...) -> uref(dim, x..., tlims[1]), tlims[1]; psolver, ) - ut, pt = create_initial_conditions( + ut = create_initial_conditions( setup, (dim, x...) -> uref(dim, x..., tlims[2]), tlims[2]; psolver, - project = false, + doproject = false, ) - (; u, p, t), outputs = solve_unsteady(setup, u₀, p₀, tlims; Δt, psolver) + (; u, t), outputs = solve_unsteady(setup, u₀, tlims; Δt, psolver) (; Ip) = setup.grid a, b = T(0), T(0) for α = 1:D diff --git a/examples/TaylorGreenVortex3D.jl b/examples/TaylorGreenVortex3D.jl index d01ac0ad6..2989530cc 100644 --- a/examples/TaylorGreenVortex3D.jl +++ b/examples/TaylorGreenVortex3D.jl @@ -44,7 +44,7 @@ setup = Setup(x, y, z; Re, ArrayType); psolver = SpectralPressureSolver(setup); # Initial conditions -u₀, p₀ = create_initial_conditions( +u₀ = create_initial_conditions( setup, (dim, x, y, z) -> dim() == 1 ? sinpi(2x) * cospi(2y) * sinpi(2z) / 2 : @@ -53,10 +53,9 @@ u₀, p₀ = create_initial_conditions( ); # Solve unsteady problem -(; u, p, t), outputs = solve_unsteady( +(; u, t), outputs = solve_unsteady( setup, u₀, - p₀, (T(0), T(1.0)); Δt = T(1e-3), processors = ( diff --git a/src/closures/create_les_data.jl b/src/closures/create_les_data.jl index 195f2e31f..f8077e144 100644 --- a/src/closures/create_les_data.jl +++ b/src/closures/create_les_data.jl @@ -34,11 +34,10 @@ end function lesdatagen(dnsobs, les, compression, psolver) Φu = zero.(face_average(dnsobs[].u, les, compression)) - q = zero(pressure(psolver, Φu, dnsobs[].t, les)) - M = zero(q) + q = zero(Φu[1]) + div = zero(q) ΦF = zero.(Φu) FΦ = zero.(Φu) - GΦ = zero.(Φu) c = zero.(Φu) results = (; u = fill(Array.(dnsobs[].u), 0), c = fill(Array.(dnsobs[].u), 0)) on(dnsobs) do (; u, F, t) @@ -47,13 +46,8 @@ function lesdatagen(dnsobs, les, compression, psolver) face_average!(ΦF, F, les, compression) momentum!(FΦ, Φu, t, les) apply_bc_u!(FΦ, t, les; dudt = true) - divergence!(M, FΦ, les) - @. M *= les.grid.Ω - poisson!(psolver, q, M) - apply_bc_p!(q, t, les) - pressuregradient!(GΦ, q, les) + project!(FΦ, setup; psolver, div, q) for α = 1:length(u) - FΦ[α] .-= GΦ[α] c[α] .= ΦF[α] .- FΦ[α] end push!(results.u, Array.(Φu)) @@ -62,28 +56,28 @@ function lesdatagen(dnsobs, les, compression, psolver) results end -filtersaver(dns, les, compression, psolver; nupdate = 1) = +filtersaver(dns, les, compression, psolver_dns, psolver_les; nupdate = 1) = processor() do state (; dimension, x) = dns.grid T = eltype(x[1]) D = dimension() F = zero.(state[].u) - G = zero.(state[].u) + div = zero(state[].u[1]) + p = zero(state[].u[1]) dnsobs = Observable((; state[].u, F, state[].t)) - data = - [lesdatagen(dnsobs, les[i], compression[i], psolver[i]) for i = 1:length(les)] + data = [ + lesdatagen(dnsobs, les[i], compression[i], psolver_les[i]) for i = 1:length(les) + ] results = (; t = fill(zero(eltype(x[1])), 0), u = [d.u for d in data], c = [d.c for d in data], ) - on(state) do (; u, p, t, n) + on(state) do (; u, t, n) n % nupdate == 0 || return momentum!(F, u, t, dns) - pressuregradient!(G, p, dns) - for α = 1:D - F[α] .-= G[α] - end + apply_bc_u!(F, t, dns; dudt = true) + project!(F, dns; psolver = psolver_dns, div, p) push!(results.t, t) dnsobs[] = (; u, F, t) end @@ -146,7 +140,7 @@ function create_les_data( @info "Generating $datasize Mb of LES data" # Initial conditions - u₀, p₀ = random_field(dns, T(0); psolver, ic_params...) + u₀ = random_field(dns, T(0); psolver, ic_params...) # Random body force # force_dns = @@ -164,17 +158,23 @@ function create_les_data( # _les = (; les..., bodyforce = force_les) # Solve burn-in DNS - (; u, p, t), outputs = solve_unsteady(_dns, u₀, p₀, (T(0), tburn); Δt, psolver) + (; u, t), outputs = solve_unsteady(_dns, u₀, (T(0), tburn); Δt, psolver) # Solve DNS and store filtered quantities - (; u, p, t), outputs = solve_unsteady( + (; u, t), outputs = solve_unsteady( _dns, u, - p, (T(0), tsim); Δt, processors = (; - f = filtersaver(_dns, _les, compression, psolver_les; nupdate = savefreq), + f = filtersaver( + _dns, + _les, + compression, + psolver, + psolver_les; + nupdate = savefreq, + ), ), psolver, ) diff --git a/src/closures/training.jl b/src/closures/training.jl index 2fa75f620..3fdb611bb 100644 --- a/src/closures/training.jl +++ b/src/closures/training.jl @@ -128,7 +128,7 @@ function create_trajectory_loss(; function trajectory_loss(traj, θ) (; u, t) = traj v = u[1] - stepper = create_stepper(method; setup, psolver, u = v, p = zero(v[1]), t = t[1]) + stepper = create_stepper(method; setup, psolver, u = v, t = t[1]) loss = zero(eltype(v[1])) for it = 2:length(t) Δt = t[it] - t[it-1] diff --git a/src/create_initial_conditions.jl b/src/create_initial_conditions.jl index 2b1104a0c..1d289423a 100644 --- a/src/create_initial_conditions.jl +++ b/src/create_initial_conditions.jl @@ -4,10 +4,10 @@ initial_velocity, t = 0; psolver = DirectPressureSolver(setup), - project = true, + doproject = true, ) -Create initial vectors `(u, p)` at starting time `t`. +Create divergence free initial velocity field `u` at starting time `t`. The initial conditions of `u[α]` are specified by the function `initial_velocity(Dimension(α), x...)`. """ @@ -16,7 +16,7 @@ function create_initial_conditions( initial_velocity, t = convert(eltype(setup.grid.x[1]), 0); psolver = DirectPressureSolver(setup), - project = true, + doproject = true, ) (; grid) = setup (; dimension, N, Iu, Ip, x, xp, Ω) = grid @@ -35,26 +35,13 @@ function create_initial_conditions( ) u[α][Iu[α]] .= initial_velocity.((Dimension(α),), xin...)[Iu[α]] end - apply_bc_u!(u, t, setup) # Make velocity field divergence free - if project - f = divergence(u, setup) - @. f *= Ω - p = poisson(psolver, f) - apply_bc_p!(p, t, setup) - G = pressuregradient(p, setup) - for α = 1:D - u[α] .-= G[α] - end - end apply_bc_u!(u, t, setup) - - p = pressure(psolver, u, t, setup) - apply_bc_p!(p, t, setup) + doproject && (u = project(u, setup; psolver)) # Initial conditions, including initial boundary condititions - u, p + u end function create_spectrum(; setup, A, σ, s) @@ -111,22 +98,8 @@ function random_field( # Create random velocity field u = ntuple(α -> real.(ifft(create_spectrum(; setup, A, σ, s))), D) - apply_bc_u!(u, t, setup) # Make velocity field divergence free - M = divergence(u, setup) - @. M *= Ω - p = poisson(psolver, M) - apply_bc_p!(p, t, setup) - G = pressuregradient(p, setup) - for α = 1:D - @. u[α] -= G[α] - end apply_bc_u!(u, t, setup) - - # Compute pressure - p = pressure(psolver, u, t, setup) - apply_bc_p!(p, t, setup) - - u, p + project(u, setup; psolver) end diff --git a/src/processors/processors.jl b/src/processors/processors.jl index cd69498c5..e1b17f463 100644 --- a/src/processors/processors.jl +++ b/src/processors/processors.jl @@ -73,6 +73,7 @@ vtk_writer(; dir = "output", filename = "solution", fields = (:velocity, :pressure, :vorticity), + psolver = nothing, ) = processor((pvd, state) -> vtk_save(pvd)) do state (; grid) = setup @@ -81,7 +82,7 @@ vtk_writer(; ispath(dir) || mkpath(dir) pvd = paraview_collection(joinpath(dir, filename)) xparr = Array.(xp) - (; u, p) = state[] + (; u) = state[] if :velocity ∈ fields up = interpolate_u_p(u, setup) uparr = Array.(up) @@ -89,25 +90,33 @@ vtk_writer(; D == 2 && (uparr = (uparr..., zero(up[1]))) end if :pressure ∈ fields - parr = Array(p) + if isnothing(psolver) + @info "Creating new pressure solver for vtk_writer" + psolver = DirectPressureSolver(setup) + end + F = zero.(u) + div = zero(u[1]) + p = zero(u[1]) + parr = adapt(Array, p) end if :vorticity ∈ fields ω = vorticity(u, setup) ωp = interpolate_ω_p(ω, setup) - ωparr = D == 2 ? Array(ωp) : Array.(ωp) + ωparr = adapt(Array, ωp) end - on(state) do (; u, p, t, n) + on(state) do (; u, t, n) n % nupdate == 0 || return tformat = replace(string(t), "." => "p") vtk_grid("$(dir)/$(filename)_t=$tformat", xparr...) do vtk if :velocity ∈ fields interpolate_u_p!(up, u, setup) - for α = 1:D - copyto!(uparr[α], up[α]) - end + copyto!.(uparr, up) vtk["velocity"] = uparr end - :pressure ∈ fields && (vtk["pressure"] = copyto!(parr, p)) + if :pressure ∈ fields + pressure!(p, u, setup; psolver, F, div) + vtk["pressure"] = copyto!(parr, p) + end if :vorticity ∈ fields vorticity!(ω, u, setup) interpolate_ω_p!(ωp, ω, setup) @@ -123,17 +132,16 @@ vtk_writer(; """ fieldsaver(; setup, nupdate = 1) -Create processor that stores the solution every `nupdate` time step to the vector of vectors `V` and `p`. The solution times are stored in the vector `t`. +Create processor that stores the solution and time every `nupdate` time step. """ fieldsaver(; setup, nupdate = 1) = processor() do state T = eltype(setup.grid.x[1]) - (; u, p) = state[] - fields = (; u = fill(Array.(u), 0), p = fill(Array(p), 0), t = zeros(T, 0)) + (; u) = state[] + fields = (; u = fill(Array.(u), 0), t = zeros(T, 0)) on(state) do (; u, p, t, n) n % nupdate == 0 || return push!(fields.u, Array.(u)) - push!(fields.p, Array(p)) push!(fields.t, t) end fields diff --git a/src/processors/real_time_plot.jl b/src/processors/real_time_plot.jl index 267d86186..e633d1bc0 100644 --- a/src/processors/real_time_plot.jl +++ b/src/processors/real_time_plot.jl @@ -82,6 +82,7 @@ function fieldplot( state; setup, fieldname = :vorticity, + psolver = nothing, type = heatmap, equal_axis = true, docolorbar = true, @@ -94,23 +95,29 @@ function fieldplot( xf = Array.(getindex.(setup.grid.xp, Ip.indices)) - (; u, p, t) = state[] + (; u, t) = state[] _f = if fieldname in (1, 2) up = interpolate_u_p(u, setup) up[fieldname] elseif fieldname == :velocity up = interpolate_u_p(u, setup) - upnorm = zero(p) + upnorm = zero(up[1]) elseif fieldname == :vorticity ω = vorticity(u, setup) ωp = interpolate_ω_p(ω, setup) elseif fieldname == :streamfunction ψ = get_streamfunction(setup, u, t) elseif fieldname == :pressure - p + if isnothing(psolver) + @info "Creating new pressure solver for fieldplot" + psolver = DirectPressureSolver(setup) + end + F = zero.(u) + div = zero(u[1]) + p = zero(u[1]) end _f = Array(_f)[Ip] - field = lift(state) do (; u, p, t) + field = lift(state) do (; u, t) f = if fieldname in (1, 2) interpolate_u_p!(up, u, setup) up[fieldname] @@ -125,7 +132,7 @@ function fieldplot( elseif fieldname == :streamfunction get_streamfunction!(setup, ψ, u, t) elseif fieldname == :pressure - p + pressure!(p, u, t, setup; psolver, F, div) end # Array(f)[Ip] copyto!(_f, view(f, Ip)) @@ -181,6 +188,7 @@ function fieldplot( ::Dimension{3}, state; setup, + psolver = nothing, fieldname = :eig2field, alpha = convert(eltype(setup.grid.x[1]), 0.1), isorange = convert(eltype(setup.grid.x[1]), 0.5), @@ -195,23 +203,37 @@ function fieldplot( (; xlims, x, xp, Ip) = grid xf = Array.(getindex.(setup.grid.xp, Ip.indices)) - (; u, p) = state[] + (; u) = state[] if fieldname == :velocity elseif fieldname == :vorticity elseif fieldname == :streamfunction elseif fieldname == :pressure + if isnothing(psolver) + @info "Creating new pressure solver for fieldplot" + psolver = DirectPressureSolver(setup) + end + F = zero.(u) + div = zero(u[1]) + p = zero(u[1]) elseif fieldname == :Dfield - G = similar.(state[].u) - d = similar(state[].p) + if isnothing(psolver) + @info "Creating new pressure solver for fieldplot" + psolver = DirectPressureSolver(setup) + end + F = zero.(u) + div = zero(u[1]) + p = zero(u[1]) + G = similar.(u) + d = similar(u[1]) elseif fieldname == :Qfield - Q = similar(state[].p) + Q = similar(u[1]) elseif fieldname == :eig2field - λ = similar(state[].p) + λ = similar(u[1]) else error("Unknown fieldname") end - field = lift(state) do (; u, p, t) + field = lift(state) do (; u, t) f = if fieldname == :velocity up = interpolate_u_p(u, setup) map((u, v, w) -> √sum(u^2 + v^2 + w^2), up...) @@ -221,8 +243,9 @@ function fieldplot( elseif fieldname == :streamfunction get_streamfunction(setup, u, t) elseif fieldname == :pressure - p + pressure!(p, u, t, setup; psolver, F, div) elseif fieldname == :Dfield + pressure!(p, u, t, setup; psolver, F, div) Dfield!(d, G, p, setup) din = view(d, Ip) @. din = log(max(logtol, din)) @@ -277,7 +300,7 @@ Create energy history plot. function energy_history_plot(state; setup) @assert state isa Observable "Energy history requires observable state." _points = Point2f[] - points = lift(state) do (; u, p, t) + points = lift(state) do (; u, t) E = total_kinetic_energy(u, setup) push!(_points, Point2f(t, E)) end @@ -336,7 +359,7 @@ function energy_spectrum_plot(state; setup, doaverage = false) # Energy ke = kinetic_energy(state[].u, setup) - ehat = lift(state) do (; u, p, t) + ehat = lift(state) do (; u, t) kinetic_energy!(ke, u, setup) e = ke[Ip] e = fft(e)[ntuple(α -> kx[α] .+ 1, D)...] diff --git a/src/solvers/pressure/pressure.jl b/src/solvers/pressure/pressure.jl index 23c85b60e..7c03e8f69 100644 --- a/src/solvers/pressure/pressure.jl +++ b/src/solvers/pressure/pressure.jl @@ -1,37 +1,37 @@ """ - pressure!(psolver, u, p, t, setup, F, f, M) + pressure!(p, u, t, setup; psolver, F, div) Compute pressure from velocity field. This makes the pressure compatible with the velocity field, resulting in same order pressure as velocity. """ -function pressure!(psolver, u, p, t, setup, F, G, M) +function pressure!(p, u, t, setup; psolver, F, div) (; grid) = setup (; dimension, Iu, Ip, Ω) = grid D = dimension() momentum!(F, u, t, setup) apply_bc_u!(F, t, setup; dudt = true) - divergence!(M, F, setup) - @. M *= Ω - poisson!(psolver, p, M) + divergence!(div, F, setup) + @. div *= Ω + poisson!(psolver, p, div) apply_bc_p!(p, t, setup) p end """ - pressure(psolver, u, t, setup) + pressure(u, t, setup; psolver) Do additional pressure solve. This makes the pressure compatible with the velocity field, resulting in same order pressure as velocity. """ -function pressure(solver, u, t, setup) +function pressure(u, t, setup; psolver) (; grid) = setup (; dimension, Iu, Ip, Ω) = grid D = dimension() F = momentum(u, t, setup) F = apply_bc_u(F, t, setup; dudt = true) - M = divergence(F, setup) - M = @. M * Ω - p = poisson(psolver, M) + div = divergence(F, setup) + div = @. div * Ω + p = poisson(psolver, div) p = apply_bc_p(p, t, setup) p end diff --git a/src/solvers/pressure/project.jl b/src/solvers/pressure/project.jl index 67e6f0755..d0514062e 100644 --- a/src/solvers/pressure/project.jl +++ b/src/solvers/pressure/project.jl @@ -1,39 +1,31 @@ -function project(solver, u, setup) +function project(u, setup; psolver) (; Ω) = setup.grid T = eltype(u[1]) # Divergence of tentative velocity field - M = divergence(u, setup) - M = @. M * Ω + div = divergence(u, setup) + div = @. div * Ω # Solve the Poisson equation - p = poisson(solver, M) - p = apply_bc_p(p, T(0), setup) - - # Compute pressure correction term - G = pressuregradient(p, setup) + p = poisson(psolver, div) - # Update velocity, which is now divergence free - u .- G + # Apply pressure correction term + p = apply_bc_p(p, T(0), setup) + applypressure(u, p, setup) end -function project!(solver, u, setup; M, p, G) +function project!(u, setup; psolver, div, p) (; Ω) = setup.grid T = eltype(u[1]) # Divergence of tentative velocity field - divergence!(M, u, setup) - @. M *= Ω + divergence!(div, u, setup) + @. div *= Ω # Solve the Poisson equation - poisson!(solver, p, M) + poisson!(psolver, p, div) apply_bc_p!(p, T(0), setup) - # Compute pressure correction term - pressuregradient!(G, p, setup) - - # Update velocity, which is now divergence free - ntuple(length(u)) do α - @. u[α] -= G[α] - end + # Apply pressure correction term + applypressure!(u, p, setup) end diff --git a/src/solvers/solve_unsteady.jl b/src/solvers/solve_unsteady.jl index 24ce98666..a97e60058 100644 --- a/src/solvers/solve_unsteady.jl +++ b/src/solvers/solve_unsteady.jl @@ -2,7 +2,6 @@ solve_unsteady( setup, u₀, - p₀, tlims; method = RK44(; T = eltype(u₀[1])), psolver = DirectPressureSolver(setup), @@ -24,15 +23,14 @@ The `processors` are called after every time step. Note that the `state` observable passed to the `processor.initialize` function contains vector living on the device, and you may have to move them back to -the host using `Array(u)` and `Array(p)` in the processor. +the host using `Array(u)` in the processor. -Return `(; u, p, t), outputs`, where `outputs` is a named tuple with the +Return `(; u, t), outputs`, where `outputs` is a named tuple with the outputs of `processors` with the same field names. """ function solve_unsteady( setup, u₀, - p₀, tlims; method = RK44(; T = eltype(u₀[1])), psolver = DirectPressureSolver(setup), @@ -43,19 +41,16 @@ function solve_unsteady( processors = (;), θ = nothing, ) - if docopy - u₀ = copy.(u₀) - p₀ = copy(p₀) - end + docopy && (u₀ = copy.(u₀)) t_start, t_end = tlims isadaptive = isnothing(Δt) # Cache arrays for intermediate computations - cache = ode_method_cache(method, setup, u₀, p₀) + cache = ode_method_cache(method, setup, u₀) # Time stepper - stepper = create_stepper(method; setup, psolver, u = u₀, p = p₀, t = t_start) + stepper = create_stepper(method; setup, psolver, u = u₀, t = t_start) # Initialize processors for iteration results state = Observable(get_state(stepper)) @@ -72,7 +67,7 @@ function solve_unsteady( Δt = min(Δt, t_end - stepper.t) # Perform a single time step with the time integration method - stepper = timestep!(method, stepper, Δt; cache, θ) + stepper = timestep!(method, stepper, Δt; θ, cache) # Process iteration results with each processor state[] = get_state(stepper) @@ -82,7 +77,7 @@ function solve_unsteady( Δt = (t_end - t_start) / nstep for it = 1:nstep # Perform a single time step with the time integration method - stepper = timestep!(method, stepper, Δt; cache, θ) + stepper = timestep!(method, stepper, Δt; θ, cache) # Process iteration results with each processor state[] = get_state(stepper) @@ -90,7 +85,7 @@ function solve_unsteady( end # Final state - (; u, p, t) = stepper + (; u, t) = stepper # Processor outputs outputs = (; @@ -98,10 +93,10 @@ function solve_unsteady( ) # Return state and outputs - (; u, p, t), outputs + (; u, t), outputs end function get_state(stepper) - (; u, p, t, n) = stepper - (; u, p, t, n) + (; u, t, n) = stepper + (; u, t, n) end diff --git a/src/time_steppers/step_explicit_runge_kutta.jl b/src/time_steppers/step_explicit_runge_kutta.jl index 1209d63f2..66f7fd6c3 100644 --- a/src/time_steppers/step_explicit_runge_kutta.jl +++ b/src/time_steppers/step_explicit_runge_kutta.jl @@ -1,38 +1,23 @@ -create_stepper(::ExplicitRungeKuttaMethod; setup, psolver, u, p, t, n = 0) = - (; setup, psolver, u, p, t, n) +create_stepper(::ExplicitRungeKuttaMethod; setup, psolver, u, t, n = 0) = + (; setup, psolver, u, t, n) -function timestep!(method::ExplicitRungeKuttaMethod, stepper, Δt; cache, θ = nothing) +function timestep!(method::ExplicitRungeKuttaMethod, stepper, Δt; θ = nothing, cache) (; setup, psolver, u, p, t, n) = stepper (; grid) = setup (; dimension, Iu, Ip, Ω) = grid (; A, b, c, p_add_solve) = method - (; u₀, ku, v, F, M, G) = cache - + (; u₀, ku, div, p) = cache D = dimension() + nstage = length(b) # Update current solution (does not depend on previous step size) t₀ = t - for α = 1:D - u₀[α] .= u[α] - end - - # Number of stages - nstage = length(b) - - ## Start looping over stages + copyto!.(u₀, u) - # At i = 1 we calculate F₁ = F(u₀), p₁ and u₁ - # ⋮ - # At i = s we calculate Fₛ = F(uₛ₋₁), pₛ, and uₛ for i = 1:nstage # Right-hand side for tᵢ₋₁ based on current velocity field uᵢ₋₁, vᵢ₋₁ at # level i-1. This includes force evaluation at tᵢ₋₁. - momentum!(F, u, t, setup; θ) - - # Store right-hand side of stage i - for α = 1:D - @. ku[i][α] = F[α] - end + momentum!(ku[i], u, t, setup; θ) # Intermediate time step t = t₀ + c[i] * Δt @@ -40,42 +25,19 @@ function timestep!(method::ExplicitRungeKuttaMethod, stepper, Δt; cache, θ = n # Update velocity current stage by sum of Fᵢ's until this stage, weighted # with Butcher tableau coefficients. This gives vᵢ for α = 1:D - v[α] .= u₀[α] + u[α] .= u₀[α] for j = 1:i - @. v[α] += Δt * A[i, j] * ku[j][α] - # @. v[α][Iu[α]] += Δt * A[i, j] * ku[j][α][Iu[α]] + @. u[α] += Δt * A[i, j] * ku[j][α] + # @. u[α][Iu[α]] += Δt * A[i, j] * ku[j][α][Iu[α]] end end - # Boundary conditions at tᵢ - apply_bc_u!(v, t, setup) - - # Divergence of tentative velocity field - divergence!(M, v, setup) - @. M *= Ω / (c[i] * Δt) - - # Solve the Poisson equation - poisson!(psolver, p, M) - apply_bc_p!(p, t, setup) - - # Compute pressure correction term - pressuregradient!(G, p, setup) - - # Update velocity current stage, which is now divergence free - for α = 1:D - @. u[α] = v[α] - c[i] * Δt * G[α] - # @. u[α][Iu[α]] = v[α][Iu[α]] - c[i] * Δt * G[α][Iu[α]] - end + # Make velocity divergence free at time tᵢ apply_bc_u!(u, t, setup) + project!(u, p, setup; psolver, div, p) end - # Complete time step - t = t₀ + Δt - - # Do additional pressure solve to avoid first order pressure - p_add_solve && pressure!(psolver, u, p, t, setup, F, G, M) - - create_stepper(method; setup, psolver, u, p, t, n = n + 1) + create_stepper(method; setup, psolver, u, t, n = n + 1) end function timestep(method::ExplicitRungeKuttaMethod, stepper, Δt; θ = nothing) @@ -83,26 +45,15 @@ function timestep(method::ExplicitRungeKuttaMethod, stepper, Δt; θ = nothing) (; grid) = setup (; dimension) = grid (; A, b, c) = method - D = dimension() + nstage = length(b) # Update current solution (does not depend on previous step size) t₀ = t u₀ = u - - # Number of stages - nstage = length(b) - ku = () - ## Start looping over stages - - # At i = 1 we calculate F₁ = F(u₀), p₁ and u₁ - # ⋮ - # At i = s we calculate Fₛ = F(uₛ₋₁), pₛ, and uₛ for i = 1:nstage - # Right-hand side for tᵢ₋₁ based on current velocity field uᵢ₋₁, vᵢ₋₁ at - # level i-1. This includes force evaluation at tᵢ₋₁. u = apply_bc_u(u, t, setup) F = momentum(u, t, setup; θ) @@ -120,15 +71,10 @@ function timestep(method::ExplicitRungeKuttaMethod, stepper, Δt; θ = nothing) # u = tupleadd(u, @.(Δt * A[i, j] * ku[j])) end - # Boundary conditions at tᵢ + # Make velocity divergence free at time t u = apply_bc_u(u, t, setup) - - # Make divergence free u = project(psolver, u, setup) end - # Complete time step - t = t₀ + Δt - create_stepper(method; setup, psolver, u, p, t, n = n + 1) end diff --git a/src/time_steppers/time_stepper_caches.jl b/src/time_steppers/time_stepper_caches.jl index b469f2a4a..751e29580 100644 --- a/src/time_steppers/time_stepper_caches.jl +++ b/src/time_steppers/time_stepper_caches.jl @@ -34,15 +34,12 @@ function ode_method_cache(::OneLegMethod{T}, setup, V, p) where {T} (; u₋₁, p₋₁, F, f, Δp, GΔp) end -function ode_method_cache(method::ExplicitRungeKuttaMethod{T}, setup, u, p) where {T} +function ode_method_cache(method::ExplicitRungeKuttaMethod{T}, setup, u) where {T} u₀ = zero.(u) ns = nstage(method) ku = [zero.(u) for i = 1:ns] - v = zero.(u) - F = zero.(u) - G = zero.(u) - M = zero(p) - (; u₀, ku, v, F, M, G) + div = zero(u[1]) + (; u₀, ku, div) end function ode_method_cache(method::ImplicitRungeKuttaMethod{T}, setup, V, p) where {T} diff --git a/test/models.jl b/test/models.jl index c67aa81b9..15f911bc8 100644 --- a/test/models.jl +++ b/test/models.jl @@ -48,7 +48,7 @@ @info "Testing $(typeof(viscosity_model)) and $(typeof(convection_model))" setup = Setup(x, y; Re, viscosity_model, convection_model, u_bc, v_bc, bc_type) - V₀, p₀ = create_initial_conditions( + V = create_initial_conditions( setup, initial_velocity_u, initial_velocity_v, @@ -60,7 +60,7 @@ broken = convection_model isa Union{C2ConvectionModel,C4ConvectionModel} @test sum(abs, V) / length(V) < lid_vel broken = broken - (; u, p, t), outputs = solve_unsteady(setup, V₀, p₀, tlims; Δt = 0.01) + (; u, t), outputs = solve_unsteady(setup, V₀, tlims; Δt = 0.01) # Check that the average velocity is smaller than the lid velocity broken = convection_model isa Union{C2ConvectionModel,C4ConvectionModel} diff --git a/test/postprocess2D.jl b/test/postprocess2D.jl index 153eb756f..d1b26e74f 100644 --- a/test/postprocess2D.jl +++ b/test/postprocess2D.jl @@ -17,7 +17,7 @@ initial_velocity_u(x, y) = -sin(x)cos(y) initial_velocity_v(x, y) = cos(x)sin(y) initial_pressure(x, y) = 1 / 4 * (cos(2x) + cos(2y)) - V₀, p₀ = create_initial_conditions( + V = create_initial_conditions( setup, initial_velocity_u, initial_velocity_v, @@ -40,7 +40,7 @@ ) # Solve unsteady problem - state, outputs = solve_unsteady(setup, V₀, p₀, tlims; Δt = 0.01, processors, psolver) + state, outputs = solve_unsteady(setup, V₀, tlims; Δt = 0.01, processors, psolver) @testset "VTK files" begin @info "Testing 2D processors: VTK files" diff --git a/test/postprocess3D.jl b/test/postprocess3D.jl index c27ac1b4e..28e6693f1 100644 --- a/test/postprocess3D.jl +++ b/test/postprocess3D.jl @@ -20,7 +20,7 @@ initial_velocity_v(x, y, z) = -cos(x)sin(y)cos(z) initial_velocity_w(x, y, z) = zero(x) initial_pressure(x, y, z) = 1 // 4 * (cos(2x) + cos(2y) + cos(2z)) - V₀, p₀ = create_initial_conditions( + V = create_initial_conditions( setup, initial_velocity_u, initial_velocity_v, @@ -44,7 +44,7 @@ ) # Solve unsteady problem - state, outputs = solve_unsteady(setup, u₀, p₀, tlims; Δt = T(0.01), processors, psolver) + state, outputs = solve_unsteady(setup, u₀, tlims; Δt = T(0.01), processors, psolver) @testset "VTK files" begin @info "Testing 3D processors: VTK files" diff --git a/test/simulation2D.jl b/test/simulation2D.jl index 98dd40faf..a915f4f02 100644 --- a/test/simulation2D.jl +++ b/test/simulation2D.jl @@ -20,7 +20,7 @@ initial_velocity_u(x, y) = 0.0 initial_velocity_v(x, y) = 0.0 initial_pressure(x, y) = 0.0 - V₀, p₀ = create_initial_conditions( + V = create_initial_conditions( setup, initial_velocity_u, initial_velocity_v, @@ -43,11 +43,10 @@ processors = (timelogger(),) @testset "Unsteady problem" begin - (; u, p, t), outputs = solve_unsteady(setup, V₀, p₀, tlims; Δt = 0.01, processors) + (; u, t), outputs = solve_unsteady(setup, V₀, tlims; Δt = 0.01, processors) # Check that solution did not explode @test all(!isnan, u) - @test all(!isnan, p) # Check that the average velocity is smaller than the lid velocity @test sum(abs, u) / length(u) < lid_vel diff --git a/test/simulation3D.jl b/test/simulation3D.jl index 511eb88a0..7397fb4c2 100644 --- a/test/simulation3D.jl +++ b/test/simulation3D.jl @@ -18,7 +18,7 @@ initial_velocity_v(x, y, z) = 0.0 initial_velocity_w(x, y, z) = 0.0 initial_pressure(x, y, z) = 0.0 - V₀, p₀ = create_initial_conditions( + V = create_initial_conditions( setup, initial_velocity_u, initial_velocity_v, @@ -42,7 +42,7 @@ processors = (timelogger(),) @testset "Unsteady problem" begin - (; u, p, t), outputs = solve_unsteady(setup, V₀, p₀, tlims; Δt = 0.01, processors) + (; u, t), outputs = solve_unsteady(setup, V₀, tlims; Δt = 0.01, processors) # Check that solution did not explode @test all(!isnan, u) diff --git a/test/solvers.jl b/test/solvers.jl index 74b7a2258..2eb2bb80d 100644 --- a/test/solvers.jl +++ b/test/solvers.jl @@ -14,7 +14,7 @@ initial_velocity_u(x, y) = cos(x)sin(y) initial_velocity_v(x, y) = -sin(x)cos(y) initial_pressure(x, y) = -1 / 4 * (cos(2x) + cos(2y)) - V₀, p₀ = create_initial_conditions( + V = create_initial_conditions( setup, initial_velocity_u, initial_velocity_v, @@ -45,11 +45,11 @@ @testset "Explicit Runge Kutta" begin @info "Testing explicit Runge-Kutta, out-of-place version" state, outputs = - solve_unsteady(setup, V₀, p₀, tlims; Δt = 0.01, psolver, inplace = false) + solve_unsteady(setup, V₀, tlims; Δt = 0.01, psolver, inplace = false) @test norm(state.u - u_exact) / norm(u_exact) < 1e-4 @info "Testing explicit Runge-Kutta, in-place version" stateip, outputsip = - solve_unsteady(setup, V₀, p₀, tlims; Δt = 0.01, psolver, inplace = true) + solve_unsteady(setup, V₀, tlims; Δt = 0.01, psolver, inplace = true) @test stateip.u ≈ state.u @test stateip.p ≈ state.p end @@ -59,7 +59,6 @@ @test_broken solve_unsteady( setup, V₀, - p₀, tlims; method = RIA2(), Δt = 0.01, @@ -67,10 +66,9 @@ inplace = false, ) isa Tuple @info "Testing implicit Runge-Kutta, in-place version" - (; u, p, t), outputs = solve_unsteady( + (; u, t), outputs = solve_unsteady( setup, V₀, - p₀, tlims; method = RIA2(), Δt = 0.01, @@ -86,7 +84,6 @@ state, outputs = solve_unsteady( setup, V₀, - p₀, tlims; method = OneLegMethod(T), Δt = 0.01, @@ -98,7 +95,6 @@ stateip, outputsip = solve_unsteady( setup, V₀, - p₀, tlims; method = OneLegMethod(T), Δt = 0.01, @@ -114,7 +110,6 @@ state, outputs = solve_unsteady( setup, V₀, - p₀, tlims; method = AdamsBashforthCrankNicolsonMethod(T), Δt = 0.01, @@ -126,7 +121,6 @@ stateip, outputs = solve_unsteady( setup, V₀, - p₀, tlims; method = AdamsBashforthCrankNicolsonMethod(T), Δt = 0.01, From 00c78dab6447fd5061371db8face49389c69ceb9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Wed, 20 Dec 2023 16:19:18 +0100 Subject: [PATCH 216/379] Add missing function --- src/operators.jl | 55 +++++++++++++++++++++++++++++++++ src/solvers/pressure/project.jl | 3 +- 2 files changed, 57 insertions(+), 1 deletion(-) diff --git a/src/operators.jl b/src/operators.jl index 9310218c9..832325df5 100644 --- a/src/operators.jl +++ b/src/operators.jl @@ -585,6 +585,61 @@ ChainRulesCore.rrule(::typeof(pressuregradient), p, setup) = ( φ -> (NoTangent(), pressuregradient_adjoint!(similar(p), (φ...,), setup), NoTangent()), ) +""" + applypressure!(u, p, setup) + +Subtract pressure gradient (in-place). +""" +function applypressure!(u, p, setup) + (; grid, workgroupsize) = setup + (; dimension, Δu, Nu, Iu) = grid + D = dimension() + δ = Offset{D}() + @kernel function apply!(u, p, ::Val{α}, I0) where {α} + I = @index(Global, Cartesian) + I = I0 + I + u[α][I] -= (p[I+δ(α)] - p[I]) / Δu[α][I[α]] + end + D = dimension() + for α = 1:D + I0 = first(Iu[α]) + I0 -= oneunit(I0) + apply!(get_backend(u[1]), workgroupsize)(u, p, Val(α), I0; ndrange = Nu[α]) + end + u +end + +# function applypressure_adjoint!(pbar, φ, u, setup) +# (; grid, workgroupsize) = setup +# (; dimension, Δu, N, Iu) = grid +# D = dimension() +# δ = Offset{D}() +# @kernel function adj!(p, φ) +# I = @index(Global, Cartesian) +# p[I] = zero(eltype(p)) +# for α = 1:D +# I - δ(α) ∈ Iu[α] && (p[I] += φ[α][I-δ(α)] / Δu[α][I[α]-1]) +# I ∈ Iu[α] && (p[I] -= φ[α][I] / Δu[α][I[α]]) +# end +# end +# adj!(get_backend(pbar), workgroupsize)(pbar, φ; ndrange = N) +# pbar +# end +# +# """ +# applypressure(p, setup) +# +# Compute pressure gradient. +# """ +# applypressure(u, p, setup) = +# applypressure!(copy.(u), p, setup) +# +# ChainRulesCore.rrule(::typeof(applypressure), p, setup) = ( +# applypressure(u, p, setup), +# φ -> (NoTangent(), applypressure_adjoint!(similar(p), (φ...,), setup), NoTangent()), +# ) + + """ laplacian!(L, p, setup) diff --git a/src/solvers/pressure/project.jl b/src/solvers/pressure/project.jl index d0514062e..98b2f8d70 100644 --- a/src/solvers/pressure/project.jl +++ b/src/solvers/pressure/project.jl @@ -11,7 +11,8 @@ function project(u, setup; psolver) # Apply pressure correction term p = apply_bc_p(p, T(0), setup) - applypressure(u, p, setup) + G = pressuregradient(p, setup) + u .- G end function project!(u, setup; psolver, div, p) From 2e5a47fed237ca19c0033f2a2fcc638c8b0495c7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Wed, 20 Dec 2023 16:27:16 +0100 Subject: [PATCH 217/379] Minor changes --- scratch/filter_plot.jl | 16 +++++++--------- scratch/multigrid.jl | 19 ++++++++----------- scratch/testgrad.jl | 7 +++---- scratch/train_model.jl | 16 ++++------------ src/operators.jl | 1 - 5 files changed, 22 insertions(+), 37 deletions(-) diff --git a/scratch/filter_plot.jl b/scratch/filter_plot.jl index 3f66bc4b0..d380fccda 100644 --- a/scratch/filter_plot.jl +++ b/scratch/filter_plot.jl @@ -38,8 +38,8 @@ setup = Setup(x...; Re, ArrayType); # spectral pressure solver pressure_solver = SpectralPressureSolver(setup); -u₀, p₀ = random_field(setup, T(0); pressure_solver); -u, p = u₀, p₀ +u₀ = random_field(setup, T(0); pressure_solver); +u = u₀ function filter_plot(state, setup, les, comp; resolution = (1200, 600)) (; boundary_conditions, grid) = setup @@ -47,13 +47,13 @@ function filter_plot(state, setup, les, comp; resolution = (1200, 600)) D = dimension() xf = Array.(getindex.(setup.grid.xp, Ip.indices)) xfbar = Array.(getindex.(les.grid.xp, les.grid.Ip.indices)) - (; u, p, t) = state[] + (; u, t) = state[] ω = IncompressibleNavierStokes.vorticity(u, setup) ωp = IncompressibleNavierStokes.interpolate_ω_p(ω, setup) _f = Array(ωp)[Ip] f = @lift begin sleep(0.001) - (; u, p, t) = $state + (; u, t) = $state IncompressibleNavierStokes.apply_bc_u!(u, t, setup) IncompressibleNavierStokes.vorticity!(ω, u, setup) IncompressibleNavierStokes.interpolate_ω_p!(ωp, ω, setup) @@ -64,7 +64,7 @@ function filter_plot(state, setup, les, comp; resolution = (1200, 600)) ωpbar = IncompressibleNavierStokes.interpolate_ω_p(ωbar, les) _g = Array(ωpbar)[les.grid.Ip] g = @lift begin - (; u, p, t) = $state + (; u, t) = $state IncompressibleNavierStokes.face_average!(ubar, u, les, comp) IncompressibleNavierStokes.apply_bc_u!(ubar, t, les) IncompressibleNavierStokes.vorticity!(ωbar, ubar, les) @@ -123,11 +123,9 @@ comp = 8 les = Setup(x[1][1:comp:end], x[2][1:comp:end]; Re, ArrayType) # Solve unsteady problem -u, p, outputs = solve_unsteady( +(; u)outputs = solve_unsteady( setup, - copy.(u₀), - copy(p₀), - # u, p, + u₀, (T(0), T(2e0)); Δt = T(1e-4), pressure_solver, diff --git a/scratch/multigrid.jl b/scratch/multigrid.jl index 3960bbe74..5e77690b6 100644 --- a/scratch/multigrid.jl +++ b/scratch/multigrid.jl @@ -101,7 +101,7 @@ ig = 5 # field, setup = data_valid[1].u[ig], setups_valid[ig]; field, setup = data_test.u[ig], setups_test[ig]; u = device(field[1]); -o = Observable((; u, p = similar(u[1], setup.grid.N), t = nothing)); +o = Observable((; u, t = nothing)); energy_spectrum_plot(o; setup) # fieldplot( # o; @@ -364,14 +364,13 @@ for (i, setup) in enumerate(setups_test[2:4]) pressure_solver = SpectralPressureSolver(setup) u = device.(data_test.u[ig]) u₀ = device(data_test.u[ig][1]) - p₀ = IncompressibleNavierStokes.pressure(pressure_solver, u₀, T(0), setup) Δt = params_test.Δt * params_test.savefreq tlims = extrema(data_test.t) nupdate = 4 Δt /= nupdate processors = (; relerr = relerr_trajectory(u, setup; nupdate)) closedsetup = (; setup..., closure_model = wrappedclosure(closure, θ_cnn[i], setup)) - _, outputs = solve_unsteady(closedsetup, u₀, p₀, tlims; Δt, pressure_solver, processors) + _, outputs = solve_unsteady(closedsetup, u₀, tlims; Δt, pressure_solver, processors) e_cnn[i] = outputs.relerr[] end @@ -385,20 +384,19 @@ for (ig, setup) in enumerate(setups_test) pressure_solver = SpectralPressureSolver(setup) u = device.(data_test.u[ig]) u₀ = device(data_test.u[ig][1]) - p₀ = IncompressibleNavierStokes.pressure(pressure_solver, u₀, T(0), setup) Δt = params_test.Δt * params_test.savefreq tlims = extrema(data_test.t) nupdate = 4 Δt /= nupdate processors = (; relerr = relerr_trajectory(u, setup; nupdate)) - _, outputs = solve_unsteady(setup, u₀, p₀, tlims; Δt, pressure_solver, processors) + _, outputs = solve_unsteady(setup, u₀, tlims; Δt, pressure_solver, processors) e_nm[ig] = outputs.relerr[] m = smagorinsky_closure(setup) closedsetup = (; setup..., closure_model = u -> m(u, T(0.1))) - _, outputs = solve_unsteady(closedsetup, u₀, p₀, tlims; Δt, pressure_solver, processors) + _, outputs = solve_unsteady(closedsetup, u₀, tlims; Δt, pressure_solver, processors) e_smag[ig] = outputs.relerr[] closedsetup = (; setup..., closure_model = wrappedclosure(closure, θ_cnn_shared, setup)) - _, outputs = solve_unsteady(closedsetup, u₀, p₀, tlims; Δt, pressure_solver, processors) + _, outputs = solve_unsteady(closedsetup, u₀, tlims; Δt, pressure_solver, processors) e_cnn_shared[ig] = outputs.relerr[] end @@ -484,18 +482,17 @@ params = params_test pressure_solver = SpectralPressureSolver(setup); uref = device(data_test.u[ig][end]); u₀ = device(data_test.u[ig][1]); -p₀ = IncompressibleNavierStokes.pressure(pressure_solver, u₀, T(0), setup); Δt = params_test.Δt * params_test.savefreq; tlims = extrema(data_test.t); nupdate = 4; Δt /= nupdate; -state_nm, outputs = solve_unsteady(setup, u₀, p₀, tlims; Δt, pressure_solver); +state_nm, outputs = solve_unsteady(setup, u₀, tlims; Δt, pressure_solver); m = smagorinsky_closure(setup); closedsetup = (; setup..., closure_model = u -> m(u, T(0.1))); -state_smag, outputs = solve_unsteady(closedsetup, u₀, p₀, tlims; Δt, pressure_solver); +state_smag, outputs = solve_unsteady(closedsetup, u₀, tlims; Δt, pressure_solver); # closedsetup = (; setup..., closure_model = wrappedclosure(closure, θ_cnn_shared, setup)); closedsetup = (; setup..., closure_model = wrappedclosure(closure, θ_cnn[ig-1], setup)); -state_cnn, outputs = solve_unsteady(closedsetup, u₀, p₀, tlims; Δt, pressure_solver); +state_cnn, outputs = solve_unsteady(closedsetup, u₀, tlims; Δt, pressure_solver); # Plot predicted spectra fig = with_theme(; palette = (; color = ["#3366cc", "#cc0000", "#669900", "#ffcc00"])) do diff --git a/scratch/testgrad.jl b/scratch/testgrad.jl index c3821c29f..08feec54a 100644 --- a/scratch/testgrad.jl +++ b/scratch/testgrad.jl @@ -63,8 +63,8 @@ setup = Setup(x...; Re, ArrayType); # spectral pressure solver pressure_solver = SpectralPressureSolver(setup); -u₀, p₀ = random_field(setup, T(0); pressure_solver); -u, p = u₀, p₀ +u₀ = random_field(setup, T(0); pressure_solver); +u = u₀ using KernelAbstractions using Zygote @@ -99,7 +99,7 @@ gradient(u -> sum(IncompressibleNavierStokes.diffusion(u, setup)[1]), u)[1][1] # Divergence ur = randn!.(similar.(u)) -φ = IncompressibleNavierStokes.divergence!(zero(p), ur, setup) +φ = IncompressibleNavierStokes.divergence(ur, setup) φbar = randn!(similar(φ)) dot(φbar, φ) dot(IncompressibleNavierStokes.divergence_adjoint!(zero.(ur), φbar, setup), ur) @@ -158,7 +158,6 @@ function f(u, setup) setup, pressure_solver, u, - p, t = T(0), ) stepper = IncompressibleNavierStokes.timestep(method, stepper, T(1e-4)) diff --git a/scratch/train_model.jl b/scratch/train_model.jl index 4655e050f..916dd982c 100644 --- a/scratch/train_model.jl +++ b/scratch/train_model.jl @@ -104,11 +104,7 @@ end # Inspect data field = data_train[1].u[1]; u = device(field[1]) -o = Observable((; - u, - p = IncompressibleNavierStokes.pressure(pressure_solver, u, T(0), setup), - t = nothing, -)) +o = Observable((; u, t = nothing)) fieldplot( o; setup, @@ -254,10 +250,9 @@ function relerr(u, uref, setup) sqrt(a) / sqrt(b) end -u, u₀, p₀ = nothing, nothing, nothing +u, u₀, = nothing, nothing u = device.(data_test[1].u[1]); u₀ = device(data_test[1].u[1][1]); -p₀ = IncompressibleNavierStokes.pressure(pressure_solver, u₀, T(0), setup); Δt = data_test[1].t[2] - data_test[1].t[1] tlims = extrema(data_test[1].t) length(u) @@ -266,7 +261,6 @@ length(data_test[1].t) state_nm, outputs = solve_unsteady( setup, u₀, - p₀, tlims; Δt, pressure_solver, @@ -280,7 +274,6 @@ relerr_nm = outputs.relerr[] state_cnn, outputs = solve_unsteady( (; setup..., closure_model = wrappedclosure(closure, θ, setup)), u₀, - p₀, tlims; Δt, pressure_solver, @@ -300,7 +293,7 @@ dcnn function energy_history(setup, state) (; Ωp) = setup.grid points = Point2f[] - on(state) do (; V, p, t) + on(state) do (; V, t) V = Array(V) vels = get_velocity(setup, V, t) vels = reshape.(vels, :) @@ -317,10 +310,9 @@ isample = 1 forcedsetup = (; setup..., force = data_train.force[:, isample]); devsetup = device(forcedsetup); -V_nm, p_nm, outputs_nm = solve_unsteady( +V_nm, outputs_nm = solve_unsteady( forcedsetup, data_test.V[:, 1, isample], - data_test.p[:, 1, isample], (T(0), tsim); Δt = T(2e-4), processors = ( diff --git a/src/operators.jl b/src/operators.jl index 832325df5..0350f08e0 100644 --- a/src/operators.jl +++ b/src/operators.jl @@ -639,7 +639,6 @@ end # φ -> (NoTangent(), applypressure_adjoint!(similar(p), (φ...,), setup), NoTangent()), # ) - """ laplacian!(L, p, setup) From 6b04f1d0efdc40145876c17db3c6cd318199bf87 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Wed, 20 Dec 2023 16:39:56 +0100 Subject: [PATCH 218/379] Minor fixes --- src/time_steppers/step_explicit_runge_kutta.jl | 8 ++++---- src/time_steppers/time_stepper_caches.jl | 3 ++- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/src/time_steppers/step_explicit_runge_kutta.jl b/src/time_steppers/step_explicit_runge_kutta.jl index 66f7fd6c3..b07a75401 100644 --- a/src/time_steppers/step_explicit_runge_kutta.jl +++ b/src/time_steppers/step_explicit_runge_kutta.jl @@ -2,7 +2,7 @@ create_stepper(::ExplicitRungeKuttaMethod; setup, psolver, u, t, n = 0) = (; setup, psolver, u, t, n) function timestep!(method::ExplicitRungeKuttaMethod, stepper, Δt; θ = nothing, cache) - (; setup, psolver, u, p, t, n) = stepper + (; setup, psolver, u, t, n) = stepper (; grid) = setup (; dimension, Iu, Ip, Ω) = grid (; A, b, c, p_add_solve) = method @@ -34,14 +34,14 @@ function timestep!(method::ExplicitRungeKuttaMethod, stepper, Δt; θ = nothing, # Make velocity divergence free at time tᵢ apply_bc_u!(u, t, setup) - project!(u, p, setup; psolver, div, p) + project!(u, setup; psolver, div, p) end create_stepper(method; setup, psolver, u, t, n = n + 1) end function timestep(method::ExplicitRungeKuttaMethod, stepper, Δt; θ = nothing) - (; setup, psolver, u, p, t, n) = stepper + (; setup, psolver, u, t, n) = stepper (; grid) = setup (; dimension) = grid (; A, b, c) = method @@ -76,5 +76,5 @@ function timestep(method::ExplicitRungeKuttaMethod, stepper, Δt; θ = nothing) u = project(psolver, u, setup) end - create_stepper(method; setup, psolver, u, p, t, n = n + 1) + create_stepper(method; setup, psolver, u, t, n = n + 1) end diff --git a/src/time_steppers/time_stepper_caches.jl b/src/time_steppers/time_stepper_caches.jl index 751e29580..20a6ea9bc 100644 --- a/src/time_steppers/time_stepper_caches.jl +++ b/src/time_steppers/time_stepper_caches.jl @@ -39,7 +39,8 @@ function ode_method_cache(method::ExplicitRungeKuttaMethod{T}, setup, u) where { ns = nstage(method) ku = [zero.(u) for i = 1:ns] div = zero(u[1]) - (; u₀, ku, div) + p = zero(u[1]) + (; u₀, ku, div, p) end function ode_method_cache(method::ImplicitRungeKuttaMethod{T}, setup, V, p) where {T} From f1d8151a780d3332741da332daa7f99d5af9e46e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Wed, 20 Dec 2023 18:02:45 +0100 Subject: [PATCH 219/379] Add missing BC --- scratch/multigrid.jl | 8 ++--- src/closures/create_les_data.jl | 6 ++-- src/create_initial_conditions.jl | 8 +++-- src/operators.jl | 6 ++-- .../step_explicit_runge_kutta.jl | 31 ++++++++++++------- 5 files changed, 36 insertions(+), 23 deletions(-) diff --git a/scratch/multigrid.jl b/scratch/multigrid.jl index 5e77690b6..e3f4e7061 100644 --- a/scratch/multigrid.jl +++ b/scratch/multigrid.jl @@ -237,7 +237,7 @@ fig = with_theme(; palette = (; color = ["#3366cc", "#cc0000", "#669900", "#ffcc ) for (i, it) in enumerate((1, length(sample.t))) u = device.(sample.u[ig][it]) - ke = kinetic_energy(u, setup) + ke = IncompressibleNavierStokes.kinetic_energy(u, setup) e = ke[Ip] e = fft(e)[ntuple(α -> kx[α] .+ 1, D)...] e = abs.(e) ./ size(e, 1) @@ -558,7 +558,7 @@ fig = with_theme(; palette = (; color = ["#3366cc", "#cc0000", "#669900", "#ffcc (state_cnn.u, "CNN (specialized)"), (uref, "Reference"), ) - ke = kinetic_energy(u, setup) + ke = IncompressibleNavierStokes.kinetic_energy(u, setup) e = ke[Ip] e = fft(e)[ntuple(α -> kx[α] .+ 1, D)...] e = abs.(e) ./ size(e, 1) @@ -632,7 +632,7 @@ fig = with_theme(; palette = (; color = ["#3366cc", "#cc0000", "#669900", "#ffcc yscale = log10, limits = (extrema(kint)..., T(1e-8), T(1)), ) - ke = kinetic_energy(uref, setup) + ke = IncompressibleNavierStokes.kinetic_energy(uref, setup) e = ke[Ip] e = fft(e)[ntuple(α -> kx[α] .+ 1, D)...] e = abs.(e) ./ size(e, 1) @@ -643,7 +643,7 @@ fig = with_theme(; palette = (; color = ["#3366cc", "#cc0000", "#669900", "#ffcc (state_smag.u, "Smagorinsky"), (state_cnn.u, "CNN (specialized)"), ) - ke = kinetic_energy(u, setup) + ke = IncompressibleNavierStokes.kinetic_energy(u, setup) e = ke[Ip] e = fft(e)[ntuple(α -> kx[α] .+ 1, D)...] e = abs.(e) ./ size(e, 1) diff --git a/src/closures/create_les_data.jl b/src/closures/create_les_data.jl index f8077e144..67493ba60 100644 --- a/src/closures/create_les_data.jl +++ b/src/closures/create_les_data.jl @@ -34,8 +34,8 @@ end function lesdatagen(dnsobs, les, compression, psolver) Φu = zero.(face_average(dnsobs[].u, les, compression)) - q = zero(Φu[1]) - div = zero(q) + p = zero(Φu[1]) + div = zero(p) ΦF = zero.(Φu) FΦ = zero.(Φu) c = zero.(Φu) @@ -46,7 +46,7 @@ function lesdatagen(dnsobs, les, compression, psolver) face_average!(ΦF, F, les, compression) momentum!(FΦ, Φu, t, les) apply_bc_u!(FΦ, t, les; dudt = true) - project!(FΦ, setup; psolver, div, q) + project!(FΦ, les; psolver, div, p) for α = 1:length(u) c[α] .= ΦF[α] .- FΦ[α] end diff --git a/src/create_initial_conditions.jl b/src/create_initial_conditions.jl index 1d289423a..e8c41c525 100644 --- a/src/create_initial_conditions.jl +++ b/src/create_initial_conditions.jl @@ -38,9 +38,12 @@ function create_initial_conditions( # Make velocity field divergence free apply_bc_u!(u, t, setup) - doproject && (u = project(u, setup; psolver)) + if doproject + u = project(u, setup; psolver) + apply_bc_u!(u, t, setup) + end - # Initial conditions, including initial boundary condititions + # Initial conditions, including initial boundary conditions u end @@ -102,4 +105,5 @@ function random_field( # Make velocity field divergence free apply_bc_u!(u, t, setup) project(u, setup; psolver) + apply_bc_u!(u, t, setup) end diff --git a/src/operators.jl b/src/operators.jl index 0350f08e0..41093ce89 100644 --- a/src/operators.jl +++ b/src/operators.jl @@ -487,9 +487,9 @@ function momentum!(F, u, t, setup; θ = nothing) F end -monitor(u) = (@info("Forward", typeof(u)); u) -ChainRulesCore.rrule(::typeof(monitor), u) = - (monitor(u), φ -> (@info("Reverse", typeof(φ)); (NoTangent(), φ))) +# monitor(u) = (@info("Forward", typeof(u)); u) +# ChainRulesCore.rrule(::typeof(monitor), u) = +# (monitor(u), φ -> (@info("Reverse", typeof(φ)); (NoTangent(), φ))) # tupleadd(u...) = ntuple(α -> sum(u -> u[α], u), length(u[1])) # ChainRulesCore.rrule(::typeof(tupleadd), u...) = diff --git a/src/time_steppers/step_explicit_runge_kutta.jl b/src/time_steppers/step_explicit_runge_kutta.jl index b07a75401..3e9da8d00 100644 --- a/src/time_steppers/step_explicit_runge_kutta.jl +++ b/src/time_steppers/step_explicit_runge_kutta.jl @@ -4,26 +4,25 @@ create_stepper(::ExplicitRungeKuttaMethod; setup, psolver, u, t, n = 0) = function timestep!(method::ExplicitRungeKuttaMethod, stepper, Δt; θ = nothing, cache) (; setup, psolver, u, t, n) = stepper (; grid) = setup - (; dimension, Iu, Ip, Ω) = grid - (; A, b, c, p_add_solve) = method + (; dimension, Iu) = grid + (; A, b, c) = method (; u₀, ku, div, p) = cache D = dimension() nstage = length(b) - # Update current solution (does not depend on previous step size) + # Update current solution t₀ = t copyto!.(u₀, u) for i = 1:nstage - # Right-hand side for tᵢ₋₁ based on current velocity field uᵢ₋₁, vᵢ₋₁ at - # level i-1. This includes force evaluation at tᵢ₋₁. + # Compute force at current stage i + apply_bc_u!(u, t, setup) momentum!(ku[i], u, t, setup; θ) # Intermediate time step t = t₀ + c[i] * Δt - # Update velocity current stage by sum of Fᵢ's until this stage, weighted - # with Butcher tableau coefficients. This gives vᵢ + # Apply stage forces for α = 1:D u[α] .= u₀[α] for j = 1:i @@ -32,11 +31,16 @@ function timestep!(method::ExplicitRungeKuttaMethod, stepper, Δt; θ = nothing, end end - # Make velocity divergence free at time tᵢ + # Make velocity divergence free at time t apply_bc_u!(u, t, setup) project!(u, setup; psolver, div, p) end + # This is redundant, but Neumann BC need to have _exact_ copies + # since we divide by an infinitely thin (eps(T)) volume width in the + # diffusion term + apply_bc_u!(u, t, setup) + create_stepper(method; setup, psolver, u, t, n = n + 1) end @@ -54,6 +58,7 @@ function timestep(method::ExplicitRungeKuttaMethod, stepper, Δt; θ = nothing) ku = () for i = 1:nstage + # Compute force at current stage i u = apply_bc_u(u, t, setup) F = momentum(u, t, setup; θ) @@ -63,8 +68,7 @@ function timestep(method::ExplicitRungeKuttaMethod, stepper, Δt; θ = nothing) # Intermediate time step t = t₀ + c[i] * Δt - # Update velocity current stage by sum of Fᵢ's until this stage, weighted - # with Butcher tableau coefficients. This gives vᵢ + # Apply stage forces u = u₀ for j = 1:i u = @. u + Δt * A[i, j] * ku[j] @@ -73,8 +77,13 @@ function timestep(method::ExplicitRungeKuttaMethod, stepper, Δt; θ = nothing) # Make velocity divergence free at time t u = apply_bc_u(u, t, setup) - u = project(psolver, u, setup) + u = project(u, setup; psolver) end + # This is redundant, but Neumann BC need to have _exact_ copies + # since we divide by an infinitely thin (eps(T)) volume width in the + # diffusion term + u = apply_bc_u(u, t, setup) + create_stepper(method; setup, psolver, u, t, n = n + 1) end From 7665b1138c9fa24a4a5a3e410181fbc41a8da9b8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Thu, 21 Dec 2023 09:22:10 +0100 Subject: [PATCH 220/379] Add different setup --- scratch/multigrid.jl | 5 +++-- scratch/train_model.jl | 7 ++++--- src/closures/create_les_data.jl | 33 ++++++++++++++++++++++----------- 3 files changed, 29 insertions(+), 16 deletions(-) diff --git a/scratch/multigrid.jl b/scratch/multigrid.jl index e3f4e7061..1e470d10e 100644 --- a/scratch/multigrid.jl +++ b/scratch/multigrid.jl @@ -59,9 +59,10 @@ get_params(nles) = (; tburn = T(0.05), tsim = T(0.5), Δt = T(1e-4), - nles, - ndns = 2048, + nles = map(n -> (n, n), nles), + ndns = (2048, 2048), ArrayType, + PSolver = SpectralPressureSolver, ) params_train = (; get_params([64, 128, 256])..., savefreq = 5); diff --git a/scratch/train_model.jl b/scratch/train_model.jl index 916dd982c..a03c1734f 100644 --- a/scratch/train_model.jl +++ b/scratch/train_model.jl @@ -57,14 +57,15 @@ params = (; D = 2, Re = T(6_000), lims = (T(0), T(1)), - nles = [nles], - # ndns = 512, - ndns = 1024, + nles = [(nles, nles)], + # ndns = (512, 512), + ndns = (1024, 1024), tburn = T(0.05), tsim = T(0.5), Δt = T(1e-4), savefreq = 5, ArrayType, + PSolver = SpectralPressureSolver, ) # Create LES data from DNS diff --git a/src/closures/create_les_data.jl b/src/closures/create_les_data.jl index 67493ba60..9e75b2674 100644 --- a/src/closures/create_les_data.jl +++ b/src/closures/create_les_data.jl @@ -105,30 +105,41 @@ function create_les_data( T; D = 2, Re = T(2_000), - lims = (T(0), T(1)), - nles = [64], - ndns = 256, + lims = ntuple(α -> (T(0), T(1)), D), + nles = [ntuple(α -> 64, D)], + ndns = ntuple(α -> 256, D), tburn = T(0.1), tsim = T(0.1), Δt = T(1e-4), + PSolver = SpectralPressureSolver, savefreq = 1, ArrayType = Array, - ic_params = (;), + icfunc = (setup, psolver) -> random_field(setup, T(0); psolver), + boundary_conditions = ntuple(α -> (PeriodicBC(), PeriodicBC()), D), ) compression = @. ndns ÷ nles @assert all(@.(compression * nles == ndns)) # Build setup and assemble operators - dns = Setup(ntuple(α -> LinRange(lims..., ndns + 1), D)...; Re, ArrayType) + dns = Setup( + ntuple(α -> LinRange(lims[α]..., ndns[α] + 1), D)...; + Re, + boundary_conditions, + ArrayType, + ) les = [ - Setup(ntuple(α -> LinRange(lims..., nles + 1), D)...; Re, ArrayType) for - nles in nles + Setup( + ntuple(α -> LinRange(lims[α], nles[α] + 1), D)...; + Re, + boundary_conditions, + ArrayType, + ) for nles in nles ] # Since the grid is uniform and identical for x and y, we may use a specialized # spectral pressure solver - psolver = SpectralPressureSolver(dns) - psolver_les = SpectralPressureSolver.(les) + psolver = PSolver(dns) + psolver_les = PSolver.(les) # Number of time steps to save nt = round(Int, tsim / Δt) @@ -136,11 +147,11 @@ function create_les_data( # datasize = Base.summarysize(filtered) / 1e6 datasize = - (nt ÷ savefreq + 1) * sum(nles .^ D) * 3 * 2 * length(bitstring(zero(T))) / 8 / 1e6 + (nt ÷ savefreq + 1) * sum(prod.(nles)) * D * 2 * length(bitstring(zero(T))) / 8 / 1e6 @info "Generating $datasize Mb of LES data" # Initial conditions - u₀ = random_field(dns, T(0); psolver, ic_params...) + u₀ = icfunc(dns, psolver) # Random body force # force_dns = From 0e11e84a7bc37ff574868bbf22c6c341730e08f5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Fri, 22 Dec 2023 11:08:52 +0100 Subject: [PATCH 221/379] Some fixes --- scratch/train_model.jl | 10 +++++----- src/closures/cnn.jl | 10 ++++++++-- src/closures/create_les_data.jl | 17 ++++++++++------- 3 files changed, 23 insertions(+), 14 deletions(-) diff --git a/scratch/train_model.jl b/scratch/train_model.jl index a03c1734f..15c862ab4 100644 --- a/scratch/train_model.jl +++ b/scratch/train_model.jl @@ -86,7 +86,7 @@ x = ntuple(α -> LinRange(params.lims..., nles + 1), params.D) setup = Setup(x...; params.Re, ArrayType); # Uniform periodic grid -pressure_solver = SpectralPressureSolver(setup); +psolver = SpectralPressureSolver(setup); # Inspect data (; Ip) = setup.grid; @@ -181,7 +181,7 @@ dataloaders = [dataloader] dataloader() # A-posteriori loss -loss = IncompressibleNavierStokes.create_trajectory_loss(; setup, pressure_solver, closure); +loss = IncompressibleNavierStokes.create_trajectory_loss(; setup, psolver, closure); dataloaders = [ IncompressibleNavierStokes.createtrajectoryloader(data_train; device, nunroll = 20) for _ = 1:4 @@ -264,7 +264,7 @@ state_nm, outputs = solve_unsteady( u₀, tlims; Δt, - pressure_solver, + psolver, processors = (; relerr = relerr_trajectory(u, setup), log = timelogger(; nupdate = 1000), @@ -277,7 +277,7 @@ state_cnn, outputs = solve_unsteady( u₀, tlims; Δt, - pressure_solver, + psolver, processors = (relerr = relerr_trajectory(u, setup), log = timelogger(; nupdate = 1)), ) relerr_cnn = outputs.relerr[] @@ -321,7 +321,7 @@ V_nm, outputs_nm = solve_unsteady( energy_history_writer(forcedsetup), step_logger(; nupdate = 10), ), - pressure_solver, + psolver, inplace = false, device, devsetup, diff --git a/src/closures/cnn.jl b/src/closures/cnn.jl index 8cd8ff1b8..53ed71353 100644 --- a/src/closures/cnn.jl +++ b/src/closures/cnn.jl @@ -22,7 +22,7 @@ function cnn(; rng = Random.default_rng(), ) r, c, σ, b = radii, channels, activations, use_bias - (; T, grid) = setup + (; T, grid, boundary_conditions) = setup (; dimension) = grid D = dimension() @@ -43,7 +43,13 @@ function cnn(; collocate, # Add padding so that output has same shape as commutator error - u -> pad_circular(u, sum(r)), + ntuple( + α -> + boundary_conditions[α][1] isa PeriodicBC ? + u -> pad_circular(u, sum(r); dims = α) : + u -> pad_repeat(u, sum(r); dims = α), + D, + ), # Some convolutional layers ( diff --git a/src/closures/create_les_data.jl b/src/closures/create_les_data.jl index 9e75b2674..73349a704 100644 --- a/src/closures/create_les_data.jl +++ b/src/closures/create_les_data.jl @@ -115,24 +115,26 @@ function create_les_data( savefreq = 1, ArrayType = Array, icfunc = (setup, psolver) -> random_field(setup, T(0); psolver), - boundary_conditions = ntuple(α -> (PeriodicBC(), PeriodicBC()), D), + kwargs..., ) - compression = @. ndns ÷ nles - @assert all(@.(compression * nles == ndns)) + compression = [ndns[1] ÷ nles[1] for nles in nles] + for (c, n) in zip(compression, nles), α = 1:D + @assert c * n[α] == ndns[α] + end # Build setup and assemble operators dns = Setup( ntuple(α -> LinRange(lims[α]..., ndns[α] + 1), D)...; Re, - boundary_conditions, ArrayType, + kwargs..., ) les = [ Setup( - ntuple(α -> LinRange(lims[α], nles[α] + 1), D)...; + ntuple(α -> LinRange(lims[α]..., nles[α] + 1), D)...; Re, - boundary_conditions, ArrayType, + kwargs..., ) for nles in nles ] @@ -147,7 +149,8 @@ function create_les_data( # datasize = Base.summarysize(filtered) / 1e6 datasize = - (nt ÷ savefreq + 1) * sum(prod.(nles)) * D * 2 * length(bitstring(zero(T))) / 8 / 1e6 + (nt ÷ savefreq + 1) * sum(prod.(nles)) * D * 2 * length(bitstring(zero(T))) / 8 / + 1e6 @info "Generating $datasize Mb of LES data" # Initial conditions From b1a89f7294db9d32cf11e40bd5f53da6245fe520 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Fri, 22 Dec 2023 11:09:54 +0100 Subject: [PATCH 222/379] Add some sketches --- scratch/Channel.jl | 299 +++++++++++++++++++++++++++++++++++++++ scratch/trajectoryfit.jl | 292 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 591 insertions(+) create mode 100644 scratch/Channel.jl create mode 100644 scratch/trajectoryfit.jl diff --git a/scratch/Channel.jl b/scratch/Channel.jl new file mode 100644 index 000000000..229377a41 --- /dev/null +++ b/scratch/Channel.jl @@ -0,0 +1,299 @@ +# Little LSP hack to get function signatures, go #src +# to definition etc. #src +if isdefined(@__MODULE__, :LanguageServer) #src + include("../src/IncompressibleNavierStokes.jl") #src + using .IncompressibleNavierStokes #src +end #src + +using WGLMakie +# using GLMakie +# using CairoMakie +using IncompressibleNavierStokes +using FFTW +using Adapt + +# Output directory +output = "output/Channel" + +# Floating point type +T = Float64 + +# Array type +ArrayType = Array +## using CUDA; ArrayType = CuArray +## using AMDGPU; ArrayType = ROCArray +## using oneAPI; ArrayType = oneArray +## using Metal; ArrayType = MtlArray + +using CUDA; +T = Float32; +ArrayType = CuArray; +CUDA.allowscalar(false); +device = x -> adapt(CuArray{T}, x) + +set_theme!(; GLMakie = (; scalefactor = 1.5)) + +# Reynolds number +Re = T(4_000) + +# Boundary conditions +boundary_conditions = ( + ## x left, x right + (PeriodicBC(), PeriodicBC()), + + ## y rear, y front + (DirichletBC(), DirichletBC()), +) + +getdir(x = rand(T)) = T[cospi(2x), sinpi(2x)] +createbodyforce((Ax, Ay), (a, b), r) = function bodyforce(dim, x, y, t) + A = Ax * (dim() == 1) + Ay * (dim() == 2) + R = (x - a)^2 / r^2 + (y - b)^2 / r^2 + A * max(0, 1 - R) +end +createmanyforce(forces...) = function (dim, x, y, t) + out = zero(x) + for f in forces + out += f(dim, x, y, t) + end + out +end + +bodyforce = createmanyforce( + createbodyforce(2 * getdir(+0.00), T[1.0, +0.0], T(0.6)), + createbodyforce(4 * getdir(-0.10), T[2.0, +0.5], T(0.2)), + createbodyforce(3 * getdir(+0.05), T[3.5, -0.3], T(0.3)), + createbodyforce(4 * getdir(-0.05), T[4.5, +0.1], T(0.2)), + createbodyforce(4 * getdir(-0.45), T[6.5, +0.5], T(0.2)), + createbodyforce(3 * getdir(+0.05), T[7.5, +0.0], T(0.3)), + createbodyforce(4 * getdir(-0.50), T[9.0, -0.5], T(0.2)), +) + +bodyforce = createmanyforce( + createbodyforce(4 * getdir(+0.10), T[1.0, +0.4], T(0.3)), + createbodyforce(3 * getdir(-0.45), T[2.3, -0.1], T(0.2)), + createbodyforce(4 * getdir(+0.05), T[3.0, +0.2], T(0.2)), + createbodyforce(2 * getdir(-0.02), T[5.0, -0.1], T(0.5)), + createbodyforce(4 * getdir(-0.45), T[7.0, +0.4], T(0.2)), + createbodyforce(3 * getdir(+0.05), T[7.5, -0.5], T(0.3)), + createbodyforce(4 * getdir(+0.00), T[9.0, +0.0], T(0.2)), +) + +# A 2D grid is a Cartesian product of two vectors. Here we refine the grid near +# the walls. +n = 64 +# n = 128 +x = LinRange(T(0), T(10), 8n + 1) +y = LinRange(-T(1), T(1), 2n + 1) +plotgrid(x, y) + +bodyforce.([IncompressibleNavierStokes.Dimension(1)], x, y', T(0)) |> heatmap +bodyforce.([IncompressibleNavierStokes.Dimension(2)], x, y', T(0)) |> heatmap + +# Build setup and assemble operators +setup = Setup(x, y; Re, bodyforce, boundary_conditions, ArrayType); + +psolver = DirectPressureSolver(setup); + +# Initial conditions +u₀ = create_initial_conditions( + setup, + (dim, x, y) -> (dim() == 1) * 3 * (1 + y) * (1 - y); + # (dim, x, y) -> zero(x); + psolver, +); +u = u₀; + +# Solve unsteady problem +state, outputs = solve_unsteady( + setup, + # u₀, + u, + (T(0), T(4.0)); + Δt = T(0.005), + psolver, + processors = ( + rtp = realtimeplotter(; + setup, + # type = contourf, + # plot = fieldplot, + # fieldname = :velocity, + # plot = energy_history_plot, + ## plot = energy_spectrum_plot, + nupdate = 1, + size = (1200, 600), + docolorbar = false, + ), + ## anim = animator(; setup, path = "$output/vorticity.mkv", nupdate = 20), + ## vtk = vtk_writer(; setup, nupdate = 10, dir = output, filename = "solution"), + ## field = fieldsaver(; setup, nupdate = 10), + log = timelogger(; nupdate = 1), + ), +); +(; u) = state; + +using CairoMakie +GLMakie.activate!() +CairoMakie.activate!() + +# Plot pressure +fieldplot( + state; + setup, + psolver, + # type = contourf, + # fieldname = :pressure, + fieldname = :velocity, + # fieldname = :vorticity, + docolorbar = false, + # size = (1200, 350), + # size = (900, 250), + size = (650, 200), +) + +name = "vorticity_10.pdf" +name = "velocity_10.pdf" +save(name, current_figure()) +run(`mv $name ../SupervisedClosure/figures/channel/`) + +(; Ip, Iu, xp) = setup.grid + +fig = with_theme(; palette = (; color = ["#3366cc", "#cc0000", "#669900", "#ffcc00"])) do + fig = Figure() + ax = Axis(fig[1, 1]; xlabel = "x") + lines!(ax, Array(xp[1][2:end-1]), u[2][Iu[2]][:, 32] |> Array) + fig +end +name = "midline_10.pdf" + +IncompressibleNavierStokes.kinetic_energy(u, setup)[Ip] |> Array |> heatmap +sum(IncompressibleNavierStokes.kinetic_energy(u, setup)[Ip]; dims = 2) |> lines +sum(IncompressibleNavierStokes.kinetic_energy(u, setup)[Ip]; dims = 1)[:] |> Array |> lines + +e = IncompressibleNavierStokes.kinetic_energy(u, setup)[Ip] +e[:, 40] + +ehat = fft(e[:, n])[1:5n] |> x -> abs.(x) |> Array +ehat2 = fft(e[:, 60])[1:5n] |> x -> abs.(x) |> Array +lines(ehat; axis = (; xscale = log10, yscale = log10)) +lines!(ehat2) + +fig = Figure() +Axis(fig[1, 1]) +u[2][Iu[2]][:, 32] |> Array |> lines! +# u[2][Iu[2]][:, 40] |> Array |> lines! + +fig = Figure() +Axis(fig[1, 1]) +u[1][Iu[1]][20, :] |> Array |> lines! +u[1][Iu[1]][40, :] |> Array |> lines! +u[1][Iu[1]][300, :] |> Array |> lines! + +# ## Post-process +# +# We may visualize or export the computed fields + +# Export to VTK +save_vtk(setup, state.u, state.p, "$output/solution") + +Makie.available_gradients() + +####################################################################### + +bodyforce_train = createmanyforce( + createbodyforce(2 * getdir(+0.00), T[1.0, +0.0], T(0.6)), + createbodyforce(4 * getdir(-0.10), T[2.0, +0.5], T(0.2)), + createbodyforce(3 * getdir(+0.05), T[3.5, -0.3], T(0.3)), + createbodyforce(4 * getdir(-0.05), T[4.5, +0.1], T(0.2)), + createbodyforce(4 * getdir(-0.45), T[6.5, +0.5], T(0.2)), + createbodyforce(3 * getdir(+0.05), T[7.5, +0.0], T(0.3)), + createbodyforce(4 * getdir(-0.50), T[9.0, -0.5], T(0.2)), +) + +bodyforce_test = createmanyforce( + createbodyforce(4 * getdir(+0.10), T[1.0, +0.4], T(0.3)), + createbodyforce(3 * getdir(-0.45), T[2.3, -0.1], T(0.2)), + createbodyforce(4 * getdir(+0.05), T[3.0, +0.2], T(0.2)), + createbodyforce(2 * getdir(-0.02), T[5.0, -0.1], T(0.5)), + createbodyforce(4 * getdir(-0.45), T[7.0, +0.4], T(0.2)), + createbodyforce(3 * getdir(+0.05), T[7.5, -0.5], T(0.3)), + createbodyforce(4 * getdir(+0.00), T[9.0, +0.0], T(0.2)), +) + +# Parameters +# nles = 50 +# nles = 64 +# nles = 128 +# ndns = 200 +nles = 32 +ndns = 128 +params = (; + D = 2, + Re = T(4_000), + lims = ((T(0), T(10)), (T(-1), T(1))), + nles = [(8nles, 2nles)], + ndns = (8ndns, 2ndns), + # tburn = T(0.1), + tsim = T(10), + Δt = T(1e-3), + savefreq = 5, + ArrayType, + boundary_conditions = ((PeriodicBC(), PeriodicBC()), (DirichletBC(), DirichletBC())), + PSolver = DirectPressureSolver, + icfunc = (setup, psolver) -> create_initial_conditions( + setup, + (dim, x, y) -> (dim() == 1) * 3 * (1 + y) * (1 - y); + # (dim, x, y) -> zero(x); + psolver, + ), +) + +# Create LES data from DNS +data_train = [create_les_data(T; params..., bodyforce = bodyforce_train) for _ = 1:1]; +data_test = [create_les_data(T; params..., bodyforce = bodyforce_test) for _ = 1:1]; + +data_train[1].u[1][1][1] + +# Build LES setup and assemble operators +x = ntuple(α -> LinRange(params.lims[α]..., params.nles[1][α] + 1), params.D) +setup = Setup(x...; params.boundary_conditions, params.Re, ArrayType); + +# Uniform periodic grid +psolver = params.PSolver(setup); + +# Inspect data +field = data_train[1].u[1]; + +u = device(field[1]) +# u = device(field[201]) +# u = device(field[1201]) +# u = device(field[2001]) +o = Observable((; u, t = nothing)) +fieldplot( + o; + setup, + # fieldname = :velocity, + # fieldname = 2, +) + +# energy_spectrum_plot(o; setup) +for i = 1:length(field) + o[] = (; o[]..., u = device(field[i])) + sleep(0.001) +end + +io_train = create_io_arrays(data_train, [setup]); +io_test = create_io_arrays(data_test, [setup]); + +closure, θ₀ = cnn(; + setup, + radii = [2, 2, 2, 2], + channels = [5, 5, 5, params.D], + activations = [leakyrelu, leakyrelu, leakyrelu, identity], + use_bias = [true, true, true, false], + rng, +); +closure.chain + +sample = io_train[1].u[:, :, :, 1:5] +closure(sample, θ₀) |> size diff --git a/scratch/trajectoryfit.jl b/scratch/trajectoryfit.jl new file mode 100644 index 000000000..0dd117376 --- /dev/null +++ b/scratch/trajectoryfit.jl @@ -0,0 +1,292 @@ +# Little LSP hack to get function signatures, go #src +# to definition etc. #src +if isdefined(@__MODULE__, :LanguageServer) #src + include("../src/IncompressibleNavierStokes.jl") #src + using .IncompressibleNavierStokes #src +end #src + +# # Train closure model +# +# Here, we consider a periodic box ``[0, 1]^2``. It is discretized with a +# uniform Cartesian grid with square cells. + +using Adapt +using GLMakie +using IncompressibleNavierStokes +using JLD2 +using LinearAlgebra +using Lux +using NNlib +using Optimisers +using Random +using Zygote + +# Random number generator +rng = Random.default_rng() +Random.seed!(rng, 123) + +set_theme!(; GLMakie = (; scalefactor = 1.5)) + +# Floating point precision +T = Float64 + +# Array type +ArrayType = Array +device = identity +## using CUDA; ArrayType = CuArray +## using AMDGPU; ArrayType = ROCArray +## using oneAPI; ArrayType = oneArray +## using Metal; ArrayType = MtlArray + +using LuxCUDA +using CUDA; +T = Float32; +# T = Float64; +ArrayType = CuArray; +CUDA.allowscalar(false); +# device = cu +device = x -> adapt(CuArray{T}, x) + +# Parameters +# nles = 50 +# nles = 64 +# nles = 128 +# ndns = 200 +nles = 256 +params = (; + D = 2, + Re = T(6_000), + lims = (T(0), T(1)), + nles = [nles], + # ndns = 512, + # ndns = 1024, + ndns = 2048, + tburn = T(0.05), + tsim = T(0.5), + Δt = T(1e-4), + savefreq = 5, + ArrayType, +) + +# Create LES data from DNS +data_train = [create_les_data(T; params...) for _ = 1:3]; +data_valid = [create_les_data(T; params...) for _ = 1:1]; +data_test = [create_les_data(T; params..., tsim = T(0.2)) for _ = 1:1]; + +data_test[1].u[1][1][1] + +# # Save filtered DNS data +# jldsave("output/forced/data.jld2"; data_train, data_valid, data_test) + +# # Load previous LES data +# data_train, data_valid, data_test = load("output/forced/data.jld2", "data_train", "data_valid", "data_test") + +# Build LES setup and assemble operators +x = ntuple(α -> LinRange(params.lims..., nles + 1), params.D) +setup = Setup(x...; params.Re, ArrayType); + +# Uniform periodic grid +pressure_solver = SpectralPressureSolver(setup); + +# Inspect data +(; Ip) = setup.grid; +field = data_train[1].u[1]; +α = 2 +# j = 13 +o = Observable(field[1][α][Ip]) +# o = Observable(field[1][α][:, :, j]) +heatmap(o) +for i = 1:length(field) + o[] = field[i][α][Ip] + # o[] = field[i][α][:, :, j] + sleep(0.001) +end + +# Inspect data +field = data_train[1].u[1]; +u = device(field[1]) +o = Observable((; u, t = nothing)) +fieldplot( + o; + setup, + # fieldname = :velocity, + # fieldname = 2, +) +# energy_spectrum_plot(o; setup) +for i = 1:length(field) + o[] = (; o[]..., u = device(field[i])) + sleep(0.001) +end + +# Create input/output arrays +io_train = create_io_arrays(data_train, [setup]); +io_valid = create_io_arrays(data_valid, [setup]); +io_test = create_io_arrays(data_test, [setup]); + +size(io_train[1].u) +size(io_valid[1].u) +size(io_test[1].u) + +Base.summarysize(io_train) / 1e9 + +closure, θ₀ = cnn(; + setup, + radii = [2, 2, 2, 2], + channels = [5, 5, 5, params.D], + activations = [leakyrelu, leakyrelu, leakyrelu, identity], + use_bias = [true, true, true, false], + rng, +); +closure.chain + +sample = io_train[1].u[:, :, :, 1:5] +closure(sample, θ₀) |> size + +θ₀.layer_5 +θ₀.layer_6 + +# closure, θ₀ = fno(; +# setup, +# kmax = [20, 20, 20, 20], +# c = [24, 12, 8, 8], +# # σ = [gelu, gelu, gelu, identity], +# σ = [leakyrelu, leakyrelu, leakyrelu, identity], +# # σ = [tanh, tanh, tanh, identity], +# # ψ = gelu, +# ψ = tanh, +# rng, +# ); +# +# closure.chain +# +# θ₀.layer_4.spectral_weights |> size + +# θ₀ +# θ₀ = 2*θ₀ +# θ₀.layer_6 ./= 20 + +# Prepare training +θ = T(1.0e-1) * device(θ₀); +# θ = device(θ₀); +# θ = 2 * device(θ₀); +opt = Optimisers.setup(Adam(T(1.0e-2)), θ); +callbackstate = Point2f[]; +it = rand(1:size(io_valid[1].u, 4), 50); +validset = map(v -> v[:, :, :, it], io_valid[1]); + +# A-priori loss +loss = createloss(mean_squared_error, closure); +dataloader = createdataloader(io_train[1]; batchsize = 50, device); +dataloaders = [dataloader] +dataloader() + +# A-posteriori loss +loss = IncompressibleNavierStokes.create_trajectory_loss(; setup, pressure_solver, closure); +dataloaders = [ + IncompressibleNavierStokes.createtrajectoryloader(data_train; device, nunroll = 20) + for _ = 1:4 +]; +loss(dataloaders[1](), device(θ₀)) + +# Warm-up +loss(dataloaders[1](), θ) +@time loss(dataloaders[1](), θ); +b = dataloaders[1](); +first(gradient(θ -> loss(b, θ), θ)); +@time first(gradient(θ -> loss(b, θ), θ)); +GC.gc() +CUDA.reclaim() + +map() do + i = 3 + # h = 1000 * sqrt(eps(T)) + h = cbrt(eps(T)) + θ1 = copy(θ) + θ2 = copy(θ) + CUDA.@allowscalar θ1[i] -= h / 2 + CUDA.@allowscalar θ2[i] += h / 2 + b = dataloaders[1]() + @show loss(b, θ2) loss(b, θ1) + a = (loss(b, θ2) - loss(b, θ1)) / h + b = CUDA.@allowscalar first(gradient(θ -> loss(b, θ), θ))[i] + [a; b] +end + +# Training +# Note: The states `opt`, `θ`, and `callbackstate` +# will not be overwritten until training is finished. +# This allows for cancelling with "Control-C" should errors explode. +(; opt, θ, callbackstate) = train( + dataloaders, + loss, + opt, + θ; + niter = 1000, + ncallback = 10, + callbackstate, + callback = create_callback(closure, device(validset)...; state = callbackstate), +); +GC.gc() +CUDA.reclaim() + +Array(θ) + +# # Save trained parameters +# jldsave("output/forced/theta_cnn.jld2"; theta = Array(θ)) +# jldsave("output/forced/theta_fno.jld2"; theta = Array(θ)) + +# # Load trained parameters +# θθ = load("output/theta_cnn.jld2") +# θθ = load("output/theta_fno.jld2") +# copyto!(θ, θθ["theta"]) + +function relerr(u, uref, setup) + (; dimension, Ip) = setup.grid + D = dimension() + a, b = T(0), T(0) + for α = 1:D + a += sum(abs2, u[α][Ip] - uref[α][Ip]) + b += sum(abs2, uref[α][Ip]) + end + sqrt(a) / sqrt(b) +end + +u, u₀ = nothing, nothing +u = device.(data_test[1].u[1]); +u₀ = device(data_test[1].u[1][1]); +Δt = data_test[1].t[2] - data_test[1].t[1] +tlims = extrema(data_test[1].t) +length(u) +length(data_test[1].t) + +state_nm, outputs = solve_unsteady( + setup, + u₀, + tlims; + Δt, + pressure_solver, + processors = (; + relerr = relerr_trajectory(u, setup), + log = timelogger(; nupdate = 1000), + ), +) +relerr_nm = outputs.relerr[] + +state_cnn, outputs = solve_unsteady( + (; setup..., closure_model = wrappedclosure(closure, θ, setup)), + u₀, + tlims; + Δt, + pressure_solver, + processors = (relerr = relerr_trajectory(u, setup), log = timelogger(; nupdate = 1)), +) +relerr_cnn = outputs.relerr[] + +relerr_nm +relerr_cnn + +# dnm = relerr_nm +# dcnn = relerr_cnn + +dnm +dcnn From 8e61bcce772af6e4ddf32bc45eae49022c0a47b0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Tue, 9 Jan 2024 13:46:19 +0100 Subject: [PATCH 223/379] Add volume averaging filter --- docs/src/features/closure.md | 1 + src/filter.jl | 39 ++++++++++++++++++++++++++++++++++-- 2 files changed, 38 insertions(+), 2 deletions(-) diff --git a/docs/src/features/closure.md b/docs/src/features/closure.md index 6ff6016ec..a901fc6c3 100644 --- a/docs/src/features/closure.md +++ b/docs/src/features/closure.md @@ -40,6 +40,7 @@ M \bar{v} & = 0, \\ ```@docs face_average! +volume_average! create_les_data create_io_arrays ``` diff --git a/src/filter.jl b/src/filter.jl index 38b577e1a..19e47f02b 100644 --- a/src/filter.jl +++ b/src/filter.jl @@ -1,7 +1,7 @@ """ face_average!(v, u, setup, comp) -Average `u` over volume faces. Put result in `v`. +Average fine grid `u` over coarse volume face. Put result in `v`. """ function face_average!(v, u, setup_les, comp) (; grid, workgroupsize) = setup_les @@ -24,7 +24,6 @@ function face_average!(v, u, setup_les, comp) face = CartesianIndices(ntuple(β -> β == α ? (comp:comp) : (1:comp), D)) Φ!(get_backend(v[1]), workgroupsize)(v, u, Val(α), face, I0; ndrange) end - # synchronize(get_backend(u[1])) v end @@ -34,3 +33,39 @@ face_average(u, setup_les, comp) = face_average!( setup_les, comp, ) + +""" + volume_average!(v, u, setup, comp) + +Average fine grid `u` over coarse volume. Put result in `v`. +""" +function volume_average!(v, u, setup_les, comp) + (; grid, workgroupsize) = setup_les + (; Nu, Iu) = grid + D = length(u) + δ = Offset{D}() + @kernel function Φ!(v, u, ::Val{α}, volume, I0) where {α} + I = @index(Global, Cartesian) + J = I0 + comp * (I - oneunit(I)) + s = zero(eltype(v[α])) + for i in volume + s += u[α][J+i] + end + v[α][I0+I] = s / comp^D + end + for α = 1:D + ndrange = Nu[α] + I0 = first(Iu[α]) + I0 -= oneunit(I0) + volume = CartesianIndices(ntuple(β -> 1:comp, D)) + Φ!(get_backend(v[1]), workgroupsize)(v, u, Val(α), volume, I0; ndrange) + end + v +end + +volume_average(u, setup_les, comp) = volume_average!( + ntuple(α -> similar(u[1], setup_les.grid.N), length(u)), + u, + setup_les, + comp, +) From d37206f5514e651a83cf38165f6a5a6044b77f04 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Tue, 16 Jan 2024 16:58:09 +0100 Subject: [PATCH 224/379] fix: Volume average --- src/filter.jl | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/src/filter.jl b/src/filter.jl index 19e47f02b..090bb5d8e 100644 --- a/src/filter.jl +++ b/src/filter.jl @@ -40,24 +40,37 @@ face_average(u, setup_les, comp) = face_average!( Average fine grid `u` over coarse volume. Put result in `v`. """ function volume_average!(v, u, setup_les, comp) - (; grid, workgroupsize) = setup_les - (; Nu, Iu) = grid + (; grid, boundary_conditions, workgroupsize) = setup_les + (; N, Nu, Iu) = grid D = length(u) δ = Offset{D}() + @assert all(bc -> bc[1] isa PeriodicBC && bc[2] isa PeriodicBC, boundary_conditions) @kernel function Φ!(v, u, ::Val{α}, volume, I0) where {α} I = @index(Global, Cartesian) J = I0 + comp * (I - oneunit(I)) s = zero(eltype(v[α])) for i in volume - s += u[α][J+i] + # Periodic extension + K = J+i + K = mod1.(K.I, comp .* (N .- 2)) + K = CartesianIndex(K) + s += u[α][K] end - v[α][I0+I] = s / comp^D + n = (iseven(comp) ? comp : comp + 1) * comp^(D - 1) + v[α][I0+I] = s / n end for α = 1:D ndrange = Nu[α] I0 = first(Iu[α]) I0 -= oneunit(I0) - volume = CartesianIndices(ntuple(β -> 1:comp, D)) + volume = CartesianIndices( + ntuple( + β -> + α == β ? iseven(comp) ? (comp÷2:comp÷2+comp) + : (comp+1:2comp+1) : (1:comp), + D, + ), + ) Φ!(get_backend(v[1]), workgroupsize)(v, u, Val(α), volume, I0; ndrange) end v From 025ff5e7ffa8d0a65c9c4ca525f4441dbd6cbb59 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Tue, 16 Jan 2024 17:03:41 +0100 Subject: [PATCH 225/379] Add script --- scratch/divergence.jl | 78 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 78 insertions(+) create mode 100644 scratch/divergence.jl diff --git a/scratch/divergence.jl b/scratch/divergence.jl new file mode 100644 index 000000000..1de7a27ea --- /dev/null +++ b/scratch/divergence.jl @@ -0,0 +1,78 @@ +# Little LSP hack to get function signatures, go #src +# to definition etc. #src +if isdefined(@__MODULE__, :LanguageServer) #src + include("../src/IncompressibleNavierStokes.jl") #src + using .IncompressibleNavierStokes #src +end #src + +#md using CairoMakie +using GLMakie #!md +using IncompressibleNavierStokes +using LinearAlgebra + +# Output directory +output = "output/DecayingTurbulence2D" + +# Floating point precision +T = Float64 + +# Array type +ArrayType = Array +## using CUDA; ArrayType = CuArray +## using AMDGPU; ArrayType = ROCArray +## using oneAPI; ArrayType = oneArray +## using Metal; ArrayType = MtlArray + +# Viscosity model +Re = T(10_000) + +# A 2D grid is a Cartesian product of two vectors +nles = 256 +comp = 2 +ndns = nles * comp +lims = T(0), T(1) +D = 2 +dns = Setup(ntuple(α -> LinRange(lims..., ndns + 1), D)...; Re, ArrayType); +les = Setup(ntuple(α -> LinRange(lims..., nles + 1), D)...; Re, ArrayType); + +# Since the grid is uniform and identical for x and y, we may use a specialized +# spectral pressure solver +psolver_dns = SpectralPressureSolver(dns); +psolver_les = SpectralPressureSolver(les); + +# Create random initial conditions +u₀ = random_field(dns, T(0); psolver); + +# Solve unsteady problem +state, outputs = solve_unsteady( + dns, + u₀, + (T(0), T(0.1)); + Δt = T(1e-3), + psolver_dns, + processors = ( + rtp = realtimeplotter(; setup = dns, nupdate = 1), + # ehist = realtimeplotter(; + # setup, + # plot = energy_history_plot, + # nupdate = 10, + # displayfig = false, + # ), + # espec = realtimeplotter(; setup, plot = energy_spectrum_plot, nupdate = 10), + # anim = animator(; setup, path = "$output/solution.mkv", nupdate = 20), + # vtk = vtk_writer(; setup, nupdate = 10, dir = output, filename = "solution"), + # field = fieldsaver(; setup, nupdate = 10), + log = timelogger(; nupdate = 100), + ), +); +(; u, t) = state; + +ubar = IncompressibleNavierStokes.face_average(u, les, comp); +ubar = IncompressibleNavierStokes.volume_average(u, les, comp); +fieldplot((; u = ubar, t); setup = les) + +IncompressibleNavierStokes.apply_bc_u!(ubar, t, les) +div = IncompressibleNavierStokes.divergence(ubar, les)[les.grid.Ip] + +norm(div) +norm(ubar[1][les.grid.Iu[1]]) From 90ae4ac1b46facaa8e910bba734b80733a399548 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Thu, 18 Jan 2024 17:03:53 +0100 Subject: [PATCH 226/379] Create filter structs --- src/IncompressibleNavierStokes.jl | 2 ++ src/closures/create_les_data.jl | 14 ++++++------ src/filter.jl | 36 +++++++++++++------------------ 3 files changed, 25 insertions(+), 27 deletions(-) diff --git a/src/IncompressibleNavierStokes.jl b/src/IncompressibleNavierStokes.jl index 908079a0c..a41a64781 100644 --- a/src/IncompressibleNavierStokes.jl +++ b/src/IncompressibleNavierStokes.jl @@ -130,6 +130,8 @@ export mean_squared_error, relative_error, relerr_trajectory export createloss, createdataloader, create_callback, create_les_data, create_io_arrays export wrappedclosure +export FaceAverage, VolumeAverage + # ODE methods export AdamsBashforthCrankNicolsonMethod, OneLegMethod diff --git a/src/closures/create_les_data.jl b/src/closures/create_les_data.jl index 73349a704..f97aa5aab 100644 --- a/src/closures/create_les_data.jl +++ b/src/closures/create_les_data.jl @@ -32,8 +32,8 @@ function gaussian_force( force end -function lesdatagen(dnsobs, les, compression, psolver) - Φu = zero.(face_average(dnsobs[].u, les, compression)) +function lesdatagen(dnsobs, les, Φ, compression, psolver) + Φu = zero.(Φ(dnsobs[].u, les, compression)) p = zero(Φu[1]) div = zero(p) ΦF = zero.(Φu) @@ -41,9 +41,9 @@ function lesdatagen(dnsobs, les, compression, psolver) c = zero.(Φu) results = (; u = fill(Array.(dnsobs[].u), 0), c = fill(Array.(dnsobs[].u), 0)) on(dnsobs) do (; u, F, t) - face_average!(Φu, u, les, compression) + Φ(Φu, u, les, compression) apply_bc_u!(Φu, t, les) - face_average!(ΦF, F, les, compression) + Φ(ΦF, F, les, compression) momentum!(FΦ, Φu, t, les) apply_bc_u!(FΦ, t, les; dudt = true) project!(FΦ, les; psolver, div, p) @@ -56,7 +56,7 @@ function lesdatagen(dnsobs, les, compression, psolver) results end -filtersaver(dns, les, compression, psolver_dns, psolver_les; nupdate = 1) = +filtersaver(dns, les, Φ, compression, psolver_dns, psolver_les; nupdate = 1) = processor() do state (; dimension, x) = dns.grid T = eltype(x[1]) @@ -66,7 +66,7 @@ filtersaver(dns, les, compression, psolver_dns, psolver_les; nupdate = 1) = p = zero(state[].u[1]) dnsobs = Observable((; state[].u, F, state[].t)) data = [ - lesdatagen(dnsobs, les[i], compression[i], psolver_les[i]) for i = 1:length(les) + lesdatagen(dnsobs, les[i], Φ[i], compression[i], psolver_les[i]) for i = 1:length(les) ] results = (; t = fill(zero(eltype(x[1])), 0), @@ -115,6 +115,7 @@ function create_les_data( savefreq = 1, ArrayType = Array, icfunc = (setup, psolver) -> random_field(setup, T(0); psolver), + Φ = map(() -> FaceAverage(), nles), kwargs..., ) compression = [ndns[1] ÷ nles[1] for nles in nles] @@ -184,6 +185,7 @@ function create_les_data( f = filtersaver( _dns, _les, + Φ, compression, psolver, psolver_les; diff --git a/src/filter.jl b/src/filter.jl index 090bb5d8e..ae85711c1 100644 --- a/src/filter.jl +++ b/src/filter.jl @@ -1,9 +1,17 @@ +abstract type AbstractFilter end + +struct FaceAverage <: AbstractFilter end +struct VolumeAverage <: AbstractFilter end + +(Φ::AbstractFilter)(u, setup_les, compression) = + Φ(ntuple(α -> similar(u[1], setup_les.grid.N), length(u)), u, setup_les, compression) + """ - face_average!(v, u, setup, comp) + (::FaceAverage)(v, u, setup_les) Average fine grid `u` over coarse volume face. Put result in `v`. """ -function face_average!(v, u, setup_les, comp) +function (::FaceAverage)(v, u, setup_les, comp) (; grid, workgroupsize) = setup_les (; Nu, Iu) = grid D = length(u) @@ -27,19 +35,12 @@ function face_average!(v, u, setup_les, comp) v end -face_average(u, setup_les, comp) = face_average!( - ntuple(α -> similar(u[1], setup_les.grid.N), length(u)), - u, - setup_les, - comp, -) - """ - volume_average!(v, u, setup, comp) + (::VolumeAverage)(v, u, setup_les, comp) Average fine grid `u` over coarse volume. Put result in `v`. """ -function volume_average!(v, u, setup_les, comp) +function (::VolumeAverage)(v, u, setup_les, comp) (; grid, boundary_conditions, workgroupsize) = setup_les (; N, Nu, Iu) = grid D = length(u) @@ -51,7 +52,7 @@ function volume_average!(v, u, setup_les, comp) s = zero(eltype(v[α])) for i in volume # Periodic extension - K = J+i + K = J + i K = mod1.(K.I, comp .* (N .- 2)) K = CartesianIndex(K) s += u[α][K] @@ -66,8 +67,8 @@ function volume_average!(v, u, setup_les, comp) volume = CartesianIndices( ntuple( β -> - α == β ? iseven(comp) ? (comp÷2:comp÷2+comp) - : (comp+1:2comp+1) : (1:comp), + α == β ? iseven(comp) ? (comp÷2:comp÷2+comp) : (comp+1:2comp+1) : + (1:comp), D, ), ) @@ -75,10 +76,3 @@ function volume_average!(v, u, setup_les, comp) end v end - -volume_average(u, setup_les, comp) = volume_average!( - ntuple(α -> similar(u[1], setup_les.grid.N), length(u)), - u, - setup_les, - comp, -) From da341d4feddc350dd6ff1c066f70a2b676a589b1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Thu, 18 Jan 2024 17:04:33 +0100 Subject: [PATCH 227/379] Update script --- scratch/divergence.jl | 151 ++++++++++++++++++++++++++++++++++++++---- 1 file changed, 139 insertions(+), 12 deletions(-) diff --git a/scratch/divergence.jl b/scratch/divergence.jl index 1de7a27ea..76e6dabae 100644 --- a/scratch/divergence.jl +++ b/scratch/divergence.jl @@ -6,9 +6,11 @@ if isdefined(@__MODULE__, :LanguageServer) #src end #src #md using CairoMakie -using GLMakie #!md +using GLMakie using IncompressibleNavierStokes +using IncompressibleNavierStokes: momentum!, divergence!, project!, apply_bc_u! using LinearAlgebra +using Printf # Output directory output = "output/DecayingTurbulence2D" @@ -23,35 +25,129 @@ ArrayType = Array ## using oneAPI; ArrayType = oneArray ## using Metal; ArrayType = MtlArray +function observe_v(dnsobs, Φ, les, compression, psolver) + (; ArrayType, grid) = les + (; dimension, N, Iu, Ip) = grid + D = dimension() + Mα = N[1] - 2 + v = zero.(Φ(dnsobs[].u, les, compression)) + Pv = zero.(v) + p = zero(v[1]) + div = zero(p) + ΦPF = zero.(v) + PFΦ = zero.(v) + c = zero.(v) + T = eltype(v[1]) + results = (; + Φ, + Mα, + t = zeros(T, 0), + Dv = zeros(T, 0), + Pv = zeros(T, 0), + Pc = zeros(T, 0), + c = zeros(T, 0), + ) + on(dnsobs) do (; u, PF, t) + push!(results.t, t) + + Φ(v, u, les, compression) + apply_bc_u!(v, t, les) + Φ(ΦPF, PF, les, compression) + momentum!(PFΦ, v, t, les) + apply_bc_u!(PFΦ, t, les; dudt = true) + project!(PFΦ, les; psolver, div, p) + foreach(α -> c[α] .= ΦPF[α] .- PFΦ[α], 1:D) + apply_bc_u!(c, t, les) + divergence!(div, v, les) + norm_Du = norm(div[Ip]) + norm_v = sqrt(sum(α -> sum(abs2, v[α][Iu[α]]), 1:D)) + push!(results.Dv, norm_Du / norm_v) + + copyto!.(Pv, v) + project!(Pv, les; psolver, div, p) + foreach(α -> Pv[α] .= Pv[α] .- v[α], 1:D) + norm_vmPv = sqrt(sum(α -> sum(abs2, Pv[α][Iu[α]]), 1:D)) + push!(results.Pv, norm_vmPv / norm_v) + + Pc = Pv + copyto!.(Pc, c) + project!(Pc, les; psolver, div, p) + foreach(α -> Pc[α] .= Pc[α] .- c[α], 1:D) + norm_cmPc = sqrt(sum(α -> sum(abs2, Pc[α][Iu[α]]), 1:D)) + norm_c = sqrt(sum(α -> sum(abs2, c[α][Iu[α]]), 1:D)) + push!(results.Pc, norm_cmPc / norm_c) + + norm_ΦPF = sqrt(sum(α -> sum(abs2, ΦPF[α][Iu[α]]), 1:D)) + push!(results.c, norm_c / norm_ΦPF) + end + results +end + +observe_u(dns, psolver_dns, filters; nupdate = 1) = + processor() do state + (; dimension, x) = dns.grid + T = eltype(x[1]) + D = dimension() + PF = zero.(state[].u) + div = zero(state[].u[1]) + p = zero(state[].u[1]) + dnsobs = Observable((; state[].u, PF, state[].t)) + results = [ + observe_v(dnsobs, Φ, setup, compression, psolver) for + (; setup, Φ, compression, psolver) in filters + ] + on(state) do (; u, t, n) + n % nupdate == 0 || return + apply_bc_u!(u, t, dns) + momentum!(PF, u, t, dns) + apply_bc_u!(PF, t, dns; dudt = true) + project!(PF, dns; psolver = psolver_dns, div, p) + dnsobs[] = (; u, PF, t) + end + # state[] = state[] # Save initial conditions + results + end + # Viscosity model Re = T(10_000) # A 2D grid is a Cartesian product of two vectors -nles = 256 -comp = 2 -ndns = nles * comp +ndns = 1024 lims = T(0), T(1) D = 2 dns = Setup(ntuple(α -> LinRange(lims..., ndns + 1), D)...; Re, ArrayType); -les = Setup(ntuple(α -> LinRange(lims..., nles + 1), D)...; Re, ArrayType); + +filters = map([ + (FaceAverage(), 64), + (FaceAverage(), 128), + (FaceAverage(), 256), + (VolumeAverage(), 64), + (VolumeAverage(), 128), + (VolumeAverage(), 256), +]) do (Φ, nles) + compression = ndns ÷ nles + setup = Setup(ntuple(α -> LinRange(lims..., nles + 1), D)...; Re, ArrayType) + psolver = SpectralPressureSolver(setup) + (; setup, Φ, compression, psolver) +end; # Since the grid is uniform and identical for x and y, we may use a specialized # spectral pressure solver psolver_dns = SpectralPressureSolver(dns); -psolver_les = SpectralPressureSolver(les); # Create random initial conditions -u₀ = random_field(dns, T(0); psolver); +u₀ = random_field(dns, T(0); psolver = psolver_dns); # Solve unsteady problem state, outputs = solve_unsteady( dns, u₀, - (T(0), T(0.1)); - Δt = T(1e-3), - psolver_dns, + (T(0), T(0.01)); + Δt = T(1e-4), + psolver = psolver_dns, processors = ( rtp = realtimeplotter(; setup = dns, nupdate = 1), + obs = observe_u(dns, psolver_dns, filters), # ehist = realtimeplotter(; # setup, # plot = energy_history_plot, @@ -67,9 +163,40 @@ state, outputs = solve_unsteady( ); (; u, t) = state; -ubar = IncompressibleNavierStokes.face_average(u, les, comp); -ubar = IncompressibleNavierStokes.volume_average(u, les, comp); +begin + println("Φ\t\tM\tDu\tPv\tPc\tc") + for o in outputs.obs + nt = length(o.t) + Dv = sum(o.Dv) / nt + Pc = sum(o.Pc) / nt + Pv = sum(o.Pv) / nt + c = sum(o.c) / nt + @printf( + # "%s\t%d^%d\t%.2g\t%.2g\t%.2g\t%.2g\n", + "%s &\t\$%d^%d\$ &\t\$%.2g\$ &\t\$%.2g\$ &\t\$%.2g\$ &\t\$%.2g\$\n", + typeof(o.Φ), + o.Mα, + D, + Dv, + Pv, + Pc, + c + ) + end +end; + +o = outputs.obs[1] +o.Dv +o.Pc +o.Pv +o.c + +# apply_bc_u!(u, t, dns) +ubar = FaceAverage()(u, les, comp); +ubar = VolumeAverage()(u, les, comp); +# apply_bc_u!(ubar, t, les) fieldplot((; u = ubar, t); setup = les) +fieldplot((; u, t); setup = dns) IncompressibleNavierStokes.apply_bc_u!(ubar, t, les) div = IncompressibleNavierStokes.divergence(ubar, les)[les.grid.Ip] From 0ecedd8cb42e2caba3c5c64ea1bff4da1a50c51d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Fri, 19 Jan 2024 01:03:07 +0100 Subject: [PATCH 228/379] Update script --- scratch/divergence.jl | 86 +++++++++++++++++++++++++++++++++++++------ 1 file changed, 74 insertions(+), 12 deletions(-) diff --git a/scratch/divergence.jl b/scratch/divergence.jl index 76e6dabae..ccf5794ff 100644 --- a/scratch/divergence.jl +++ b/scratch/divergence.jl @@ -20,11 +20,19 @@ T = Float64 # Array type ArrayType = Array -## using CUDA; ArrayType = CuArray +# using CUDA; ArrayType = CuArray; ## using AMDGPU; ArrayType = ROCArray ## using oneAPI; ArrayType = oneArray ## using Metal; ArrayType = MtlArray +using CUDA; +# T = Float64; +T = Float32; +ArrayType = CuArray; +CUDA.allowscalar(false); + +set_theme!(; GLMakie = (; scalefactor = 1.5)) + function observe_v(dnsobs, Φ, les, compression, psolver) (; ArrayType, grid) = les (; dimension, N, Iu, Ip) = grid @@ -109,21 +117,25 @@ observe_u(dns, psolver_dns, filters; nupdate = 1) = end # Viscosity model -Re = T(10_000) +# Re = T(10_000) +Re = T(6_000) # A 2D grid is a Cartesian product of two vectors -ndns = 1024 +# ndns = 4096 +ndns = 512 lims = T(0), T(1) -D = 2 +D = 3 dns = Setup(ntuple(α -> LinRange(lims..., ndns + 1), D)...; Re, ArrayType); filters = map([ + (FaceAverage(), 32), (FaceAverage(), 64), (FaceAverage(), 128), - (FaceAverage(), 256), + # (FaceAverage(), 256), + (VolumeAverage(), 32), (VolumeAverage(), 64), (VolumeAverage(), 128), - (VolumeAverage(), 256), + # (VolumeAverage(), 256), ]) do (Φ, nles) compression = ndns ÷ nles setup = Setup(ntuple(α -> LinRange(lims..., nles + 1), D)...; Re, ArrayType) @@ -138,15 +150,19 @@ psolver_dns = SpectralPressureSolver(dns); # Create random initial conditions u₀ = random_field(dns, T(0); psolver = psolver_dns); +GC.gc() +CUDA.reclaim() + # Solve unsteady problem -state, outputs = solve_unsteady( +@time state, outputs = solve_unsteady( dns, u₀, - (T(0), T(0.01)); - Δt = T(1e-4), + (T(0), T(0.1)); + Δt = T(5e-5), + docopy = false, psolver = psolver_dns, processors = ( - rtp = realtimeplotter(; setup = dns, nupdate = 1), + # rtp = realtimeplotter(; setup = dns, nupdate = 5), obs = observe_u(dns, psolver_dns, filters), # ehist = realtimeplotter(; # setup, @@ -158,10 +174,54 @@ state, outputs = solve_unsteady( # anim = animator(; setup, path = "$output/solution.mkv", nupdate = 20), # vtk = vtk_writer(; setup, nupdate = 10, dir = output, filename = "solution"), # field = fieldsaver(; setup, nupdate = 10), - log = timelogger(; nupdate = 100), + log = timelogger(; nupdate = 1), ), ); -(; u, t) = state; + +fieldplot( + state; + setup = dns, + fieldname = :eig2field, + levels = LinRange(T(12), T(16), 5), + # levels = LinRange(-1.0f0, 3.0f0, 5), + # levels = LinRange(-2.0f0, 2.0f0, 5), + # levels = 5, + docolorbar = false, +) + +field = IncompressibleNavierStokes.eig2field(state.u, dns)[dns.grid.Ip] +# hist(vec(Array(log(max(eps(T), field))) +hist(vec(Array(log.(max.(eps(T), .-field))))) +field = nothing + +# Float32, 1024^2: +# +# 5.711019 seconds (46.76 M allocations: 2.594 GiB, 4.59% gc time, 2.47% compilation time) +# 5.584943 seconds (46.60 M allocations: 2.583 GiB, 4.43% gc time) + +# Float64, 1024^2: +# +# 9.584393 seconds (46.67 M allocations: 2.601 GiB, 2.93% gc time) +# 9.672491 seconds (46.67 M allocations: 2.601 GiB, 2.93% gc time) + +# Float64, 4096^2: +# +# 114.006495 seconds (47.90 M allocations: 15.499 GiB, 0.28% gc time) +# 100.907239 seconds (46.45 M allocations: 2.588 GiB, 0.26% gc time) + +# Float32, 512^3: +# +# 788.762194 seconds (162.34 M allocations: 11.175 GiB, 0.12% gc time) + +9.584393 / 5.711019 +9.672491 / 5.584943 + +# 1.0 * nbyte(Float32) * N * α * (u0 + ui + k1,k2,k3,k4 + p + maybe(complexFFT(Lap)) + maybe(boundaryfree p)) +1.0 * 4 * 1024^3 * 3 * (1 + 1 + 4 + 1 / 3 + 0 * 1 * 2 + 0 * 1 / 3) # RK4: 81.6GB (111GB) +1.0 * 4 * 1024^3 * 3 * (1 + 0 + 1 + 1 / 3 + 0 * 1 * 2 + 0 * 1 / 3) # RK1: 30.0GB (60.1 GB) + +1.0 * 4 * 512^3 * 3 * (1 + 1 + 4 + 1 / 3 + 0 * 1 * 2 + 0 * 1 / 3) # RK4: 10.2GB (13.9GB) +1.0 * 4 * 512^3 * 3 * (1 + 0 + 1 + 1 / 3 + 1 * 1 * 2 + 1 * 1 / 3) # RK1: 3.76GB (7.52GB) begin println("Φ\t\tM\tDu\tPv\tPc\tc") @@ -185,6 +245,8 @@ begin end end; +(; u, t) = state; + o = outputs.obs[1] o.Dv o.Pc From eec3781f113650653858ab45367c3802e86c4286 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Sat, 20 Jan 2024 17:37:34 +0100 Subject: [PATCH 229/379] Update IC expression --- src/create_initial_conditions.jl | 154 +++++++++++++++++++++++++------ 1 file changed, 128 insertions(+), 26 deletions(-) diff --git a/src/create_initial_conditions.jl b/src/create_initial_conditions.jl index e8c41c525..5ca7bc35c 100644 --- a/src/create_initial_conditions.jl +++ b/src/create_initial_conditions.jl @@ -38,7 +38,7 @@ function create_initial_conditions( # Make velocity field divergence free apply_bc_u!(u, t, setup) - if doproject + if doproject u = project(u, setup; psolver) apply_bc_u!(u, t, setup) end @@ -47,52 +47,148 @@ function create_initial_conditions( u end -function create_spectrum(; setup, A, σ, s) +# function create_spectrum(; setup, A, σ, s) +# (; dimension, x, N) = setup.grid +# T = eltype(x[1]) +# D = dimension() +# K = N .÷ 2 +# k = ntuple( +# α -> reshape(1:K[α], ntuple(Returns(1), α - 1)..., :, ntuple(Returns(1), D - α)...), +# D, +# ) +# a = fill!(similar(x[1], Complex{T}, K), 1) +# τ = T(2π) +# a .*= prod(N) * A / sqrt(τ^2 * 2σ^2) +# for α = 1:D +# kα = k[α] +# @. a *= exp(-max(abs(kα) - s, 0)^2 / 2σ^2) +# end +# @. a *= randn(T) * exp(im * τ * rand(T)) +# for α = 1:D +# a = cat(a, reverse(a; dims = α); dims = α) +# end +# a +# end + +function create_spectrum(; setup, kp) (; dimension, x, N) = setup.grid T = eltype(x[1]) D = dimension() - K = N .÷ 2 - k = ntuple( - α -> reshape(1:K[α], ntuple(Returns(1), α - 1)..., :, ntuple(Returns(1), D - α)...), + τ = T(2π) + + # Maximum wavenumber (remove ghost volumes) + K = @. (N - 2) ÷ 2 + + # Wavenumber vectors + kk = ntuple( + α -> reshape( + 0:K[α]-1, + ntuple(Returns(1), α - 1)..., + :, + ntuple(Returns(1), D - α)..., + ), D, ) - a = fill!(similar(x[1], Complex{T}, K), 1) - τ = T(2π) - a .*= prod(N) * A / sqrt(τ^2 * 2σ^2) + + # Wavevector magnitude + k = fill!(similar(x[1], K), 0) for α = 1:D - kα = k[α] - @. a *= exp(-max(abs(kα) - s, 0)^2 / 2σ^2) + @. k += kk[α]^2 end - @. a *= randn(T) * exp(im * τ * rand(T)) + k .= sqrt.(k) + + # Shared magnitude + A = T(8τ / 3) / kp^5 + + # Velocity magnitude + a = @. complex(1) * sqrt(A * k^4 * exp(-τ * (k / kp)^2)) + a .*= prod(N) + + # Apply random phase shift + ξ = ntuple(α -> rand!(similar(x[1], K)), D) for α = 1:D a = cat(a, reverse(a; dims = α); dims = α) + ξ = ntuple(D) do β + s = α == β ? -1 : 1 + ξβ = ξ[β] + cat(ξβ, reverse(s .* ξβ; dims = α); dims = α) + end + end + ξ = sum(ξ) + a = @. exp(im * τ * ξ) * a + + KK = 2 .* K + kkkk = ntuple( + α -> reshape( + 0:KK[α]-1, + ntuple(Returns(1), α - 1)..., + :, + ntuple(Returns(1), D - α)..., + ), + D, + ) + knorm = fill!(similar(x[1], KK), 0) + for α = 1:D + @. knorm += kkkk[α]^2 + end + knorm .= sqrt.(knorm) + + # Create random unit vector for each wavenumber + if D == 2 + θ = rand!(similar(x[1], KK)) + e = (cospi.(2 .* θ), sinpi.(2 .* θ)) + elseif D == 3 + θ = rand!(similar(x[1], KK)) + ϕ = rand!(similar(x[1], KK)) + e = (sinpi.(θ) .* cospi.(2 .* ϕ), sinpi.(θ) .* sinpi.(2 .* ϕ), cospi.(θ)) + end + + # Remove non-divergence free part: (I - k k^T / k^2) e + ke = sum(α -> e[α] .* kkkk[α], 1:D) + CUDA.@allowscalar e0 = getindex.(e, 1) + for α = 1:D + @. e[α] -= kkkk[α] * ke / knorm ^ 2 + end + + # Restore k=0 component, which is divergence free anyways + CUDA.@allowscalar setindex!.(e, e0, 1) + + # Normalize + enorm = sqrt.(sum(α -> e[α] .^ 2, 1:D)) + for α = 1:D + e[α] ./= enorm + end + + # Split velocity magnitude a into velocity components a*eα + uhat = ntuple(D) do α + eα = e[α] + # for β = 1:D + # eα = cat(eα, reverse(eα; dims = β); dims = β) + # end + a .* eα end - a end """ random_field( setup, t = 0; - A = 10, - σ = 30, - s = 5, - psolver = DirectPressureSolver(setup), + A = 1, + kp = 10, + psolver = SpectralPressureSolver(setup), ) -Create random field. +Create random field, as in [Orlandi2000](@cite). - `K`: Maximum wavenumber -- `A`: Eddy amplitude -- `σ`: Variance -- `s` Wavenumber offset before energy starts decaying +- `A`: Eddy amplitude scaling +- `kp`: Peak energy wavenumber """ function random_field( setup, t = zero(eltype(setup.grid.x[1])); - A = convert(eltype(setup.grid.x[1]), 10), - σ = convert(eltype(setup.grid.x[1]), 30), - s = convert(eltype(setup.grid.x[1]), 5), - psolver = DirectPressureSolver(setup), + A = 1, + kp = 10, + psolver = SpectralPressureSolver(setup), ) (; dimension, x, Ip, Ω) = setup.grid D = dimension() @@ -100,9 +196,15 @@ function random_field( backend = get_backend(x[1]) # Create random velocity field - u = ntuple(α -> real.(ifft(create_spectrum(; setup, A, σ, s))), D) + uhat = create_spectrum(; setup, kp) + u = ifft.(uhat) + u = map(u -> A .* real.(u), u) - # Make velocity field divergence free + # Add ghost volumes (one on each side for periodic) + u = pad_circular.(u, 1; dims = 1:D) + + # Make velocity field divergence free on staggered grid + # (it is already diergence free on the "spectral grid") apply_bc_u!(u, t, setup) project(u, setup; psolver) apply_bc_u!(u, t, setup) From 083dcbcd765ff3ae502b9ca1b8063a839542b3a6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Sat, 20 Jan 2024 17:38:16 +0100 Subject: [PATCH 230/379] fix: image plot type --- src/processors/real_time_plot.jl | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/processors/real_time_plot.jl b/src/processors/real_time_plot.jl index e633d1bc0..066b18209 100644 --- a/src/processors/real_time_plot.jl +++ b/src/processors/real_time_plot.jl @@ -90,7 +90,7 @@ function fieldplot( kwargs..., ) (; boundary_conditions, grid) = setup - (; dimension, xlims, x, xp, Ip) = grid + (; dimension, xlims, x, xp, Ip, Δ) = grid D = dimension() xf = Array.(getindex.(setup.grid.xp, Ip.indices)) @@ -176,6 +176,14 @@ function fieldplot( ) equal_axis && (axis = (axis..., aspect = DataAspect())) + # Image requires boundary coordinates only + if type == image + Δx = first.(Array.(Δ)) + @assert all(≈(Δx[1]), Δx) "Image requires rectangular pixels" + @assert(all(α -> all(≈(Δx[α]), Δ[α]), 1:D), "Image requires uniform grid",) + xf = map(extrema, xf) + end + size = isnothing(size) ? (;) : (; size) fig = Figure(; size...) ax, hm = type(fig[1, 1], xf..., field; axis, kwargs...) From 947ca115275afe20581d0768dc9297d531390bf6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Mon, 22 Jan 2024 11:28:30 +0100 Subject: [PATCH 231/379] Update spectra --- src/IncompressibleNavierStokes.jl | 1 + src/processors/real_time_plot.jl | 51 ++++++++----------------------- src/utils/spectral_stuff.jl | 42 +++++++++++++++++++++++++ 3 files changed, 56 insertions(+), 38 deletions(-) create mode 100644 src/utils/spectral_stuff.jl diff --git a/src/IncompressibleNavierStokes.jl b/src/IncompressibleNavierStokes.jl index a41a64781..efb4f05ff 100644 --- a/src/IncompressibleNavierStokes.jl +++ b/src/IncompressibleNavierStokes.jl @@ -86,6 +86,7 @@ include("utils/plotgrid.jl") include("utils/save_vtk.jl") include("utils/get_lims.jl") include("utils/plotmat.jl") +include("utils/spectral_stuff.jl") # Closure models include("closures/closure.jl") diff --git a/src/processors/real_time_plot.jl b/src/processors/real_time_plot.jl index 066b18209..bfbe21d1c 100644 --- a/src/processors/real_time_plot.jl +++ b/src/processors/real_time_plot.jl @@ -331,59 +331,34 @@ as in San and Staples [San2012](@cite). """ function energy_spectrum_plot(state; setup, doaverage = false) state isa Observable || (state = Observable(state)) + (; dimension, xp, Ip) = setup.grid - backend = get_backend(xp[1]) T = eltype(xp[1]) D = dimension() - K = size(Ip) .÷ 2 - kx = ntuple(α -> 0:K[α]-1, D) - k = fill!(similar(xp[1], length.(kx)), 0) - for α = 1:D - kα = reshape(kx[α], ntuple(Returns(1), α - 1)..., :, ntuple(Returns(1), D - α)...) - k .+= kα .^ 2 - end - k .= sqrt.(k) - k = reshape(k, :) - - # Sum or average wavenumbers between k and k+1 - kmax = minimum(K) - 1 - nk = ceil(Int, maximum(k)) - kint = 1:kmax - ia = similar(xp[1], Int, 0) - ib = sortperm(k) - vals = similar(xp[1], 0) - ksort = k[ib] - jprev = 2 # Do not include constant mode - for ki = 1:kmax - j = findfirst(>(ki + 1), ksort) - isnothing(j) && (j = length(k) + 1) - ia = [ia; fill!(similar(ia, j - jprev), ki)] - val = doaverage ? T(1) / (j - jprev) : T(π) * ((ki + 1)^2 - ki^2) / (j - jprev) - vals = [vals; fill!(similar(vals, j - jprev), val)] - jprev = j - end - ib = ib[2:jprev-1] - A = sparse(ia, ib, vals, kmax, length(k)) + + (; K, kmax, k, A) = spectral_stuff(setup; doaverage) # Energy ke = kinetic_energy(state[].u, setup) ehat = lift(state) do (; u, t) kinetic_energy!(ke, u, setup) e = ke[Ip] - e = fft(e)[ntuple(α -> kx[α] .+ 1, D)...] - e = abs.(e) ./ size(e, 1) + e = fft(e)[ntuple(α -> 1:K[α], D)...] + # e = abs.(e) ./ size(e, 1) + e = abs.(e) ./ prod(size(e)) e = A * reshape(e, :) e = max.(e, eps(T)) # Avoid log(0) Array(e) end # Build inertial slope above energy - # krange = LinRange(extrema(kint)..., 100) - # krange = collect(extrema(kint)) - krange = [cbrt(T(kmax)), T(kmax)] + # krange = LinRange(1, kmax, 100) + # krange = collect(1, kmax) + # krange = [cbrt(T(kmax)), T(kmax)] + krange = [T(kmax)^(T(2) / 3), T(kmax)] slope, slopelabel = D == 2 ? (-T(3), L"$k^{-3}") : (-T(5 / 3), L"$k^{-5/3}") inertia = lift(ehat) do ehat - slopeconst = maximum(ehat ./ kint .^ slope) + slopeconst = maximum(ehat ./ (1:kmax) .^ slope) 2 .* slopeconst .* krange .^ slope end @@ -399,9 +374,9 @@ function energy_spectrum_plot(state; setup, doaverage = false) ylabel = "e(k)", xscale = log10, yscale = log10, - limits = (extrema(kint)..., T(1e-8), T(1)), + limits = (1, kmax, T(1e-8), T(1)), ) - lines!(ax, kint, ehat; label = "Kinetic energy") + lines!(ax, 1:kmax, ehat; label = "Kinetic energy") lines!(ax, krange, inertia; label = slopelabel, linestyle = :dash) axislegend(ax) # autolimits!(ax) diff --git a/src/utils/spectral_stuff.jl b/src/utils/spectral_stuff.jl new file mode 100644 index 000000000..b1f313b13 --- /dev/null +++ b/src/utils/spectral_stuff.jl @@ -0,0 +1,42 @@ +function spectral_stuff(setup; doaverage) + (; dimension, xp, Ip) = setup.grid + T = eltype(xp[1]) + D = dimension() + K = size(Ip) .÷ 2 + kx = ntuple(α -> 0:K[α]-1, D) + k = fill!(similar(xp[1], length.(kx)), 0) + for α = 1:D + kα = reshape(kx[α], ntuple(Returns(1), α - 1)..., :, ntuple(Returns(1), D - α)...) + k .+= kα .^ 2 + end + k .= sqrt.(k) + k = reshape(k, :) + + # Sum or average wavenumbers between k and k+1 + kmax = minimum(K) - 1 + nk = ceil(Int, maximum(k)) + kint = 1:kmax + # ia = similar(xp[1], Int, 0) + ia = similar(xp[1], Int, length(k)) + ib = sortperm(k) + # vals = similar(xp[1], 0) + vals = similar(xp[1], length(k)) + ksort = k[ib] + jprev = 2 # Do not include constant mode + for ki = 1:kmax + j = findfirst(>(ki + 1), ksort) + isnothing(j) && (j = length(k) + 1) + # ia = [ia; fill!(similar(ia, j - jprev), ki)] + ia[jprev:j-1] .= ki + val = doaverage ? T(1) / (j - jprev) : T(π) * ((ki + 1)^2 - ki^2) / (j - jprev) + # vals = [vals; fill!(similar(vals, j - jprev), val)] + vals[jprev:j-1] .= val + jprev = j + end + ia = ia[2:jprev-1] + ib = ib[2:jprev-1] + vals = vals[2:jprev-1] + A = sparse(ia, ib, vals, kmax, length(k)) + + (; K, kmax, kx, k, A) +end From cd19ad899da1aa9243a339f091d88fe596829af6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Mon, 22 Jan 2024 11:30:21 +0100 Subject: [PATCH 232/379] Update script --- docs/references.bib | 7 ++ scratch/divergence.jl | 206 +++++++++++++++++++++++++++++++++++++----- src/filter.jl | 12 ++- 3 files changed, 198 insertions(+), 27 deletions(-) diff --git a/docs/references.bib b/docs/references.bib index fb6f0f995..b35e3baba 100644 --- a/docs/references.bib +++ b/docs/references.bib @@ -140,6 +140,13 @@ @misc{MacArt2021 url = {https://arxiv.org/abs/2105.01030}, year = {2021}, } +@book{Orlandi2000, + title={Fluid flow phenomena: a numerical toolkit}, + author={Orlandi, Paolo}, + volume={55}, + year={2000}, + publisher={Springer Science \& Business Media} +} @book{Sagaut2006, address = {Berlin/Heidelberg}, author = {Sagaut, Pierre}, diff --git a/scratch/divergence.jl b/scratch/divergence.jl index ccf5794ff..7d5da7706 100644 --- a/scratch/divergence.jl +++ b/scratch/divergence.jl @@ -7,13 +7,16 @@ end #src #md using CairoMakie using GLMakie +# using CairoMakie using IncompressibleNavierStokes -using IncompressibleNavierStokes: momentum!, divergence!, project!, apply_bc_u! +using IncompressibleNavierStokes: momentum!, divergence!, project!, apply_bc_u!, spectral_stuff, kinetic_energy using LinearAlgebra using Printf +using FFTW # Output directory -output = "output/DecayingTurbulence2D" +output = "output/divergence" +output = "../SupervisedClosure/figures" # Floating point precision T = Float64 @@ -26,15 +29,16 @@ ArrayType = Array ## using Metal; ArrayType = MtlArray using CUDA; -# T = Float64; -T = Float32; +T = Float64; +# T = Float32; ArrayType = CuArray; CUDA.allowscalar(false); +set_theme!() set_theme!(; GLMakie = (; scalefactor = 1.5)) function observe_v(dnsobs, Φ, les, compression, psolver) - (; ArrayType, grid) = les + (; grid) = les (; dimension, N, Iu, Ip) = grid D = dimension() Mα = N[1] - 2 @@ -93,9 +97,6 @@ end observe_u(dns, psolver_dns, filters; nupdate = 1) = processor() do state - (; dimension, x) = dns.grid - T = eltype(x[1]) - D = dimension() PF = zero.(state[].u) div = zero(state[].u[1]) p = zero(state[].u[1]) @@ -117,25 +118,27 @@ observe_u(dns, psolver_dns, filters; nupdate = 1) = end # Viscosity model -# Re = T(10_000) -Re = T(6_000) +Re = T(10_000) +# Re = T(6_000) # A 2D grid is a Cartesian product of two vectors -# ndns = 4096 -ndns = 512 +ndns = 4096 +# ndns = 256 +# ndns = 512 lims = T(0), T(1) -D = 3 +D = 2 +# D = 3 dns = Setup(ntuple(α -> LinRange(lims..., ndns + 1), D)...; Re, ArrayType); filters = map([ - (FaceAverage(), 32), + # (FaceAverage(), 32), (FaceAverage(), 64), (FaceAverage(), 128), - # (FaceAverage(), 256), - (VolumeAverage(), 32), + (FaceAverage(), 256), + # (VolumeAverage(), 32), (VolumeAverage(), 64), (VolumeAverage(), 128), - # (VolumeAverage(), 256), + (VolumeAverage(), 256), ]) do (Φ, nles) compression = ndns ÷ nles setup = Setup(ntuple(α -> LinRange(lims..., nles + 1), D)...; Re, ArrayType) @@ -148,22 +151,42 @@ end; psolver_dns = SpectralPressureSolver(dns); # Create random initial conditions -u₀ = random_field(dns, T(0); psolver = psolver_dns); +# u₀ = random_field(dns, T(0); kp = 5, psolver = psolver_dns); +u₀ = random_field(dns, T(0); kp = 20, psolver = psolver_dns); +state = (; u = u₀, t = T(0)); GC.gc() CUDA.reclaim() +energy_spectrum_plot((; u = u₀, t = T(0)); setup = dns, doaverage = false) + +fieldplot( + (; u = u₀, t = T(0)); + setup = dns, + # type = image, + # colormap = :viridis, +) + # Solve unsteady problem @time state, outputs = solve_unsteady( dns, u₀, - (T(0), T(0.1)); + (T(0), T(1e-1)); + # Δt = T(1e-4), Δt = T(5e-5), - docopy = false, + docopy = true, psolver = psolver_dns, processors = ( - # rtp = realtimeplotter(; setup = dns, nupdate = 5), - obs = observe_u(dns, psolver_dns, filters), + # anim = animator(; path = "$output/solution.mkv", + # rtp = realtimeplotter(; + # setup = dns, + # nupdate = 50, + # fieldname = :eig2field, + # levels = LinRange(T(2), T(10), 10), + # # levels = 5, + # docolorbar = false, + # ), + obs = observe_u(dns, psolver_dns, filters; nupdate = 20), # ehist = realtimeplotter(; # setup, # plot = energy_history_plot, @@ -178,22 +201,75 @@ CUDA.reclaim() ), ); +# 103.5320324 + +state.u[1] + +fieldplot( + (; u = u₀, t = T(0)); + setup = dns, + # type = image, + # colormap = :viridis, + docolorbar = false, + size = (500, 500), +) + +save("$output/vorticity_start.png", current_figure()) + +fieldplot( + state, + setup = dns, + # type = image, + # colormap = :viridis, + docolorbar = false, + size = (500, 500), +) + +save("$output/vorticity_end.png", current_figure()) + +i = 1 +fieldplot( + (; u = filters[i].Φ(state.u, filters[i].setup, filters[i].compression), t = T(0)); + setup = filters[i].setup, + # type = image, + # colormap = :viridis, + docolorbar = false, + size = (500, 500), +) + +save("$output/vorticity_end_$(filters[i].compression).png", current_figure()) + fieldplot( - state; + (; u = u₀, t = T(0)); + # state; setup = dns, fieldname = :eig2field, - levels = LinRange(T(12), T(16), 5), + levels = LinRange(T(2), T(10), 10), + # levels = LinRange(T(4), T(12), 10), # levels = LinRange(-1.0f0, 3.0f0, 5), # levels = LinRange(-2.0f0, 2.0f0, 5), # levels = 5, docolorbar = false, + size = (800, 800), ) +save("$output/lambda2_start.png", current_figure()) +save("$output/lambda2_end.png", current_figure()) + field = IncompressibleNavierStokes.eig2field(state.u, dns)[dns.grid.Ip] # hist(vec(Array(log(max(eps(T), field))) hist(vec(Array(log.(max.(eps(T), .-field))))) field = nothing +energy_spectrum_plot(state; setup = dns, doaverage = false) + +i = 6 +ubar = filters[i].Φ(state.u, filters[i].setup, filters[i].compression) +energy_spectrum_plot((; u = ubar, t = T(0)); filters[i].setup, doaverage = false) + +state.u + + # Float32, 1024^2: # # 5.711019 seconds (46.76 M allocations: 2.594 GiB, 4.59% gc time, 2.47% compilation time) @@ -223,6 +299,8 @@ field = nothing 1.0 * 4 * 512^3 * 3 * (1 + 1 + 4 + 1 / 3 + 0 * 1 * 2 + 0 * 1 / 3) # RK4: 10.2GB (13.9GB) 1.0 * 4 * 512^3 * 3 * (1 + 0 + 1 + 1 / 3 + 1 * 1 * 2 + 1 * 1 / 3) # RK1: 3.76GB (7.52GB) +1.0 * 8 * 512^3 * 3 * (1 + 1 + 3 + 1 / 3 + 0 * 1 * 2 + 0 * 1 / 3) # RK4: 10.2GB (13.9GB) + begin println("Φ\t\tM\tDu\tPv\tPc\tc") for o in outputs.obs @@ -265,3 +343,83 @@ div = IncompressibleNavierStokes.divergence(ubar, les)[les.grid.Ip] norm(div) norm(ubar[1][les.grid.Iu[1]]) + +GLMakie.activate!() + +using CairoMakie +CairoMakie.activate!() + +# Plot predicted spectra +fig = with_theme(; palette = (; color = ["#3366cc", "#cc0000", "#669900", "#ffcc00"])) do + fields = [state.u, u₀, (f.Φ(state.u, f.setup, f.compression) for f in filters)...] + setups = [dns, dns, (f.setup for f in filters)...] + specs = map(fields, setups) do u, setup + GC.gc() + CUDA.reclaim() + (; dimension, xp, Ip) = setup.grid + T = eltype(xp[1]) + D = dimension() + (; K, kmax, k, A) = spectral_stuff(setup; doaverage = false) + ke = kinetic_energy(u, setup; interpolate_first = false) + e = ke[Ip] + e = fft(e)[ntuple(α -> 1:K[α], D)...] + e = abs.(e) ./ prod(size(e)) + # nn = sqrt(T(prod(size(e)))) + # nn = T(size(e, 1))^T(3.1) + # e = abs.(e) ./ nn + e = A * reshape(e, :) + e = max.(e, eps(T)) # Avoid log(0) + ehat = Array(e) + (; kmax, ehat) + end + (; kmax) = specs[1] + # Build inertial slope above energy + if D == 2 + # krange = [T(kmax)^(T(1) / 2), T(kmax)] + krange = [T(50), T(400)] + elseif D == 3 + krange = [T(kmax)^(T(2) / 3), T(kmax)] + end + slope, slopelabel = D == 2 ? (-T(3), L"$\kappa^{-3}") : (-T(5 / 3), L"$\kappa^{-5/3}") + # slope, slopelabel = D == 2 ? (-T(3), "|k|⁻³") : (-T(5 / 3), "|k|⁻⁵³") + # slope, slopelabel = D == 2 ? (-T(3), "κ⁻³") : (-T(5 / 3), "κ⁻⁵³") + slopeconst = maximum(specs[1].ehat ./ (1:kmax) .^ slope) + inertia = 2 .* slopeconst .* krange .^ slope + # Nice ticks + logmax = round(Int, log2(kmax + 1)) + xticks = T(2) .^ (0:logmax) + # Make plot + fig = Figure(; size = (500, 400)) + ax = Axis( + fig[1, 1]; + xticks, + # xlabel = "k", + xlabel = "κ", + ylabel = "e(κ)", + xscale = log10, + yscale = log10, + limits = (1, kmax, T(1e-8), T(1)), + title = "Kinetic energy", + ) + lines!(ax, 1:specs[1].kmax, specs[1].ehat; color = Cycled(1), label = "DNS") + lines!(ax, 1:specs[2].kmax, specs[2].ehat; color = Cycled(4), label = "DNS, t = 0") + lines!(ax, 1:specs[3].kmax, specs[3].ehat; color = Cycled(2), label = "Face average") + lines!(ax, 1:specs[4].kmax, specs[4].ehat; color = Cycled(2)) + lines!(ax, 1:specs[5].kmax, specs[5].ehat; color = Cycled(2)) + lines!(ax, 1:specs[6].kmax, specs[6].ehat; color = Cycled(3), label = "Volume average") + lines!(ax, 1:specs[7].kmax, specs[7].ehat; color = Cycled(3)) + lines!(ax, 1:specs[8].kmax, specs[8].ehat; color = Cycled(3)) + lines!(ax, krange, inertia; color = Cycled(1), label = slopelabel, linestyle = :dash) + axislegend(ax; position = :lb) + autolimits!(ax) + if D == 2 + limits!(ax, (T(0.68), T(520)), (T(1e-3), T(3e1))) + elseif D == 3 + limits!(ax, ax.xaxis.attributes.limits[], (T(1e-1), T(3e1))) + end + fig +end +GC.gc() +CUDA.reclaim() + +save("$output/priorspectra_$(D)D.pdf", fig) diff --git a/src/filter.jl b/src/filter.jl index ae85711c1..db01cc15c 100644 --- a/src/filter.jl +++ b/src/filter.jl @@ -3,8 +3,12 @@ abstract type AbstractFilter end struct FaceAverage <: AbstractFilter end struct VolumeAverage <: AbstractFilter end -(Φ::AbstractFilter)(u, setup_les, compression) = - Φ(ntuple(α -> similar(u[1], setup_les.grid.N), length(u)), u, setup_les, compression) +(Φ::AbstractFilter)(u, setup_les, compression) = Φ( + ntuple(α -> fill!(similar(u[1], setup_les.grid.N), 0), length(u)), + u, + setup_les, + compression, +) """ (::FaceAverage)(v, u, setup_les) @@ -50,14 +54,16 @@ function (::VolumeAverage)(v, u, setup_les, comp) I = @index(Global, Cartesian) J = I0 + comp * (I - oneunit(I)) s = zero(eltype(v[α])) + # n = 0 for i in volume # Periodic extension K = J + i K = mod1.(K.I, comp .* (N .- 2)) K = CartesianIndex(K) s += u[α][K] + # n += 1 end - n = (iseven(comp) ? comp : comp + 1) * comp^(D - 1) + n = (iseven(comp) ? comp + 1 : comp) * comp^(D - 1) v[α][I0+I] = s / n end for α = 1:D From f74cb29ec5232231793686b7f2c1b6f7445dec59 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Mon, 22 Jan 2024 13:45:35 +0100 Subject: [PATCH 233/379] Add reference --- src/operators.jl | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/operators.jl b/src/operators.jl index 41093ce89..f9bb86fdc 100644 --- a/src/operators.jl +++ b/src/operators.jl @@ -1115,7 +1115,8 @@ Qfield(u, setup) = Qfield!(similar(u[1], setup.grid.N), u, setup) """ eig2field!(λ, u, setup; ϵ = eps(eltype(λ))) -Compute the second eigenvalue of ``S^2 + \\Omega^2``. +Compute the second eigenvalue of ``S^2 + \\Omega^2``, +as proposed by Jeong and Hussain [Jeong1995](@cite). """ function eig2field!(λ, u, setup) (; grid, workgroupsize) = setup @@ -1139,7 +1140,8 @@ end """ eig2field(u, setup) -Compute the second eigenvalue of ``S^2 + \\Omega^2``. +Compute the second eigenvalue of ``S^2 + \\Omega^2``, +as proposed by Jeong and Hussain [Jeong1995](@cite). """ eig2field(u, setup) = eig2field!(similar(u[1], setup.grid.N), u, setup) From 15ea58d7315051f11ea2da004273d52262ccf97f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Wed, 24 Jan 2024 12:52:25 +0100 Subject: [PATCH 234/379] Add different LES models --- scratch/divergence.jl | 56 +- scratch/divfree.jl | 611 ++++++++++++++++++ src/closures/create_les_data.jl | 69 +- src/closures/training.jl | 6 +- src/operators.jl | 8 +- src/setup.jl | 2 + .../step_explicit_runge_kutta.jl | 33 +- 7 files changed, 730 insertions(+), 55 deletions(-) create mode 100644 scratch/divfree.jl diff --git a/scratch/divergence.jl b/scratch/divergence.jl index 7d5da7706..6e303a4c3 100644 --- a/scratch/divergence.jl +++ b/scratch/divergence.jl @@ -9,7 +9,8 @@ end #src using GLMakie # using CairoMakie using IncompressibleNavierStokes -using IncompressibleNavierStokes: momentum!, divergence!, project!, apply_bc_u!, spectral_stuff, kinetic_energy +using IncompressibleNavierStokes: + momentum!, divergence!, project!, apply_bc_u!, spectral_stuff, kinetic_energy using LinearAlgebra using Printf using FFTW @@ -197,7 +198,7 @@ fieldplot( # anim = animator(; setup, path = "$output/solution.mkv", nupdate = 20), # vtk = vtk_writer(; setup, nupdate = 10, dir = output, filename = "solution"), # field = fieldsaver(; setup, nupdate = 10), - log = timelogger(; nupdate = 1), + log = timelogger(; nupdate = 5), ), ); @@ -240,8 +241,8 @@ fieldplot( save("$output/vorticity_end_$(filters[i].compression).png", current_figure()) fieldplot( - (; u = u₀, t = T(0)); - # state; + # (; u = u₀, t = T(0)); + state; setup = dns, fieldname = :eig2field, levels = LinRange(T(2), T(10), 10), @@ -250,13 +251,36 @@ fieldplot( # levels = LinRange(-2.0f0, 2.0f0, 5), # levels = 5, docolorbar = false, - size = (800, 800), + # size = (800, 800), + size = (500, 500), ) save("$output/lambda2_start.png", current_figure()) save("$output/lambda2_end.png", current_figure()) -field = IncompressibleNavierStokes.eig2field(state.u, dns)[dns.grid.Ip] +i = 2 +fieldplot( + (; u = filters[i].Φ(state.u, filters[i].setup, filters[i].compression), t = T(0)); + setup = filters[i].setup, + fieldname = :eig2field, + levels = LinRange(T(2), T(10), 10), + # levels = LinRange(T(4), T(12), 10), + # levels = LinRange(-1.0f0, 3.0f0, 5), + # levels = LinRange(-2.0f0, 2.0f0, 5), + # levels = 5, + docolorbar = false, + # size = (800, 800), + size = (500, 500), +) + +save("$output/lambda2_end_filtered.png", current_figure()) + +i = 2 +# field = IncompressibleNavierStokes.eig2field(state.u, dns)[dns.grid.Ip] +field = IncompressibleNavierStokes.eig2field( + filters[i].Φ(state.u, filters[i].setup, filters[i].compression), + filters[i].setup, +)[filters[i].setup.grid.Ip] # hist(vec(Array(log(max(eps(T), field))) hist(vec(Array(log.(max.(eps(T), .-field))))) field = nothing @@ -269,7 +293,6 @@ energy_spectrum_plot((; u = ubar, t = T(0)); filters[i].setup, doaverage = false state.u - # Float32, 1024^2: # # 5.711019 seconds (46.76 M allocations: 2.594 GiB, 4.59% gc time, 2.47% compilation time) @@ -380,7 +403,8 @@ fig = with_theme(; palette = (; color = ["#3366cc", "#cc0000", "#669900", "#ffcc elseif D == 3 krange = [T(kmax)^(T(2) / 3), T(kmax)] end - slope, slopelabel = D == 2 ? (-T(3), L"$\kappa^{-3}") : (-T(5 / 3), L"$\kappa^{-5/3}") + slope, slopelabel = + D == 2 ? (-T(3), L"$\kappa^{-3}") : (-T(5 / 3), L"$\kappa^{-5/3}") # slope, slopelabel = D == 2 ? (-T(3), "|k|⁻³") : (-T(5 / 3), "|k|⁻⁵³") # slope, slopelabel = D == 2 ? (-T(3), "κ⁻³") : (-T(5 / 3), "κ⁻⁵³") slopeconst = maximum(specs[1].ehat ./ (1:kmax) .^ slope) @@ -401,16 +425,18 @@ fig = with_theme(; palette = (; color = ["#3366cc", "#cc0000", "#669900", "#ffcc limits = (1, kmax, T(1e-8), T(1)), title = "Kinetic energy", ) + # plotparts(i) = 1:specs[i].kmax, specs[i].ehat + plotparts(i) = 1:specs[i].kmax+1, [specs[i].ehat; eps(T)] lines!(ax, 1:specs[1].kmax, specs[1].ehat; color = Cycled(1), label = "DNS") lines!(ax, 1:specs[2].kmax, specs[2].ehat; color = Cycled(4), label = "DNS, t = 0") - lines!(ax, 1:specs[3].kmax, specs[3].ehat; color = Cycled(2), label = "Face average") - lines!(ax, 1:specs[4].kmax, specs[4].ehat; color = Cycled(2)) - lines!(ax, 1:specs[5].kmax, specs[5].ehat; color = Cycled(2)) - lines!(ax, 1:specs[6].kmax, specs[6].ehat; color = Cycled(3), label = "Volume average") - lines!(ax, 1:specs[7].kmax, specs[7].ehat; color = Cycled(3)) - lines!(ax, 1:specs[8].kmax, specs[8].ehat; color = Cycled(3)) + lines!(ax, plotparts(3)...; color = Cycled(2), label = "Face average") + lines!(ax, plotparts(4)...; color = Cycled(2)) + lines!(ax, plotparts(5)...; color = Cycled(2)) + lines!(ax, plotparts(6)...; color = Cycled(3), label = "Volume average") + lines!(ax, plotparts(7)...; color = Cycled(3)) + lines!(ax, plotparts(8)...; color = Cycled(3)) lines!(ax, krange, inertia; color = Cycled(1), label = slopelabel, linestyle = :dash) - axislegend(ax; position = :lb) + # axislegend(ax; position = :lb) autolimits!(ax) if D == 2 limits!(ax, (T(0.68), T(520)), (T(1e-3), T(3e1))) diff --git a/scratch/divfree.jl b/scratch/divfree.jl new file mode 100644 index 000000000..97df49c36 --- /dev/null +++ b/scratch/divfree.jl @@ -0,0 +1,611 @@ +# Little LSP hack to get function signatures, go #src +# to definition etc. #src +if isdefined(@__MODULE__, :LanguageServer) #src + include("../src/IncompressibleNavierStokes.jl") #src + using .IncompressibleNavierStokes #src +end #src + +# # Train closure model +# +# Here, we consider a periodic box ``[0, 1]^2``. It is discretized with a +# uniform Cartesian grid with square cells. + +using CairoMakie +using GLMakie +using IncompressibleNavierStokes +using JLD2 +using LaTeXStrings +using LinearAlgebra +using Lux +using NNlib +using Optimisers +using Random +using Zygote +using SparseArrays +using KernelAbstractions +using FFTW + +GLMakie.activate!() + +set_theme!(; GLMakie = (; scalefactor = 1.5)) + +output = "../SupervisedClosure/figures/" + +# Random number generator +rng = Random.default_rng() +Random.seed!(rng, 123) + +# Floating point precision +T = Float64 + +# Array type +ArrayType = Array +device = identity +## using CUDA; ArrayType = CuArray +## using AMDGPU; ArrayType = ROCArray +## using oneAPI; ArrayType = oneArray +## using Metal; ArrayType = MtlArray + +using LuxCUDA +using CUDA; +T = Float32; +ArrayType = CuArray; +CUDA.allowscalar(false); +device = cu + +# Parameters +get_params() = (; + D = 2, + Re = T(10_000), + tburn = T(0.05), + tsim = T(0.5), + Δt = T(1e-4), + nles = map(n -> (n, n), [32, 64, 128, 256]), + # ndns = (n -> (n, n))(1024), + ndns = (n -> (n, n))(2048), + # ndns = (n -> (n, n))(4096), + filters = (FaceAverage(), VolumeAverage()), + ArrayType, + PSolver = SpectralPressureSolver, + icfunc = (setup, psolver) -> random_field( + setup, + t = zero(eltype(setup.grid.x[1])); + A = 1, + kp = 20, + psolver, + ), +) + +params_train = (; get_params()..., savefreq = 5); +params_valid = (; get_params()..., savefreq = 20); +params_test = (; get_params()..., tsim = T(0.2), savefreq = 5); + +# Create LES data from DNS +data_train = [create_les_data(; params_train...) for _ = 1:5]; +data_valid = [create_les_data(; params_valid...) for _ = 1:1]; +data_test = create_les_data(; params_test...); + +# # Save filtered DNS data +# jldsave("output/forced/data.jld2"; data_train, data_valid, data_test) + +# # Load previous LES data +# data_train, data_valid, data_test = load("output/forced/data.jld2", "data_train", "data_valid", "data_test") + +# Build LES setup and assemble operators +getsetups(params) = [ + Setup( + ntuple(α -> LinRange(T(0), T(1), nles[α] + 1), params.D)...; + params.Re, + params.ArrayType, + ) for nles in params.nles +] +setups_train = getsetups(params_train); +setups_valid = getsetups(params_valid); +setups_test = getsetups(params_test); + +data_train[1].t +data_train[1].data |> size +data_train[1].data[1, 1].u[end][1] + +# Create input/output arrays +io_train = create_io_arrays(data_train, setups_train); +io_valid = create_io_arrays(data_valid, setups_valid); + +# Inspect data +ig = 4 +ifil = 2 +field, setup = data_train[1].data[ig, ifil].u, setups_train[ig]; +# field, setup = data_valid[1].u[ig], setups_valid[ig]; +# field, setup = data_test.u[ig], setups_test[ig]; +u = device(field[1]); +o = Observable((; u, t = nothing)); +# energy_spectrum_plot(o; setup) +fieldplot( + o; + setup, + # fieldname = :velocity, + # fieldname = 2, +) +# energy_spectrum_plot( o; setup,) +for i = 1:length(field) + o[] = (; o[]..., u = device(field[i])) + sleep(0.001) +end + +GLMakie.activate!() +CairoMakie.activate!() + +# Training data plot +ifil = 1 +boxx = T(0.3), T(0.5) +boxy = T(0.5), T(0.7) +box = [ + Point2f(boxx[1], boxy[1]), + Point2f(boxx[2], boxy[1]), + Point2f(boxx[2], boxy[2]), + Point2f(boxx[1], boxy[2]), + Point2f(boxx[1], boxy[1]), +] +# fig = with_theme() do +fig = with_theme(; palette = (; color = ["#3366cc", "#cc0000", "#669900", "#ffcc00"])) do + sample = data_train[1] + fig = Figure() + for (i, it) in enumerate((1, length(sample.t))) + # for (j, ig) in enumerate((1, 2, 3)) + for (j, ig) in enumerate((1, 2)) + setup = setups_train[ig] + xf = Array.(getindex.(setup.grid.xp, setup.grid.Ip.indices)) + u = sample.data[ig, ifil].u[it] |> device + ωp = + IncompressibleNavierStokes.interpolate_ω_p( + IncompressibleNavierStokes.vorticity(u, setup), + setup, + )[setup.grid.Ip] |> Array + colorrange = IncompressibleNavierStokes.get_lims(ωp) + opts = (; + xticksvisible = false, + xticklabelsvisible = false, + yticklabelsvisible = false, + yticksvisible = false, + ) + i == 2 && ( + opts = (; + opts..., + xlabel = "x", + xticksvisible = true, + xticklabelsvisible = true, + ) + ) + j == 1 && ( + opts = (; + opts..., + ylabel = "y", + yticklabelsvisible = true, + yticksvisible = true, + ) + ) + ax = Axis( + fig[i, j]; + opts..., + title = "n = $(params_train.nles[ig]), t = $(round(sample.t[it]; digits = 1))", + aspect = DataAspect(), + limits = (T(0), T(1), T(0), T(1)), + ) + heatmap!(ax, xf..., ωp; colorrange) + # lines!(ax, box; color = Cycled(2)) + end + end + fig +end + +save("$output/training_data.pdf", fig) + +closure, θ₀ = cnn(; + setup = setups_train[1], + radii = [2, 2, 2, 2], + channels = [20, 20, 20, params_train.D], + activations = [leakyrelu, leakyrelu, leakyrelu, identity], + use_bias = [true, true, true, false], + rng, +); +closure.chain + +# Prepare training +loss = createloss(mean_squared_error, closure); +dataloaders = createdataloader.(io_train; batchsize = 50, device); +# dataloaders[1]() +loss(dataloaders[1](), device(θ₀)) + +# Prepare training +θ = T(1.0e-1) * device(θ₀); +opt = Optimisers.setup(Adam(T(1.0e-3)), θ); +callbackstate = Point2f[]; + +# Train grid-specialized closure models +θ_cnn = map(CartesianIndices(size(io_train))) do I + # Prepare training + ig, ifil = I.I + @show ig ifil + d = createdataloader(io_train[ig, ifil]; batchsize = 50, device); + θ = T(1.0e-1) * device(θ₀) + opt = Optimisers.setup(Adam(T(1.0e-3)), θ) + callbackstate = Point2f[] + it = rand(1:size(io_valid[ig, ifil].u, 4), 50); + validset = map(v -> v[:, :, :, it], io_valid[ig, ifil]); + (; opt, θ, callbackstate) = train( + [d], + loss, + opt, + θ; + niter = 5000, + ncallback = 20, + callbackstate, + callback = create_callback(closure, device(validset)...; state = callbackstate), + ) + θ +end +GC.gc() +CUDA.reclaim() + +# # Save trained parameters +# jldsave("output/divfree/theta_cnn.jld2"; theta = Array.(θ_cnn)) + +# # Load trained parameters +# θθ = load("output/divfree/theta_cnn.jld2") +# copyto!.(θ_cnn, θθ["theta"]) + +# Train Smagorinsky closure model +ig = 2; +setup = setups_train[ig]; +sample = data_train[1]; +m = smagorinsky_closure(setup); +θ = T(0.05) +e_smag = sum(2:length(sample.t)) do it + It = setup.grid.Ip + u = sample.u[ig][it] |> device + c = sample.c[ig][it] |> device + mu = m(u, θ) + e = zero(eltype(u[1])) + for α = 1:D + # e += sum(abs2, mu[α][Ip] .- c[α][Ip]) / sum(abs2, c[α][Ip]) + e += norm(mu[α][Ip] .- c[α][Ip]) / norm(c[α][Ip]) + end + e / D +end / length(sample.t) +# for θ in LinRange(T(0), T(1), 100)]; + +# lines(LinRange(T(0), T(1), 100), e_smag) + +# Errors for grid-specialized closure models +e_smag, e_cnn = let + e_smag = zeros(T, size(data_test.data)..., 2) + e_cnn = zeros(T, size(data_test.data)..., 2) + for iorder = 1:2, ifil = 1:2, ig = 1:size(data_test.data, 1) + @show iorder ifil ig + setup = setups_test[ig] + params = params_test[ig] + psolver = SpectralPressureSolver(setup) + u = device.(data_test.data[ig,ifil].u) + u₀ = device(data_test.data[ig,ifil].u[1]) + Δt = params_test.Δt * params_test.savefreq + tlims = extrema(data_test.t) + nupdate = 4 + Δt /= nupdate + processors = (; relerr = relerr_trajectory(u, setup; nupdate)) + # Smagorinsky + closedsetup = (; setup..., + closure_model = smagorinsky_closure(setup), + unproject_closure = iorder == 2, + ) + _, outputs = solve_unsteady( + closedsetup, u₀, tlims; Δt, psolver, processors, + θ = T(0.1), + ) + e_smag[ig, ifil, iorder] = outputs.relerr[] + # CNN + closedsetup = (; setup..., + closure_model = wrappedclosure(closure, setup), + unproject_closure = iorder == 2, + ) + _, outputs = solve_unsteady( + closedsetup, u₀, tlims; Δt, psolver, processors, + θ = θ_cnn[ig, ifil], + ) + e_cnn[ig, ifil, iorder] = outputs.relerr[] + end + e_smag, e_cnn +end +e_smag +e_cnn + +# No model +e_nm = let + e_nm = zeros(T, size(data_test.data)...) + for ifil = 1:2, ig = 1:size(data_test.data, 1) + @show ifil ig + setup = setups_test[ig] + params = params_test[ig] + psolver = SpectralPressureSolver(setup) + u = device.(data_test.data[ig,ifil].u) + u₀ = device(data_test.data[ig,ifil].u[1]) + Δt = params_test.Δt * params_test.savefreq + tlims = extrema(data_test.t) + nupdate = 4 + Δt /= nupdate + processors = (; relerr = relerr_trajectory(u, setup; nupdate)) + _, outputs = solve_unsteady( + (; setup..., unproject_closure = false), u₀, tlims; Δt, psolver, processors, + ) + e_nm[ig, ifil] = outputs.relerr[] + end + e_nm +end +e_nm + +GC.gc() +CUDA.reclaim() + +e_nm +e_smag +e_cnn + +CairoMakie.activate!() +GLMakie.activate!() + +# Plot convergence +with_theme(; + # linewidth = 5, + # markersize = 10, + # markersize = 20, + # fontsize = 20, + palette = (; color = ["#3366cc", "#cc0000", "#669900", "#ffcc00"]), +) do + iorder = 1 + nles = [n[1] for n in params_test.nles] + fig = Figure(; size = (500, 400)) + ax = Axis( + fig[1, 1]; + xscale = log10, + yscale = log10, + xticks = nles, + xlabel = "n", + title = "Relative error (DNS: n = $(params_test.ndns[1]))", + ) + for ifil = 1:2 + linestyle = ifil == 1 ? :solid : :dash + label = "No closure" + label = label * (ifil == 1 ? " (FA)" : " (VA)") + scatterlines!(nles, e_nm[:, ifil]; color = Cycled(1), linestyle, marker = :circle, label) + end + for ifil = 1:2 + linestyle = ifil == 1 ? :solid : :dash + label = "Smagorinsky" + label = label * (ifil == 1 ? " (FA)" : " (VA)") + scatterlines!(nles, e_smag[:, ifil, iorder]; color = Cycled(2), linestyle, marker = :utriangle, label) + end + for ifil = 1:2 + linestyle = ifil == 1 ? :solid : :dash + label = "CNN" + label = label * (ifil == 1 ? " (FA)" : " (VA)") + scatterlines!(nles, e_cnn[:, ifil, iorder]; color = Cycled(3), linestyle, marker = :rect, label) + end + # lines!( + # collect(extrema(nles[3:end])), + # n -> 2e4 * n^-2.0; + # linestyle = :dash, + # label = "n⁻²", + # ) + axislegend(; position = :lb) + fig +end + +save("$output/convergence.pdf", current_figure()) + +markers_labels = [ + (:circle, ":circle"), + (:rect, ":rect"), + (:diamond, ":diamond"), + (:hexagon, ":hexagon"), + (:cross, ":cross"), + (:xcross, ":xcross"), + (:utriangle, ":utriangle"), + (:dtriangle, ":dtriangle"), + (:ltriangle, ":ltriangle"), + (:rtriangle, ":rtriangle"), + (:pentagon, ":pentagon"), + (:star4, ":star4"), + (:star5, ":star5"), + (:star6, ":star6"), + (:star8, ":star8"), + (:vline, ":vline"), + (:hline, ":hline"), + ('a', "'a'"), + ('B', "'B'"), + ('↑', "'\\uparrow'"), + ('😄', "'\\:smile:'"), + ('✈', "'\\:airplane:'"), +] + +# Final spectra +ig = 4 +setup = setups_test[ig]; +params = params_test +pressure_solver = SpectralPressureSolver(setup); +uref = device(data_test.u[ig][end]); +u₀ = device(data_test.u[ig][1]); +Δt = params_test.Δt * params_test.savefreq; +tlims = extrema(data_test.t); +nupdate = 4; +Δt /= nupdate; +state_nm, outputs = solve_unsteady(setup, u₀, tlims; Δt, pressure_solver); +m = smagorinsky_closure(setup); +closedsetup = (; setup..., closure_model = u -> m(u, T(0.1))); +state_smag, outputs = solve_unsteady(closedsetup, u₀, tlims; Δt, pressure_solver); +# closedsetup = (; setup..., closure_model = wrappedclosure(closure, θ_cnn_shared, setup)); +closedsetup = (; setup..., closure_model = wrappedclosure(closure, θ_cnn[ig-1], setup)); +state_cnn, outputs = solve_unsteady(closedsetup, u₀, tlims; Δt, pressure_solver); + +# Plot predicted spectra +fig = with_theme(; palette = (; color = ["#3366cc", "#cc0000", "#669900", "#ffcc00"])) do + (; xp, Ip) = setup.grid + D = params.D + K = size(Ip) .÷ 2 + kx = ntuple(α -> 0:K[α]-1, D) + k = fill!(similar(xp[1], length.(kx)), 0) + for α = 1:D + kα = reshape(kx[α], ntuple(Returns(1), α - 1)..., :, ntuple(Returns(1), D - α)...) + k .+= kα .^ 2 + end + k .= sqrt.(k) + k = reshape(k, :) + # Sum or average wavenumbers between k and k+1 + nk = ceil(Int, maximum(k)) + kmax = nk - 1 + # kmax = minimum(K) - 1 + kint = 1:kmax + ia = similar(xp[1], Int, 0) + ib = sortperm(k) + vals = similar(xp[1], 0) + ksort = k[ib] + jprev = 2 # Do not include constant mode + for ki = 1:kmax + j = findfirst(>(ki + 1), ksort) + isnothing(j) && (j = length(k) + 1) + ia = [ia; fill!(similar(ia, j - jprev), ki)] + # val = doaverage ? T(1) / (j - jprev) : T(1) + val = T(π) * ((ki + 1)^2 - ki^2) / (j - jprev) + # val = T(1) / (j - jprev) + vals = [vals; fill!(similar(vals, j - jprev), val)] + jprev = j + end + ib = ib[2:jprev-1] + A = sparse(ia, ib, vals, kmax, length(k)) + # Build inertial slope above energy + # krange = [cbrt(T(kmax)), T(kmax)] + krange = [T(kmax)^(T(2) / 3), T(kmax)] + # slope, slopelabel = D == 2 ? (-T(3), L"k^{-3}") : (-T(5 / 3), L"k^{-5/3}") + slope, slopelabel = D == 2 ? (-T(3), "|k|⁻³") : (-T(5 / 3), "|k|⁻⁵³") + slopeconst = T(0) + # Nice ticks + logmax = round(Int, log2(kmax + 1)) + xticks = T(2) .^ (0:logmax) + # Make plot + fig = Figure(; size = (500, 400)) + ax = Axis( + fig[1, 1]; + xticks, + xlabel = "|k|", + ylabel = "e(|k|)", + # title = "Kinetic energy (n = $(params.nles[ig])) at time t = $(round(data_test.t[end]; digits = 1))", + title = "Kinetic energy (n = $(params.nles[ig]))", + xscale = log10, + yscale = log10, + limits = (extrema(kint)..., T(1e-8), T(1)), + ) + for (u, label) in ( + # (uref, "Reference"), + (state_nm.u, "No closure"), + (state_smag.u, "Smagorinsky"), + (state_cnn.u, "CNN (specialized)"), + (uref, "Reference"), + ) + ke = IncompressibleNavierStokes.kinetic_energy(u, setup) + e = ke[Ip] + e = fft(e)[ntuple(α -> kx[α] .+ 1, D)...] + e = abs.(e) ./ size(e, 1) + e = A * reshape(e, :) + ehat = max.(e, eps(T)) # Avoid log(0) + slopeconst = max(slopeconst, maximum(ehat ./ kint .^ slope)) + lines!(ax, kint, Array(ehat); label) + end + inertia = 2 .* slopeconst .* krange .^ slope + lines!(ax, krange, inertia; linestyle = :dash, label = slopelabel) + axislegend(ax; position = :lb) + autolimits!(ax) + fig +end + +save("predicted_spectra.pdf", fig) + +# Plot spectrum errors +fig = with_theme(; palette = (; color = ["#3366cc", "#cc0000", "#669900", "#ffcc00"])) do + (; xp, Ip) = setup.grid + D = params.D + K = size(Ip) .÷ 2 + kx = ntuple(α -> 0:K[α]-1, D) + k = fill!(similar(xp[1], length.(kx)), 0) + for α = 1:D + kα = reshape(kx[α], ntuple(Returns(1), α - 1)..., :, ntuple(Returns(1), D - α)...) + k .+= kα .^ 2 + end + k .= sqrt.(k) + k = reshape(k, :) + # Sum or average wavenumbers between k and k+1 + nk = ceil(Int, maximum(k)) + # kmax = nk - 1 + kmax = minimum(K) - 1 + kint = 1:kmax + ia = similar(xp[1], Int, 0) + ib = sortperm(k) + vals = similar(xp[1], 0) + ksort = k[ib] + jprev = 2 # Do not include constant mode + for ki = 1:kmax + j = findfirst(>(ki + 1), ksort) + isnothing(j) && (j = length(k) + 1) + ia = [ia; fill!(similar(ia, j - jprev), ki)] + # val = doaverage ? T(1) / (j - jprev) : T(1) + val = T(π) * ((ki + 1)^2 - ki^2) / (j - jprev) + vals = [vals; fill!(similar(vals, j - jprev), val)] + jprev = j + end + ib = ib[2:jprev-1] + A = sparse(ia, ib, vals, kmax, length(k)) + # Build inertial slope above energy + # krange = [cbrt(T(kmax)), T(kmax)] + krange = [T(kmax)^(T(2) / 3), T(kmax)] + # slope, slopelabel = D == 2 ? (-T(3), L"k^{-3}") : (-T(5 / 3), L"k^{-5/3}") + slope, slopelabel = D == 2 ? (-T(3), "|k|⁻³") : (-T(5 / 3), "|k|⁻⁵³") + slopeconst = T(0) + # Nice ticks + logmax = round(Int, log2(kmax + 1)) + xticks = T(2) .^ (0:logmax) + # Make plot + fig = Figure(; size = (500, 400)) + ax = Axis( + fig[1, 1]; + xticks, + xlabel = "|k|", + ylabel = "e(|k|)", + # title = "Kinetic energy (n = $(params.nles[ig])) at time t = $(round(data_test.t[end]; digits = 1))", + title = "Relative energy error (n = $(params.nles[ig]))", + xscale = log10, + yscale = log10, + limits = (extrema(kint)..., T(1e-8), T(1)), + ) + ke = IncompressibleNavierStokes.kinetic_energy(uref, setup) + e = ke[Ip] + e = fft(e)[ntuple(α -> kx[α] .+ 1, D)...] + e = abs.(e) ./ size(e, 1) + e = A * reshape(e, :) + eref = max.(e, eps(T)) # Avoid log(0) + for (u, label) in ( + (state_nm.u, "No closure"), + (state_smag.u, "Smagorinsky"), + (state_cnn.u, "CNN (specialized)"), + ) + ke = IncompressibleNavierStokes.kinetic_energy(u, setup) + e = ke[Ip] + e = fft(e)[ntuple(α -> kx[α] .+ 1, D)...] + e = abs.(e) ./ size(e, 1) + e = A * reshape(e, :) + ehat = max.(e, eps(T)) # Avoid log(0) + ee = @. abs(ehat - eref) / abs(eref) + lines!(ax, kint, Array(ee); label) + end + axislegend(ax; position = :lt) + autolimits!(ax) + fig +end + +save("spectrum_error.pdf", fig) diff --git a/src/closures/create_les_data.jl b/src/closures/create_les_data.jl index f97aa5aab..311a45409 100644 --- a/src/closures/create_les_data.jl +++ b/src/closures/create_les_data.jl @@ -32,14 +32,17 @@ function gaussian_force( force end -function lesdatagen(dnsobs, les, Φ, compression, psolver) +function lesdatagen(dnsobs, Φ, les, compression, psolver) Φu = zero.(Φ(dnsobs[].u, les, compression)) p = zero(Φu[1]) div = zero(p) ΦF = zero.(Φu) FΦ = zero.(Φu) c = zero.(Φu) - results = (; u = fill(Array.(dnsobs[].u), 0), c = fill(Array.(dnsobs[].u), 0)) + results = (; + u = fill(Array.(dnsobs[].u), 0), + c = fill(Array.(dnsobs[].u), 0), + ) on(dnsobs) do (; u, F, t) Φ(Φu, u, les, compression) apply_bc_u!(Φu, t, les) @@ -56,23 +59,22 @@ function lesdatagen(dnsobs, les, Φ, compression, psolver) results end -filtersaver(dns, les, Φ, compression, psolver_dns, psolver_les; nupdate = 1) = +filtersaver(dns, les, filters, compression, psolver_dns, psolver_les; nupdate = 1) = processor() do state - (; dimension, x) = dns.grid + (; x) = dns.grid T = eltype(x[1]) - D = dimension() F = zero.(state[].u) div = zero(state[].u[1]) p = zero(state[].u[1]) dnsobs = Observable((; state[].u, F, state[].t)) data = [ - lesdatagen(dnsobs, les[i], Φ[i], compression[i], psolver_les[i]) for i = 1:length(les) + lesdatagen(dnsobs, Φ, les[i], compression[i], psolver_les[i]) for + i = 1:length(les), Φ in filters ] results = (; - t = fill(zero(eltype(x[1])), 0), - u = [d.u for d in data], - c = [d.c for d in data], - ) + data, + t = zeros(T, 0), + ) on(state) do (; u, t, n) n % nupdate == 0 || return momentum!(F, u, t, dns) @@ -93,6 +95,7 @@ filtersaver(dns, les, Φ, compression, psolver_dns, psolver_les; nupdate = 1) = lims = (T(0), T(1)), nles = [64], ndns = 256, + filters = (FaceAverage(),), tburn = T(0.1), tsim = T(0.1), Δt = T(1e-4), @@ -101,23 +104,24 @@ filtersaver(dns, les, Φ, compression, psolver_dns, psolver_les; nupdate = 1) = Create filtered DNS data. """ -function create_les_data( - T; +function create_les_data(; D = 2, - Re = T(2_000), - lims = ntuple(α -> (T(0), T(1)), D), + Re = 2e3, + lims = ntuple(α -> (typeof(Re)(0), typeof(Re)(1)), D), nles = [ntuple(α -> 64, D)], ndns = ntuple(α -> 256, D), - tburn = T(0.1), - tsim = T(0.1), - Δt = T(1e-4), + filters = (FaceAverage(),), + tburn = typeof(Re)(0.1), + tsim = typeof(Re)(0.1), + Δt = typeof(Re)(1e-4), PSolver = SpectralPressureSolver, savefreq = 1, ArrayType = Array, - icfunc = (setup, psolver) -> random_field(setup, T(0); psolver), - Φ = map(() -> FaceAverage(), nles), + icfunc = (setup, psolver) -> random_field(setup, typeof(Re)(0); psolver), kwargs..., ) + T = typeof(Re) + compression = [ndns[1] ÷ nles[1] for nles in nles] for (c, n) in zip(compression, nles), α = 1:D @assert c * n[α] == ndns[α] @@ -150,8 +154,12 @@ function create_les_data( # datasize = Base.summarysize(filtered) / 1e6 datasize = - (nt ÷ savefreq + 1) * sum(prod.(nles)) * D * 2 * length(bitstring(zero(T))) / 8 / - 1e6 + length(filters) * + (nt ÷ savefreq + 1) * + sum(prod.(nles)) * + D * + 2 * + length(bitstring(zero(T))) / 8 / 1e6 @info "Generating $datasize Mb of LES data" # Initial conditions @@ -185,18 +193,20 @@ function create_les_data( f = filtersaver( _dns, _les, - Φ, + filters, compression, psolver, psolver_les; nupdate = savefreq, ), + # plot = realtimeplotter(; setup = dns, nupdate = 10), + log = timelogger(; nupdate = 100), ), psolver, ) # Store result for current IC - outputs[1] + outputs.f end """ @@ -206,18 +216,19 @@ Create ``(\\bar{u}, c)`` pairs for training. """ function create_io_arrays(data, setups) nsample = length(data) - ngrid = length(setups) - nt = length(data[1].u[1]) - 1 - T = eltype(data[1].u[1][1][1]) - map(1:ngrid) do ig + ngrid, nfilter = size(data[1].data) + nt = length(data[1].t) - 1 + T = eltype(data[1].t) + map(CartesianIndices((ngrid, nfilter))) do I + ig, ifil = I.I (; dimension, N, Iu) = setups[ig].grid D = dimension() u = zeros(T, (N .- 2)..., D, nt + 1, nsample) c = zeros(T, (N .- 2)..., D, nt + 1, nsample) ifield = ntuple(Returns(:), D) for is = 1:nsample, it = 1:nt+1, α = 1:D - copyto!(view(u, ifield..., α, it, is), view(data[is].u[ig][it][α], Iu[α])) - copyto!(view(c, ifield..., α, it, is), view(data[is].c[ig][it][α], Iu[α])) + copyto!(view(u, ifield..., α, it, is), view(data[is].data[ig, ifil].u[it][α], Iu[α])) + copyto!(view(c, ifield..., α, it, is), view(data[is].data[ig, ifil].c[it][α], Iu[α])) end (; u = reshape(u, (N .- 2)..., D, :), c = reshape(c, (N .- 2)..., D, :)) end diff --git a/src/closures/training.jl b/src/closures/training.jl index 3fdb611bb..9ab7132e9 100644 --- a/src/closures/training.jl +++ b/src/closures/training.jl @@ -81,8 +81,8 @@ Compute MSE between `f(x, θ)` and `y`. The MSE is further divided by `normalize(y)`. """ -mean_squared_error(f, x, y, θ; normalize = y -> sum(abs2, y), λ = sqrt(eps(eltype(x)))) = - sum(abs2, f(x, θ) - y) / normalize(y) + λ * sum(abs2, θ) / length(θ) +mean_squared_error(f, x, y, θ; normalize = y -> sum(abs2, y), λ = sqrt(eltype(x)(1e-8))) = + sum(abs2, f(x, θ) - y) / normalize(y) + λ * sum(abs2, θ) """ relative_error(x, y) @@ -170,7 +170,7 @@ function create_callback(f, x, y; state = Point2f[], display_each_iteration = fa @info "Iteration $i \trelative error: $e" state = push!(copy(state), Point2f(istart + i, e)) obs[] = state - i < 10 || autolimits!(fig.axis) + i < 30 || autolimits!(fig.axis) display_each_iteration && display(fig) state end diff --git a/src/operators.jl b/src/operators.jl index f9bb86fdc..27137fe5b 100644 --- a/src/operators.jl +++ b/src/operators.jl @@ -467,7 +467,7 @@ Right hand side of momentum equations, excluding pressure gradient. Put the result in ``F``. """ function momentum!(F, u, t, setup; θ = nothing) - (; grid, closure_model) = setup + (; grid, closure_model, unproject_closure) = setup (; dimension) = grid D = dimension() for α = 1:D @@ -477,7 +477,7 @@ function momentum!(F, u, t, setup; θ = nothing) # convection!(F, u, setup) convectiondiffusion!(F, u, setup) bodyforce!(F, u, t, setup) - if !isnothing(closure_model) + if !isnothing(closure_model) && !unproject_closure m = closure_model mu = m(u, θ) for α = 1:D @@ -502,7 +502,7 @@ Right hand side of momentum equations, excluding pressure gradient. """ # momentum(u, t, setup) = momentum!(zero.(u), u, t, setup) function momentum(u, t, setup; θ = nothing) - (; grid, closure_model) = setup + (; grid, closure_model, unproject_closure) = setup (; dimension) = grid D = dimension() d = diffusion(u, setup) @@ -513,7 +513,7 @@ function momentum(u, t, setup; θ = nothing) # end F = @. d + c + f # F = tupleadd(d, c, f) - if !isnothing(closure_model) + if !isnothing(closure_model) && !unproject_closure m = closure_model F = F .+ m(u, θ) # F = tupleadd(F, m) diff --git a/src/setup.jl b/src/setup.jl index f10dc3222..fc3dd3938 100644 --- a/src/setup.jl +++ b/src/setup.jl @@ -21,6 +21,7 @@ function Setup( bodyforce = nothing, issteadybodyforce = true, closure_model = nothing, + unproject_closure = false, ArrayType = Array, workgroupsize = 64, ) @@ -32,6 +33,7 @@ function Setup( bodyforce, issteadybodyforce = false, closure_model, + unproject_closure, ArrayType, T = eltype(x[1]), workgroupsize, diff --git a/src/time_steppers/step_explicit_runge_kutta.jl b/src/time_steppers/step_explicit_runge_kutta.jl index 3e9da8d00..b8df934f9 100644 --- a/src/time_steppers/step_explicit_runge_kutta.jl +++ b/src/time_steppers/step_explicit_runge_kutta.jl @@ -3,12 +3,13 @@ create_stepper(::ExplicitRungeKuttaMethod; setup, psolver, u, t, n = 0) = function timestep!(method::ExplicitRungeKuttaMethod, stepper, Δt; θ = nothing, cache) (; setup, psolver, u, t, n) = stepper - (; grid) = setup + (; grid, closure_model, unproject_closure) = setup (; dimension, Iu) = grid (; A, b, c) = method (; u₀, ku, div, p) = cache D = dimension() nstage = length(b) + m = closure_model # Update current solution t₀ = t @@ -19,6 +20,16 @@ function timestep!(method::ExplicitRungeKuttaMethod, stepper, Δt; θ = nothing, apply_bc_u!(u, t, setup) momentum!(ku[i], u, t, setup; θ) + if unproject_closure + # Project F first, add closure second + apply_bc_u!(ku[i], t, setup; dudt = true) + project!(ku[i], setup; psolver, div, p) + mu = m(u, θ) + for α = 1:D + ku[i][α] .+= mu[α] + end + end + # Intermediate time step t = t₀ + c[i] * Δt @@ -33,7 +44,10 @@ function timestep!(method::ExplicitRungeKuttaMethod, stepper, Δt; θ = nothing, # Make velocity divergence free at time t apply_bc_u!(u, t, setup) - project!(u, setup; psolver, div, p) + if !unproject_closure + # Project stage u directly, closure is already included in momentum + project!(u, setup; psolver, div, p) + end end # This is redundant, but Neumann BC need to have _exact_ copies @@ -46,11 +60,12 @@ end function timestep(method::ExplicitRungeKuttaMethod, stepper, Δt; θ = nothing) (; setup, psolver, u, t, n) = stepper - (; grid) = setup + (; grid, closure_model, unproject_closure) = setup (; dimension) = grid (; A, b, c) = method D = dimension() nstage = length(b) + m = closure_model # Update current solution (does not depend on previous step size) t₀ = t @@ -62,6 +77,13 @@ function timestep(method::ExplicitRungeKuttaMethod, stepper, Δt; θ = nothing) u = apply_bc_u(u, t, setup) F = momentum(u, t, setup; θ) + if unproject_closure + # Project F first, add closure second + F = apply_bc_u(F, t, setup; dudt = true) + F = project(F, setup; psolver) + F = F .+ m(u, θ) + end + # Store right-hand side of stage i ku = (ku..., F) @@ -77,7 +99,10 @@ function timestep(method::ExplicitRungeKuttaMethod, stepper, Δt; θ = nothing) # Make velocity divergence free at time t u = apply_bc_u(u, t, setup) - u = project(u, setup; psolver) + if !unproject_closure + # Project stage u directly, closure is already included in momentum + u = project(u, setup; psolver) + end end # This is redundant, but Neumann BC need to have _exact_ copies From 832eb13e107a93ce6276c6b23cc17ac92ffbb108 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Thu, 25 Jan 2024 11:50:10 +0100 Subject: [PATCH 235/379] Update spectral plot --- scratch/divergence.jl | 157 +++++++++++++++++-------------- scratch/divfree.jl | 18 ++-- src/create_initial_conditions.jl | 4 + src/processors/real_time_plot.jl | 26 ++--- src/utils/spectral_stuff.jl | 8 +- 5 files changed, 121 insertions(+), 92 deletions(-) diff --git a/scratch/divergence.jl b/scratch/divergence.jl index 6e303a4c3..4e7e56ff4 100644 --- a/scratch/divergence.jl +++ b/scratch/divergence.jl @@ -5,39 +5,15 @@ if isdefined(@__MODULE__, :LanguageServer) #src using .IncompressibleNavierStokes #src end #src -#md using CairoMakie using GLMakie -# using CairoMakie using IncompressibleNavierStokes using IncompressibleNavierStokes: - momentum!, divergence!, project!, apply_bc_u!, spectral_stuff, kinetic_energy + momentum!, divergence!, project!, apply_bc_u!, spectral_stuff, kinetic_energy, + interpolate_u_p using LinearAlgebra using Printf using FFTW -# Output directory -output = "output/divergence" -output = "../SupervisedClosure/figures" - -# Floating point precision -T = Float64 - -# Array type -ArrayType = Array -# using CUDA; ArrayType = CuArray; -## using AMDGPU; ArrayType = ROCArray -## using oneAPI; ArrayType = oneArray -## using Metal; ArrayType = MtlArray - -using CUDA; -T = Float64; -# T = Float32; -ArrayType = CuArray; -CUDA.allowscalar(false); - -set_theme!() -set_theme!(; GLMakie = (; scalefactor = 1.5)) - function observe_v(dnsobs, Φ, les, compression, psolver) (; grid) = les (; dimension, N, Iu, Ip) = grid @@ -118,48 +94,73 @@ observe_u(dns, psolver_dns, filters; nupdate = 1) = results end -# Viscosity model +# Output directory +output = "output/divergence" +output = "../SupervisedClosure/figures" + +# Array type +ArrayType = Array +# using CUDA; ArrayType = CuArray; +## using AMDGPU; ArrayType = ROCArray +## using oneAPI; ArrayType = oneArray +## using Metal; ArrayType = MtlArray + +using CUDA; +ArrayType = CuArray; +CUDA.allowscalar(false); + +set_theme!() +set_theme!(; GLMakie = (; scalefactor = 1.5)) + +# 3D +T = Float32 Re = T(10_000) -# Re = T(6_000) +ndns = 512 +D = 3 +kp = 5 +filterdefs = [ + (FaceAverage(), 32), + (FaceAverage(), 64), + (FaceAverage(), 128), + (VolumeAverage(), 32), + (VolumeAverage(), 64), + (VolumeAverage(), 128), +] -# A 2D grid is a Cartesian product of two vectors +# 2D +T = Float64; +Re = T(10_000) ndns = 4096 -# ndns = 256 -# ndns = 512 -lims = T(0), T(1) D = 2 -# D = 3 -dns = Setup(ntuple(α -> LinRange(lims..., ndns + 1), D)...; Re, ArrayType); - -filters = map([ - # (FaceAverage(), 32), +kp = 30 +filterdefs = [ (FaceAverage(), 64), (FaceAverage(), 128), (FaceAverage(), 256), - # (VolumeAverage(), 32), (VolumeAverage(), 64), (VolumeAverage(), 128), (VolumeAverage(), 256), -]) do (Φ, nles) +] + +# Setup +lims = T(0), T(1) +dns = Setup(ntuple(α -> LinRange(lims..., ndns + 1), D)...; Re, ArrayType); +filters = map(filterdefs) do (Φ, nles) compression = ndns ÷ nles setup = Setup(ntuple(α -> LinRange(lims..., nles + 1), D)...; Re, ArrayType) psolver = SpectralPressureSolver(setup) (; setup, Φ, compression, psolver) end; - -# Since the grid is uniform and identical for x and y, we may use a specialized -# spectral pressure solver psolver_dns = SpectralPressureSolver(dns); # Create random initial conditions -# u₀ = random_field(dns, T(0); kp = 5, psolver = psolver_dns); -u₀ = random_field(dns, T(0); kp = 20, psolver = psolver_dns); +u₀ = random_field(dns, T(0); kp, psolver = psolver_dns); state = (; u = u₀, t = T(0)); GC.gc() CUDA.reclaim() -energy_spectrum_plot((; u = u₀, t = T(0)); setup = dns, doaverage = false) +energy_spectrum_plot((; u = u₀, t = T(0)); setup = dns) fieldplot( (; u = u₀, t = T(0)); @@ -201,6 +202,8 @@ fieldplot( log = timelogger(; nupdate = 5), ), ); +GC.gc() +CUDA.reclaim() # 103.5320324 @@ -252,11 +255,13 @@ fieldplot( # levels = 5, docolorbar = false, # size = (800, 800), - size = (500, 500), + size = (600, 600), ) -save("$output/lambda2_start.png", current_figure()) -save("$output/lambda2_end.png", current_figure()) +fname = "$output/lambda2_start.png" +fname = "$output/lambda2_end.png" +save(fname, current_figure()) +run(`convert $fname -trim $fname`) # Requires imagemagick i = 2 fieldplot( @@ -270,26 +275,32 @@ fieldplot( # levels = 5, docolorbar = false, # size = (800, 800), - size = (500, 500), + size = (600, 600), ) -save("$output/lambda2_end_filtered.png", current_figure()) +fname = "$output/lambda2_end_filtered.png" +save(fname, current_figure()) +run(`convert $fname -trim $fname`) # Requires imagemagick + +field = IncompressibleNavierStokes.eig2field(state.u, dns)[dns.grid.Ip] +hist(vec(Array(log.(max.(eps(T), .-field))))) +field = nothing i = 2 -# field = IncompressibleNavierStokes.eig2field(state.u, dns)[dns.grid.Ip] field = IncompressibleNavierStokes.eig2field( filters[i].Φ(state.u, filters[i].setup, filters[i].compression), filters[i].setup, )[filters[i].setup.grid.Ip] -# hist(vec(Array(log(max(eps(T), field))) hist(vec(Array(log.(max.(eps(T), .-field))))) field = nothing -energy_spectrum_plot(state; setup = dns, doaverage = false) +energy_spectrum_plot(state; setup = dns) + +save("spectrum.png", current_figure()) i = 6 ubar = filters[i].Φ(state.u, filters[i].setup, filters[i].compression) -energy_spectrum_plot((; u = ubar, t = T(0)); filters[i].setup, doaverage = false) +energy_spectrum_plot((; u = ubar, t = T(0)); filters[i].setup) state.u @@ -382,16 +393,19 @@ fig = with_theme(; palette = (; color = ["#3366cc", "#cc0000", "#669900", "#ffcc (; dimension, xp, Ip) = setup.grid T = eltype(xp[1]) D = dimension() - (; K, kmax, k, A) = spectral_stuff(setup; doaverage = false) - ke = kinetic_energy(u, setup; interpolate_first = false) - e = ke[Ip] - e = fft(e)[ntuple(α -> 1:K[α], D)...] - e = abs.(e) ./ prod(size(e)) - # nn = sqrt(T(prod(size(e)))) - # nn = T(size(e, 1))^T(3.1) - # e = abs.(e) ./ nn + (; K, kmax, k, A) = spectral_stuff(setup) + # up = interpolate_u_p(u, setup) + up = u + e = sum(up) do u + u = u[Ip] + uhat = fft(u)[ntuple(α -> 1:K[α], D)...] + # abs2.(uhat) + abs2.(uhat) ./ (2 * prod(size(u))^2) + # abs2.(uhat) ./ size(u, 1) + end e = A * reshape(e, :) e = max.(e, eps(T)) # Avoid log(0) + # ehat = (1:kmax) .* Array(e) ehat = Array(e) (; kmax, ehat) end @@ -399,9 +413,10 @@ fig = with_theme(; palette = (; color = ["#3366cc", "#cc0000", "#669900", "#ffcc # Build inertial slope above energy if D == 2 # krange = [T(kmax)^(T(1) / 2), T(kmax)] - krange = [T(50), T(400)] + # krange = [T(50), T(400)] + krange = [T(30), T(400)] elseif D == 3 - krange = [T(kmax)^(T(2) / 3), T(kmax)] + krange = [T(kmax)^(T(1.1) / 3), T(kmax)] end slope, slopelabel = D == 2 ? (-T(3), L"$\kappa^{-3}") : (-T(5 / 3), L"$\kappa^{-5/3}") @@ -427,8 +442,10 @@ fig = with_theme(; palette = (; color = ["#3366cc", "#cc0000", "#669900", "#ffcc ) # plotparts(i) = 1:specs[i].kmax, specs[i].ehat plotparts(i) = 1:specs[i].kmax+1, [specs[i].ehat; eps(T)] - lines!(ax, 1:specs[1].kmax, specs[1].ehat; color = Cycled(1), label = "DNS") - lines!(ax, 1:specs[2].kmax, specs[2].ehat; color = Cycled(4), label = "DNS, t = 0") + # lines!(ax, 1:specs[1].kmax, specs[1].ehat; color = Cycled(1), label = "DNS") + # lines!(ax, 1:specs[2].kmax, specs[2].ehat; color = Cycled(4), label = "DNS, t = 0") + lines!(ax, plotparts(1)...; color = Cycled(1), label = "DNS") + lines!(ax, plotparts(2)...; color = Cycled(4), label = "DNS, t = 0") lines!(ax, plotparts(3)...; color = Cycled(2), label = "Face average") lines!(ax, plotparts(4)...; color = Cycled(2)) lines!(ax, plotparts(5)...; color = Cycled(2)) @@ -436,12 +453,14 @@ fig = with_theme(; palette = (; color = ["#3366cc", "#cc0000", "#669900", "#ffcc lines!(ax, plotparts(7)...; color = Cycled(3)) lines!(ax, plotparts(8)...; color = Cycled(3)) lines!(ax, krange, inertia; color = Cycled(1), label = slopelabel, linestyle = :dash) - # axislegend(ax; position = :lb) + axislegend(ax; position = :lb) autolimits!(ax) if D == 2 - limits!(ax, (T(0.68), T(520)), (T(1e-3), T(3e1))) + # limits!(ax, (T(0.68), T(520)), (T(1e-3), T(3e1))) + limits!(ax, (ax.xaxis.attributes.limits[][1], T(1000)), (T(1e-15), ax.yaxis.attributes.limits[][2])) elseif D == 3 - limits!(ax, ax.xaxis.attributes.limits[], (T(1e-1), T(3e1))) + # limits!(ax, ax.xaxis.attributes.limits[], (T(1e-1), T(3e1))) + limits!(ax, ax.xaxis.attributes.limits[], (T(5e-6), ax.yaxis.attributes.limits[][2])) end fig end diff --git a/scratch/divfree.jl b/scratch/divfree.jl index 97df49c36..f9bbf3d28 100644 --- a/scratch/divfree.jl +++ b/scratch/divfree.jl @@ -54,31 +54,31 @@ CUDA.allowscalar(false); device = cu # Parameters -get_params() = (; +get_params(nlesscalar) = (; D = 2, Re = T(10_000), tburn = T(0.05), tsim = T(0.5), - Δt = T(1e-4), - nles = map(n -> (n, n), [32, 64, 128, 256]), + Δt = T(5e-5), + nles = map(n -> (n, n), nlesscalar), # ndns = (n -> (n, n))(1024), - ndns = (n -> (n, n))(2048), - # ndns = (n -> (n, n))(4096), + # ndns = (n -> (n, n))(2048), + ndns = (n -> (n, n))(4096), filters = (FaceAverage(), VolumeAverage()), ArrayType, PSolver = SpectralPressureSolver, icfunc = (setup, psolver) -> random_field( setup, - t = zero(eltype(setup.grid.x[1])); + zero(eltype(setup.grid.x[1])); A = 1, kp = 20, psolver, ), ) -params_train = (; get_params()..., savefreq = 5); -params_valid = (; get_params()..., savefreq = 20); -params_test = (; get_params()..., tsim = T(0.2), savefreq = 5); +params_train = (; get_params([32, 64, 128, 256])..., savefreq = 10); +params_valid = (; get_params([32, 64, 128, 256])..., savefreq = 40); +params_test = (; get_params([32, 64, 128, 256, 512, 1024])..., tsim = T(0.2), savefreq = 20); # Create LES data from DNS data_train = [create_les_data(; params_train...) for _ = 1:5]; diff --git a/src/create_initial_conditions.jl b/src/create_initial_conditions.jl index 5ca7bc35c..5c9e4b524 100644 --- a/src/create_initial_conditions.jl +++ b/src/create_initial_conditions.jl @@ -101,6 +101,7 @@ function create_spectrum(; setup, kp) A = T(8τ / 3) / kp^5 # Velocity magnitude + # a = @. complex(1) * sqrt(A * k^4 * exp(-(k / kp)^2)) a = @. complex(1) * sqrt(A * k^4 * exp(-τ * (k / kp)^2)) a .*= prod(N) @@ -203,6 +204,9 @@ function random_field( # Add ghost volumes (one on each side for periodic) u = pad_circular.(u, 1; dims = 1:D) + # # Interpolate to staggered grid + # interpolate_p_u!(u, setup) + # Make velocity field divergence free on staggered grid # (it is already diergence free on the "spectral grid") apply_bc_u!(u, t, setup) diff --git a/src/processors/real_time_plot.jl b/src/processors/real_time_plot.jl index bfbe21d1c..c81c490be 100644 --- a/src/processors/real_time_plot.jl +++ b/src/processors/real_time_plot.jl @@ -318,7 +318,7 @@ function energy_history_plot(state; setup) end """ - energy_spectrum_plot(state; setup, doaverage = false) + energy_spectrum_plot(state; setup) Create energy spectrum plot. The energy at a scalar wavenumber level ``\\kappa \\in \\mathbb{N}`` is defined by @@ -329,25 +329,28 @@ The energy at a scalar wavenumber level ``\\kappa \\in \\mathbb{N}`` is defined as in San and Staples [San2012](@cite). """ -function energy_spectrum_plot(state; setup, doaverage = false) +function energy_spectrum_plot(state; setup) state isa Observable || (state = Observable(state)) (; dimension, xp, Ip) = setup.grid T = eltype(xp[1]) D = dimension() - (; K, kmax, k, A) = spectral_stuff(setup; doaverage) + (; K, kmax, k, A) = spectral_stuff(setup) # Energy - ke = kinetic_energy(state[].u, setup) + # up = interpolate_u_p(state[].u, setup) ehat = lift(state) do (; u, t) - kinetic_energy!(ke, u, setup) - e = ke[Ip] - e = fft(e)[ntuple(α -> 1:K[α], D)...] - # e = abs.(e) ./ size(e, 1) - e = abs.(e) ./ prod(size(e)) + # interpolate_u_p!(up, u, setup) + up = u + e = sum(up) do u + u = u[Ip] + uhat = fft(u)[ntuple(α -> 1:K[α], D)...] + abs2.(uhat) ./ (2 * prod(size(uhat))^2) + end e = A * reshape(e, :) - e = max.(e, eps(T)) # Avoid log(0) + # e = max.(e, eps(T)) # Avoid log(0) + # (1:kmax) .* Array(e) Array(e) end @@ -355,7 +358,8 @@ function energy_spectrum_plot(state; setup, doaverage = false) # krange = LinRange(1, kmax, 100) # krange = collect(1, kmax) # krange = [cbrt(T(kmax)), T(kmax)] - krange = [T(kmax)^(T(2) / 3), T(kmax)] + krange = [kmax^T(0.3), kmax ^ (T(0.8))] + # krange = [T(kmax)^(T(2) / 3), T(kmax)] slope, slopelabel = D == 2 ? (-T(3), L"$k^{-3}") : (-T(5 / 3), L"$k^{-5/3}") inertia = lift(ehat) do ehat slopeconst = maximum(ehat ./ (1:kmax) .^ slope) diff --git a/src/utils/spectral_stuff.jl b/src/utils/spectral_stuff.jl index b1f313b13..4b5962a03 100644 --- a/src/utils/spectral_stuff.jl +++ b/src/utils/spectral_stuff.jl @@ -1,4 +1,4 @@ -function spectral_stuff(setup; doaverage) +function spectral_stuff(setup) (; dimension, xp, Ip) = setup.grid T = eltype(xp[1]) D = dimension() @@ -24,11 +24,13 @@ function spectral_stuff(setup; doaverage) ksort = k[ib] jprev = 2 # Do not include constant mode for ki = 1:kmax - j = findfirst(>(ki + 1), ksort) + j = findfirst(≥(ki + 1 / 2), ksort) isnothing(j) && (j = length(k) + 1) # ia = [ia; fill!(similar(ia, j - jprev), ki)] ia[jprev:j-1] .= ki - val = doaverage ? T(1) / (j - jprev) : T(π) * ((ki + 1)^2 - ki^2) / (j - jprev) + # val = doaverage ? T(1) / (j - jprev) : T(π) * ((ki + 1)^2 - ki^2) / (j - jprev) + # val = doaverage ? T(1) / (j - jprev) : T(1) + val = T(1) # vals = [vals; fill!(similar(vals, j - jprev), val)] vals[jprev:j-1] .= val jprev = j From 5ceb387b1a59d211d3e9426264f13f269ed97032 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Thu, 25 Jan 2024 18:21:33 +0100 Subject: [PATCH 236/379] Add dyadic binning --- src/processors/real_time_plot.jl | 8 ++++---- src/utils/spectral_stuff.jl | 25 ++++++++++++++++++------- 2 files changed, 22 insertions(+), 11 deletions(-) diff --git a/src/processors/real_time_plot.jl b/src/processors/real_time_plot.jl index c81c490be..b29c5665c 100644 --- a/src/processors/real_time_plot.jl +++ b/src/processors/real_time_plot.jl @@ -329,14 +329,14 @@ The energy at a scalar wavenumber level ``\\kappa \\in \\mathbb{N}`` is defined as in San and Staples [San2012](@cite). """ -function energy_spectrum_plot(state; setup) +function energy_spectrum_plot(state; setup, dyadic = true, a = 2) state isa Observable || (state = Observable(state)) (; dimension, xp, Ip) = setup.grid T = eltype(xp[1]) D = dimension() - (; K, kmax, k, A) = spectral_stuff(setup) + (; K, kmax, κ, A) = spectral_stuff(setup; dyadic, a) # Energy # up = interpolate_u_p(state[].u, setup) @@ -362,7 +362,7 @@ function energy_spectrum_plot(state; setup) # krange = [T(kmax)^(T(2) / 3), T(kmax)] slope, slopelabel = D == 2 ? (-T(3), L"$k^{-3}") : (-T(5 / 3), L"$k^{-5/3}") inertia = lift(ehat) do ehat - slopeconst = maximum(ehat ./ (1:kmax) .^ slope) + slopeconst = maximum(ehat ./ κ .^ slope) 2 .* slopeconst .* krange .^ slope end @@ -380,7 +380,7 @@ function energy_spectrum_plot(state; setup) yscale = log10, limits = (1, kmax, T(1e-8), T(1)), ) - lines!(ax, 1:kmax, ehat; label = "Kinetic energy") + lines!(ax, κ, ehat; label = "Kinetic energy") lines!(ax, krange, inertia; label = slopelabel, linestyle = :dash) axislegend(ax) # autolimits!(ax) diff --git a/src/utils/spectral_stuff.jl b/src/utils/spectral_stuff.jl index 4b5962a03..c6f22dff9 100644 --- a/src/utils/spectral_stuff.jl +++ b/src/utils/spectral_stuff.jl @@ -1,4 +1,4 @@ -function spectral_stuff(setup) +function spectral_stuff(setup; dyadic = true, a = 2) (; dimension, xp, Ip) = setup.grid T = eltype(xp[1]) D = dimension() @@ -15,7 +15,6 @@ function spectral_stuff(setup) # Sum or average wavenumbers between k and k+1 kmax = minimum(K) - 1 nk = ceil(Int, maximum(k)) - kint = 1:kmax # ia = similar(xp[1], Int, 0) ia = similar(xp[1], Int, length(k)) ib = sortperm(k) @@ -23,11 +22,23 @@ function spectral_stuff(setup) vals = similar(xp[1], length(k)) ksort = k[ib] jprev = 2 # Do not include constant mode - for ki = 1:kmax - j = findfirst(≥(ki + 1 / 2), ksort) + + if dyadic + a = T(a) + nκ = round(Int, log(T(kmax)) / log(a)) + 1 + κ = a .^ (0:nκ-1) + nextκ = κ -> κ * sqrt(a) + else + nκ = kmax + κ = 1:nκ + nextκ = κ -> κ + T(1) / 2 + end + + for i = 1:nκ + j = findfirst(≥(nextκ(κ[i])), ksort) isnothing(j) && (j = length(k) + 1) # ia = [ia; fill!(similar(ia, j - jprev), ki)] - ia[jprev:j-1] .= ki + ia[jprev:j-1] .= i # val = doaverage ? T(1) / (j - jprev) : T(π) * ((ki + 1)^2 - ki^2) / (j - jprev) # val = doaverage ? T(1) / (j - jprev) : T(1) val = T(1) @@ -38,7 +49,7 @@ function spectral_stuff(setup) ia = ia[2:jprev-1] ib = ib[2:jprev-1] vals = vals[2:jprev-1] - A = sparse(ia, ib, vals, kmax, length(k)) + A = sparse(ia, ib, vals, nκ, length(k)) - (; K, kmax, kx, k, A) + (; K, kmax, κ, A) end From 25f0abae80f2ab65732373b2247f714a85ff6a22 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Sun, 28 Jan 2024 22:37:35 +0100 Subject: [PATCH 237/379] Update plot --- src/closures/create_les_data.jl | 2 +- src/processors/real_time_plot.jl | 15 ++++--- src/utils/spectral_stuff.jl | 71 ++++++++++++++------------------ 3 files changed, 43 insertions(+), 45 deletions(-) diff --git a/src/closures/create_les_data.jl b/src/closures/create_les_data.jl index 311a45409..9d456f936 100644 --- a/src/closures/create_les_data.jl +++ b/src/closures/create_les_data.jl @@ -200,7 +200,7 @@ function create_les_data(; nupdate = savefreq, ), # plot = realtimeplotter(; setup = dns, nupdate = 10), - log = timelogger(; nupdate = 100), + log = timelogger(; nupdate = 10), ), psolver, ) diff --git a/src/processors/real_time_plot.jl b/src/processors/real_time_plot.jl index b29c5665c..d42638be6 100644 --- a/src/processors/real_time_plot.jl +++ b/src/processors/real_time_plot.jl @@ -329,14 +329,20 @@ The energy at a scalar wavenumber level ``\\kappa \\in \\mathbb{N}`` is defined as in San and Staples [San2012](@cite). """ -function energy_spectrum_plot(state; setup, dyadic = true, a = 2) +function energy_spectrum_plot( + state; + setup, + npoint = 100, + a = typeof(setup.Re)(1 + sqrt(5)) / 2, +) state isa Observable || (state = Observable(state)) (; dimension, xp, Ip) = setup.grid T = eltype(xp[1]) D = dimension() - (; K, kmax, κ, A) = spectral_stuff(setup; dyadic, a) + (; A, κ, K) = spectral_stuff(setup; npoint, a) + kmax = maximum(κ) # Energy # up = interpolate_u_p(state[].u, setup) @@ -350,7 +356,6 @@ function energy_spectrum_plot(state; setup, dyadic = true, a = 2) end e = A * reshape(e, :) # e = max.(e, eps(T)) # Avoid log(0) - # (1:kmax) .* Array(e) Array(e) end @@ -358,7 +363,7 @@ function energy_spectrum_plot(state; setup, dyadic = true, a = 2) # krange = LinRange(1, kmax, 100) # krange = collect(1, kmax) # krange = [cbrt(T(kmax)), T(kmax)] - krange = [kmax^T(0.3), kmax ^ (T(0.8))] + krange = [kmax^T(0.3), kmax^(T(0.8))] # krange = [T(kmax)^(T(2) / 3), T(kmax)] slope, slopelabel = D == 2 ? (-T(3), L"$k^{-3}") : (-T(5 / 3), L"$k^{-5/3}") inertia = lift(ehat) do ehat @@ -375,7 +380,7 @@ function energy_spectrum_plot(state; setup, dyadic = true, a = 2) fig[1, 1]; xticks, xlabel = "k", - ylabel = "e(k)", + # ylabel = "E(k)", xscale = log10, yscale = log10, limits = (1, kmax, T(1e-8), T(1)), diff --git a/src/utils/spectral_stuff.jl b/src/utils/spectral_stuff.jl index c6f22dff9..9b56bd9d6 100644 --- a/src/utils/spectral_stuff.jl +++ b/src/utils/spectral_stuff.jl @@ -1,12 +1,13 @@ -function spectral_stuff(setup; dyadic = true, a = 2) +function spectral_stuff(setup; npoint = 100, a = typeof(setup.Re)(1 + sqrt(5)) / 2) (; dimension, xp, Ip) = setup.grid T = eltype(xp[1]) D = dimension() + K = size(Ip) .÷ 2 - kx = ntuple(α -> 0:K[α]-1, D) - k = fill!(similar(xp[1], length.(kx)), 0) + k = zeros(T, K) for α = 1:D - kα = reshape(kx[α], ntuple(Returns(1), α - 1)..., :, ntuple(Returns(1), D - α)...) + kα = + reshape(0:K[α]-1, ntuple(Returns(1), α - 1)..., :, ntuple(Returns(1), D - α)...) k .+= kα .^ 2 end k .= sqrt.(k) @@ -14,42 +15,34 @@ function spectral_stuff(setup; dyadic = true, a = 2) # Sum or average wavenumbers between k and k+1 kmax = minimum(K) - 1 - nk = ceil(Int, maximum(k)) - # ia = similar(xp[1], Int, 0) - ia = similar(xp[1], Int, length(k)) - ib = sortperm(k) - # vals = similar(xp[1], 0) - vals = similar(xp[1], length(k)) - ksort = k[ib] - jprev = 2 # Do not include constant mode - - if dyadic - a = T(a) - nκ = round(Int, log(T(kmax)) / log(a)) + 1 - κ = a .^ (0:nκ-1) - nextκ = κ -> κ * sqrt(a) - else - nκ = kmax - κ = 1:nκ - nextκ = κ -> κ + T(1) / 2 - end + isort = sortperm(k) + ksort = k[isort] + ia = zeros(Int, 0) + ib = zeros(Int, 0) + vals = zeros(T, 0) + + # Output query points (evenly log-spaced, but only integer wavenumbers) + logκ = LinRange(T(0), log(T(kmax)), npoint) + κ = exp.(logκ) + κ = sort(unique(round.(Int, κ))) + npoint = length(κ) - for i = 1:nκ - j = findfirst(≥(nextκ(κ[i])), ksort) - isnothing(j) && (j = length(k) + 1) - # ia = [ia; fill!(similar(ia, j - jprev), ki)] - ia[jprev:j-1] .= i - # val = doaverage ? T(1) / (j - jprev) : T(π) * ((ki + 1)^2 - ki^2) / (j - jprev) - # val = doaverage ? T(1) / (j - jprev) : T(1) - val = T(1) - # vals = [vals; fill!(similar(vals, j - jprev), val)] - vals[jprev:j-1] .= val - jprev = j + for i = 1:npoint + jstart = findfirst(≥(κ[i] / a), ksort) + jstop = findfirst(≥(κ[i] * a), ksort) + isnothing(jstop) && (jstop = length(ksort) + 1) + jstop -= 1 + nk = jstop - jstart + 1 + append!(ia, fill(i, nk)) + append!(ib, isort[jstart:jstop]) + append!(vals, fill(T(1), nk)) end - ia = ia[2:jprev-1] - ib = ib[2:jprev-1] - vals = vals[2:jprev-1] - A = sparse(ia, ib, vals, nκ, length(k)) + IntArray = typeof(similar(xp[1], Int, 0)) + TArray = typeof(similar(xp[1], 0)) + ia = adapt(IntArray, ia) + ib = adapt(IntArray, ib) + vals = adapt(TArray, vals) + A = sparse(ia, ib, vals, npoint, length(k)) - (; K, kmax, κ, A) + (; A, κ, K) end From 501a32779b0e2bcbf75dcd92d71c5c94d0bfd860 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Sun, 28 Jan 2024 22:38:34 +0100 Subject: [PATCH 238/379] Update and rename loss/error --- src/IncompressibleNavierStokes.jl | 6 +- src/closures/training.jl | 116 +++++++++++++++++------------- 2 files changed, 72 insertions(+), 50 deletions(-) diff --git a/src/IncompressibleNavierStokes.jl b/src/IncompressibleNavierStokes.jl index efb4f05ff..82111a4c7 100644 --- a/src/IncompressibleNavierStokes.jl +++ b/src/IncompressibleNavierStokes.jl @@ -127,8 +127,10 @@ export plotmat export smagorinsky_closure export cnn, fno, FourierLayer export train -export mean_squared_error, relative_error, relerr_trajectory -export createloss, createdataloader, create_callback, create_les_data, create_io_arrays +export mean_squared_error, create_relerr_prior, create_relerr_post +export create_loss_prior, create_loss_post +export create_dataloader_prior, create_dataloader_post +export create_callback, create_les_data, create_io_arrays export wrappedclosure export FaceAverage, VolumeAverage diff --git a/src/closures/training.jl b/src/closures/training.jl index 9ab7132e9..6c97f6bf2 100644 --- a/src/closures/training.jl +++ b/src/closures/training.jl @@ -5,7 +5,7 @@ Create dataloader that uses a batch of `batchsize` random samples from `data` at each evaluation. The batch is moved to `device`. """ -createdataloader(data; batchsize = 50, device = identity) = function dataloader() +create_dataloader_prior(data; batchsize = 50, device = identity) = function dataloader() x, y = data nsample = size(x)[end] d = ndims(x) @@ -15,14 +15,14 @@ createdataloader(data; batchsize = 50, device = identity) = function dataloader( xuse, yuse end -createtrajectoryloader(trajectories; nunroll = 10, device = identity) = +create_dataloader_post(trajectories; nunroll = 10, device = identity) = function dataloader() (; u, t) = rand(trajectories) nt = length(t) @assert nt ≥ nunroll istart = rand(1:nt-nunroll) it = istart:istart+nunroll - (; u = device.(u[1][it]), t = t[it]) + (; u = device.(u[it]), t = t[it]) end """ @@ -66,13 +66,15 @@ function train( end """ - createloss(loss, f) + createloss_prior(loss, f) Wrap loss function `loss(batch, θ)`. The function `loss` should take inputs like `loss(f, x, y, θ)`. """ -createloss(loss, f) = ((x, y), θ) -> loss(f, x, y, θ) +create_loss_prior(loss, f) = ((x, y), θ) -> loss(f, x, y, θ) + +create_relerr_prior(f, x, y) = θ -> norm(f(x, θ) - y) / norm(y) """ mean_squared_error(f, x, y, θ; normalize = y -> sum(abs2, y), λ = sqrt(eps(eltype(x)))) @@ -84,60 +86,72 @@ The MSE is further divided by `normalize(y)`. mean_squared_error(f, x, y, θ; normalize = y -> sum(abs2, y), λ = sqrt(eltype(x)(1e-8))) = sum(abs2, f(x, θ) - y) / normalize(y) + λ * sum(abs2, θ) -""" - relative_error(x, y) - -Compute average column relative error between matrices `x` and `y`. -""" -relative_error(x, y) = - sum(norm(x - y) / norm(y) for (x, y) ∈ zip(eachcol(x), eachcol(y))) / size(x, 2) - -""" - relerr_trajectory(uref, setup) - -Processor to compute relative error between `uref` and `u` at each iteration. -""" -relerr_trajectory(uref, setup; nupdate = 1) = - processor() do state - (; dimension, x, Ip) = setup.grid - D = dimension() - T = eltype(x[1]) - e = Ref(T(0)) - on(state) do (; u, n) - n % nupdate == 0 || return - neff = n ÷ nupdate +function create_loss_post(; + setup, + method = RK44(; T = eltype(setup.grid.x[1])), + psolver, + closure, + nupdate = 1, + unproject_closure = false, +) + closure_model = wrappedclosure(closure, setup) + setup = (; setup..., closure_model, unproject_closure) + (; dimension, Iu) = setup.grid + D = dimension() + function loss_post(data, θ) + T = eltype(θ) + (; u, t) = data + v = u[1] + stepper = create_stepper(method; setup, psolver, u = v, t = t[1]) + loss = zero(eltype(v[1])) + for it = 2:length(t) + Δt = (t[it] - t[it-1]) / nupdate + for isub = 1:nupdate + stepper = timestep(method, stepper, Δt; θ) + end a, b = T(0), T(0) - for α = 1:D - # @show size(uref[n + 1]) - a += sum(abs2, u[α][Ip] - uref[neff+1][α][Ip]) - b += sum(abs2, uref[neff+1][α][Ip]) + for α = 1:length(u[1]) + a += sum(abs2, (stepper.u[α]-u[it][α])[Iu[α]]) + b += sum(abs2, u[it][α][Iu[α]]) end - e[] += sqrt(a) / sqrt(b) / (length(uref) - 1) + loss += a / b end - e + loss / (length(t) - 1) end +end -function create_trajectory_loss(; +function create_relerr_post(; + data, setup, method = RK44(; T = eltype(setup.grid.x[1])), - psolver = DirectPressureSolver(setup), + psolver, closure, + nupdate = 1, + unproject_closure = false, ) closure_model = wrappedclosure(closure, setup) - setup = (; setup..., closure_model) - function trajectory_loss(traj, θ) - (; u, t) = traj + setup = (; setup..., closure_model, unproject_closure) + (; dimension, Iu) = setup.grid + D = dimension() + function relerr_post(θ) + T = eltype(θ) + (; u, t) = data v = u[1] stepper = create_stepper(method; setup, psolver, u = v, t = t[1]) - loss = zero(eltype(v[1])) + e = zero(eltype(v[1])) for it = 2:length(t) - Δt = t[it] - t[it-1] - stepper = timestep(method, stepper, Δt; θ) - for α = 1:length(u[1]) - loss += sum(abs2, stepper.u[α] - u[it][α]) / sum(abs2, u[it][α]) + Δt = (t[it] - t[it-1]) / nupdate + for isub = 1:nupdate + stepper = timestep(method, stepper, Δt; θ) end + a, b = T(0), T(0) + for α = 1:D + a += sum(abs2, (stepper.u[α]-u[it][α])[Iu[α]]) + b += sum(abs2, u[it][α][Iu[α]]) + end + e += sqrt(a) / sqrt(b) end - loss / (length(t) - 1) + e / (length(t) - 1) end end @@ -158,19 +172,25 @@ If `state` is nonempty, it also plots previous convergence. If not using interactive GLMakie window, set `display_each_iteration` to `true`. """ -function create_callback(f, x, y; state = Point2f[], display_each_iteration = false) +function create_callback( + err; + state = Point2f[], + displayref = true, + display_each_iteration = false, +) istart = isempty(state) ? 0 : Int(first(state[end])) obs = Observable([Point2f(0, 0)]) fig = lines(obs; axis = (; title = "Relative prediction error", xlabel = "Iteration")) - hlines!([1.0f0]; linestyle = :dash) + displayref && hlines!([1.0f0]; linestyle = :dash) obs[] = state display(fig) function callback(state, i, θ) - e = norm(f(x, θ) - y) / norm(y) + e = err(θ) @info "Iteration $i \trelative error: $e" state = push!(copy(state), Point2f(istart + i, e)) obs[] = state - i < 30 || autolimits!(fig.axis) + # i < 30 || autolimits!(fig.axis) + autolimits!(fig.axis) display_each_iteration && display(fig) state end From cd6efc63978fab54d90fb78de608fa1a7bb4df74 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Sun, 28 Jan 2024 22:38:55 +0100 Subject: [PATCH 239/379] Add missing kwargs --- src/boundary_conditions.jl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/boundary_conditions.jl b/src/boundary_conditions.jl index 568e3e624..51e6cb70a 100644 --- a/src/boundary_conditions.jl +++ b/src/boundary_conditions.jl @@ -108,12 +108,12 @@ function apply_bc_p! end apply_bc_u(u, t, setup; kwargs...) = apply_bc_u!(copy.(u), t, setup; kwargs...) apply_bc_p(p, t, setup; kwargs...) = apply_bc_p!(copy(p), t, setup; kwargs...) -ChainRulesCore.rrule(::typeof(apply_bc_u), u, t, setup) = ( - apply_bc_u(u, t, setup), +ChainRulesCore.rrule(::typeof(apply_bc_u), u, t, setup; kwargs...) = ( + apply_bc_u(u, t, setup; kwargs...), # With respect to (apply_bc_u, u, t, setup) φbar -> ( NoTangent(), - apply_bc_u_pullback!(copy.((φbar...,)), (φbar...,), t, setup), + apply_bc_u_pullback!(copy.((φbar...,)), (φbar...,), t, setup; kwargs...), NoTangent(), NoTangent(), ), From c3982a901b49fe448822633ed14d64594083b521 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Sun, 28 Jan 2024 22:43:00 +0100 Subject: [PATCH 240/379] Update scripts --- scratch/divergence.jl | 49 +++++--- scratch/divfree.jl | 261 +++++++++++++++++++++++++++++++----------- 2 files changed, 223 insertions(+), 87 deletions(-) diff --git a/scratch/divergence.jl b/scratch/divergence.jl index 4e7e56ff4..3a45694d5 100644 --- a/scratch/divergence.jl +++ b/scratch/divergence.jl @@ -118,6 +118,7 @@ Re = T(10_000) ndns = 512 D = 3 kp = 5 +Δt = T(1e-4) filterdefs = [ (FaceAverage(), 32), (FaceAverage(), 64), @@ -132,7 +133,8 @@ T = Float64; Re = T(10_000) ndns = 4096 D = 2 -kp = 30 +kp = 20 +Δt = T(5e-5) filterdefs = [ (FaceAverage(), 64), (FaceAverage(), 128), @@ -174,8 +176,9 @@ fieldplot( dns, u₀, (T(0), T(1e-1)); + Δt, # Δt = T(1e-4), - Δt = T(5e-5), + # Δt = T(1e-5), docopy = true, psolver = psolver_dns, processors = ( @@ -188,7 +191,7 @@ fieldplot( # # levels = 5, # docolorbar = false, # ), - obs = observe_u(dns, psolver_dns, filters; nupdate = 20), + # obs = observe_u(dns, psolver_dns, filters; nupdate = 20), # ehist = realtimeplotter(; # setup, # plot = energy_history_plot, @@ -393,7 +396,7 @@ fig = with_theme(; palette = (; color = ["#3366cc", "#cc0000", "#669900", "#ffcc (; dimension, xp, Ip) = setup.grid T = eltype(xp[1]) D = dimension() - (; K, kmax, k, A) = spectral_stuff(setup) + K = size(Ip) .÷ 2 # up = interpolate_u_p(u, setup) up = u e = sum(up) do u @@ -403,27 +406,29 @@ fig = with_theme(; palette = (; color = ["#3366cc", "#cc0000", "#669900", "#ffcc abs2.(uhat) ./ (2 * prod(size(u))^2) # abs2.(uhat) ./ size(u, 1) end + (; A, κ, K) = spectral_stuff(setup) e = A * reshape(e, :) - e = max.(e, eps(T)) # Avoid log(0) - # ehat = (1:kmax) .* Array(e) + # e = max.(e, eps(T)) # Avoid log(0) ehat = Array(e) - (; kmax, ehat) + (; κ, ehat) end - (; kmax) = specs[1] + kmax = maximum(specs[1].κ) # Build inertial slope above energy if D == 2 # krange = [T(kmax)^(T(1) / 2), T(kmax)] # krange = [T(50), T(400)] - krange = [T(30), T(400)] + krange = [T(16), T(128)] elseif D == 3 - krange = [T(kmax)^(T(1.1) / 3), T(kmax)] + # krange = [T(kmax)^(T(1.5) / 3), T(kmax)] + krange = [T(64), T(256)] end slope, slopelabel = D == 2 ? (-T(3), L"$\kappa^{-3}") : (-T(5 / 3), L"$\kappa^{-5/3}") # slope, slopelabel = D == 2 ? (-T(3), "|k|⁻³") : (-T(5 / 3), "|k|⁻⁵³") # slope, slopelabel = D == 2 ? (-T(3), "κ⁻³") : (-T(5 / 3), "κ⁻⁵³") - slopeconst = maximum(specs[1].ehat ./ (1:kmax) .^ slope) - inertia = 2 .* slopeconst .* krange .^ slope + slopeconst = maximum(specs[1].ehat ./ specs[1].κ .^ slope) + offset = D == 2 ? 3 : 2 + inertia = offset .* slopeconst .* krange .^ slope # Nice ticks logmax = round(Int, log2(kmax + 1)) xticks = T(2) .^ (0:logmax) @@ -434,14 +439,15 @@ fig = with_theme(; palette = (; color = ["#3366cc", "#cc0000", "#669900", "#ffcc xticks, # xlabel = "k", xlabel = "κ", - ylabel = "e(κ)", + # ylabel = "e(κ)", xscale = log10, yscale = log10, limits = (1, kmax, T(1e-8), T(1)), - title = "Kinetic energy", + title = "Kinetic energy ($(D)D)", ) # plotparts(i) = 1:specs[i].kmax, specs[i].ehat - plotparts(i) = 1:specs[i].kmax+1, [specs[i].ehat; eps(T)] + # plotparts(i) = 1:specs[i].kmax+1, [specs[i].ehat; eps(T)] + plotparts(i) = specs[i].κ, specs[i].ehat # lines!(ax, 1:specs[1].kmax, specs[1].ehat; color = Cycled(1), label = "DNS") # lines!(ax, 1:specs[2].kmax, specs[2].ehat; color = Cycled(4), label = "DNS, t = 0") lines!(ax, plotparts(1)...; color = Cycled(1), label = "DNS") @@ -453,18 +459,23 @@ fig = with_theme(; palette = (; color = ["#3366cc", "#cc0000", "#669900", "#ffcc lines!(ax, plotparts(7)...; color = Cycled(3)) lines!(ax, plotparts(8)...; color = Cycled(3)) lines!(ax, krange, inertia; color = Cycled(1), label = slopelabel, linestyle = :dash) - axislegend(ax; position = :lb) + D == 2 && axislegend(ax; position = :lb) + D == 3 && axislegend(ax; position = :rt) autolimits!(ax) if D == 2 # limits!(ax, (T(0.68), T(520)), (T(1e-3), T(3e1))) - limits!(ax, (ax.xaxis.attributes.limits[][1], T(1000)), (T(1e-15), ax.yaxis.attributes.limits[][2])) + # limits!(ax, (ax.xaxis.attributes.limits[][1], T(1000)), (T(1e-15), ax.yaxis.attributes.limits[][2])) + limits!(ax, (T(0.8), T(800)), (T(1e-10), T(1))) elseif D == 3 # limits!(ax, ax.xaxis.attributes.limits[], (T(1e-1), T(3e1))) - limits!(ax, ax.xaxis.attributes.limits[], (T(5e-6), ax.yaxis.attributes.limits[][2])) + # limits!(ax, ax.xaxis.attributes.limits[], (T(1e-3), ax.yaxis.attributes.limits[][2])) + # limits!(ax, ax.xaxis.attributes.limits[], (T(3e-3), T(2e0))) + limits!(ax, (T(8e-1), T(400)), (T(2e-3), T(1.5e0))) end fig end GC.gc() CUDA.reclaim() -save("$output/priorspectra_$(D)D.pdf", fig) +# save("$output/priorspectra_$(D)D_linear.pdf", fig) +save("$output/priorspectra_$(D)D_dyadic.pdf", fig) diff --git a/scratch/divfree.jl b/scratch/divfree.jl index f9bbf3d28..9ad692ec1 100644 --- a/scratch/divfree.jl +++ b/scratch/divfree.jl @@ -10,6 +10,7 @@ end #src # Here, we consider a periodic box ``[0, 1]^2``. It is discretized with a # uniform Cartesian grid with square cells. +using Adapt using CairoMakie using GLMakie using IncompressibleNavierStokes @@ -33,7 +34,7 @@ output = "../SupervisedClosure/figures/" # Random number generator rng = Random.default_rng() -Random.seed!(rng, 123) +Random.seed!(rng, 12345) # Floating point precision T = Float64 @@ -48,10 +49,11 @@ device = identity using LuxCUDA using CUDA; +# T = Float64; T = Float32; ArrayType = CuArray; CUDA.allowscalar(false); -device = cu +device = x -> adapt(CuArray, x) # Parameters get_params(nlesscalar) = (; @@ -70,26 +72,26 @@ get_params(nlesscalar) = (; icfunc = (setup, psolver) -> random_field( setup, zero(eltype(setup.grid.x[1])); - A = 1, + # A = 1, kp = 20, psolver, ), ) -params_train = (; get_params([32, 64, 128, 256])..., savefreq = 10); -params_valid = (; get_params([32, 64, 128, 256])..., savefreq = 40); -params_test = (; get_params([32, 64, 128, 256, 512, 1024])..., tsim = T(0.2), savefreq = 20); +params_train = (; get_params([64, 128, 256])..., savefreq = 10); +params_valid = (; get_params([64, 128, 256])..., tsim = T(0.1), savefreq = 40); +params_test = (; get_params([64, 128, 256, 512, 1024])..., tsim = T(0.2), savefreq = 20); # Create LES data from DNS data_train = [create_les_data(; params_train...) for _ = 1:5]; data_valid = [create_les_data(; params_valid...) for _ = 1:1]; data_test = create_les_data(; params_test...); -# # Save filtered DNS data -# jldsave("output/forced/data.jld2"; data_train, data_valid, data_test) +# Save filtered DNS data +jldsave("output/divfree/data.jld2"; data_train, data_valid, data_test) # # Load previous LES data -# data_train, data_valid, data_test = load("output/forced/data.jld2", "data_train", "data_valid", "data_test") +# data_train, data_valid, data_test = load("output/divfree/data.jld2", "data_train", "data_valid", "data_test"); # Build LES setup and assemble operators getsetups(params) = [ @@ -111,22 +113,29 @@ data_train[1].data[1, 1].u[end][1] io_train = create_io_arrays(data_train, setups_train); io_valid = create_io_arrays(data_valid, setups_valid); +# jldsave("output/divfree/io_train.jld2"; io_train) +# jldsave("output/divfree/io_train.jld2"; io_valid) + +io_train[1].u |> extrema +io_train[1].c |> extrema +io_valid[1].u |> extrema +io_valid[1].c |> extrema + # Inspect data -ig = 4 -ifil = 2 +ig = 2 +ifil = 1 field, setup = data_train[1].data[ig, ifil].u, setups_train[ig]; -# field, setup = data_valid[1].u[ig], setups_valid[ig]; -# field, setup = data_test.u[ig], setups_test[ig]; -u = device(field[1]); +# field, setup = data_valid[1].data[ig, ifil].u, setups_valid[ig]; +# field, setup = data_test.data[ig, ifil], setups_test[ig]; +u = device.(field[1]); o = Observable((; u, t = nothing)); # energy_spectrum_plot(o; setup) fieldplot( o; setup, # fieldname = :velocity, - # fieldname = 2, + # fieldname = 1, ) -# energy_spectrum_plot( o; setup,) for i = 1:length(field) o[] = (; o[]..., u = device(field[i])) sleep(0.001) @@ -210,37 +219,27 @@ closure, θ₀ = cnn(; ); closure.chain -# Prepare training -loss = createloss(mean_squared_error, closure); -dataloaders = createdataloader.(io_train; batchsize = 50, device); -# dataloaders[1]() -loss(dataloaders[1](), device(θ₀)) - -# Prepare training -θ = T(1.0e-1) * device(θ₀); -opt = Optimisers.setup(Adam(T(1.0e-3)), θ); -callbackstate = Point2f[]; - # Train grid-specialized closure models θ_cnn = map(CartesianIndices(size(io_train))) do I # Prepare training ig, ifil = I.I @show ig ifil - d = createdataloader(io_train[ig, ifil]; batchsize = 50, device); - θ = T(1.0e-1) * device(θ₀) + d = create_dataloader_prior(io_train[ig, ifil]; batchsize = 50, device) + θ = T(1.0e0) * device(θ₀) + loss = createloss(mean_squared_error, closure); opt = Optimisers.setup(Adam(T(1.0e-3)), θ) callbackstate = Point2f[] - it = rand(1:size(io_valid[ig, ifil].u, 4), 50); - validset = map(v -> v[:, :, :, it], io_valid[ig, ifil]); + it = rand(1:size(io_valid[ig, ifil].u, 4), 50) + validset = map(v -> v[:, :, :, it], io_valid[ig, ifil]) (; opt, θ, callbackstate) = train( [d], loss, opt, θ; - niter = 5000, + niter = 10000, ncallback = 20, callbackstate, - callback = create_callback(closure, device(validset)...; state = callbackstate), + callback = create_relerr_prior(closure, validset), ) θ end @@ -248,11 +247,57 @@ GC.gc() CUDA.reclaim() # # Save trained parameters -# jldsave("output/divfree/theta_cnn.jld2"; theta = Array.(θ_cnn)) +# jldsave("output/divfree/theta_prior.jld2"; theta = Array.(θ_cnn)) # # Load trained parameters -# θθ = load("output/divfree/theta_cnn.jld2") -# copyto!.(θ_cnn, θθ["theta"]) +# θ_cnn = [device(θ₀) for _ in CartesianIndices(size(data_train[1].data))]; +# θθ = load("output/divfree/theta_prior.jld2"); +# copyto!.(θ_cnn, θθ["theta"]); + +# θ_post = map(CartesianIndices(size(io_train))) do I +# ig, ifil = I.I +θ_post = let ig = 2, ifil = 1 + iorder = 1 + setup = setups_train[ig] + psolver = SpectralPressureSolver(setup) + loss = IncompressibleNavierStokes.create_loss_post(; + setup, + psolver, + closure, + nupdate = 4, + unproject_closure = iorder == 2, + ) + data = [(; u = d.data[ig, ifil].u, d.t) for d in data_train] + d = create_dataloader_post(data; device, nunroll = 10) + # θ = copy(θ_cnn[ig, ifil]) + θ = device(θ₀) + opt = Optimisers.setup(Adam(T(1.0e-3)), θ) + callbackstate = Point2f[] + it = rand(1:size(io_valid[ig, ifil].u, 4), 50) + data = data_valid[1] + data = (; u = device.(data.data[ig, ifil].u), data.t) + (; opt, θ, callbackstate) = train( + [d], + loss, + opt, + θ; + niter = 100, + ncallback = 1, + callbackstate, + callback = create_callback( + create_relerr_post(; + data, + setup, + psolver, + closure, + unproject_closure = iorder == 2, + ); + state = callbackstate, + displayref = false, + ), + ) + θ +end # Train Smagorinsky closure model ig = 2; @@ -285,39 +330,79 @@ e_smag, e_cnn = let setup = setups_test[ig] params = params_test[ig] psolver = SpectralPressureSolver(setup) - u = device.(data_test.data[ig,ifil].u) - u₀ = device(data_test.data[ig,ifil].u[1]) + u = device.(data_test.data[ig, ifil].u) + u₀ = device(data_test.data[ig, ifil].u[1]) Δt = params_test.Δt * params_test.savefreq tlims = extrema(data_test.t) - nupdate = 4 + nupdate = 20 Δt /= nupdate processors = (; relerr = relerr_trajectory(u, setup; nupdate)) # Smagorinsky - closedsetup = (; setup..., + closedsetup = (; + setup..., closure_model = smagorinsky_closure(setup), unproject_closure = iorder == 2, ) - _, outputs = solve_unsteady( - closedsetup, u₀, tlims; Δt, psolver, processors, - θ = T(0.1), - ) + _, outputs = + solve_unsteady(closedsetup, u₀, tlims; Δt, psolver, processors, θ = T(0.1)) e_smag[ig, ifil, iorder] = outputs.relerr[] # CNN - closedsetup = (; setup..., - closure_model = wrappedclosure(closure, setup), - unproject_closure = iorder == 2, - ) - _, outputs = solve_unsteady( - closedsetup, u₀, tlims; Δt, psolver, processors, - θ = θ_cnn[ig, ifil], - ) - e_cnn[ig, ifil, iorder] = outputs.relerr[] + # The first grids are trained for + if ig ≤ size(data_train[1].data, 1) + closedsetup = (; + setup..., + closure_model = wrappedclosure(closure, setup), + unproject_closure = iorder == 2, + ) + _, outputs = solve_unsteady( + closedsetup, + u₀, + tlims; + Δt, + psolver, + processors, + θ = θ_cnn[ig, ifil], + ) + e_cnn[ig, ifil, iorder] = outputs.relerr[] + end end e_smag, e_cnn end e_smag e_cnn +# julia> e_smag julia> e_smag +# 5×2×2 Array{Float32, 3}: 5×2×2 Array{Float32, 3}: +# [:, :, 1] = [:, :, 1] = +# 0.781462 0.778116 0.781462 0.778104 +# 0.505524 0.505589 0.505524 0.505565 +# 0.30132 0.301625 0.30132 0.301601 +# 0.143863 0.144141 0.143863 0.144118 +# 0.0457445 0.0459661 0.0457429 0.0459541 +# +# [:, :, 2] = [:, :, 2] = +# 0.813456 0.826474 0.813457 0.826474 +# 0.527202 0.534727 0.527203 0.534726 +# 0.318339 0.32539 0.318339 0.325389 +# 0.152291 0.157393 0.152292 0.157392 +# 0.0482445 0.051005 0.0482446 0.0510034 +# +# julia> e_cnn julia> e_cnn +# 5×2×2 Array{Float32, 3}: 5×2×2 Array{Float32, 3}: +# [:, :, 1] = [:, :, 1] = +# 0.585886 0.5875 0.585887 0.5875 +# 0.311889 0.277596 0.311892 0.277624 +# 0.0725899 0.114744 0.0725901 0.114847 +# 0.0 0.0 0.0 0.0 +# 0.0 0.0 0.0 0.0 +# +# [:, :, 2] = [:, :, 2] = +# 0.783691 0.859665 0.783688 0.859664 +# 1.81263 NaN 1.78943 6.52473 +# NaN NaN NaN NaN +# 0.0 0.0 0.0 0.0 +# 0.0 0.0 0.0 0.0 + # No model e_nm = let e_nm = zeros(T, size(data_test.data)...) @@ -326,15 +411,20 @@ e_nm = let setup = setups_test[ig] params = params_test[ig] psolver = SpectralPressureSolver(setup) - u = device.(data_test.data[ig,ifil].u) - u₀ = device(data_test.data[ig,ifil].u[1]) + u = device.(data_test.data[ig, ifil].u) + u₀ = device(data_test.data[ig, ifil].u[1]) Δt = params_test.Δt * params_test.savefreq tlims = extrema(data_test.t) - nupdate = 4 + nupdate = 20 Δt /= nupdate processors = (; relerr = relerr_trajectory(u, setup; nupdate)) _, outputs = solve_unsteady( - (; setup..., unproject_closure = false), u₀, tlims; Δt, psolver, processors, + (; setup..., unproject_closure = false), + u₀, + tlims; + Δt, + psolver, + processors, ) e_nm[ig, ifil] = outputs.relerr[] end @@ -360,7 +450,8 @@ with_theme(; # fontsize = 20, palette = (; color = ["#3366cc", "#cc0000", "#669900", "#ffcc00"]), ) do - iorder = 1 + iorder = 2 + lesmodel = iorder == 1 ? "closure-then-project" : "project-then-closure" nles = [n[1] for n in params_test.nles] fig = Figure(; size = (500, 400)) ax = Axis( @@ -368,38 +459,72 @@ with_theme(; xscale = log10, yscale = log10, xticks = nles, - xlabel = "n", - title = "Relative error (DNS: n = $(params_test.ndns[1]))", + xlabel = "Resolution", + # xlabel = "n", + # xlabel = L"\bar{n}", + # title = "Relative error (DNS: n = $(params_test.ndns[1]))", + title = "Relative error ($lesmodel)", ) for ifil = 1:2 linestyle = ifil == 1 ? :solid : :dash - label = "No closure" + label = "No closure" label = label * (ifil == 1 ? " (FA)" : " (VA)") - scatterlines!(nles, e_nm[:, ifil]; color = Cycled(1), linestyle, marker = :circle, label) + scatterlines!( + nles, + e_nm[:, ifil]; + color = Cycled(1), + linestyle, + marker = :circle, + label, + ) end for ifil = 1:2 linestyle = ifil == 1 ? :solid : :dash label = "Smagorinsky" label = label * (ifil == 1 ? " (FA)" : " (VA)") - scatterlines!(nles, e_smag[:, ifil, iorder]; color = Cycled(2), linestyle, marker = :utriangle, label) + scatterlines!( + nles, + e_smag[:, ifil, iorder]; + color = Cycled(2), + linestyle, + marker = :utriangle, + label, + ) end for ifil = 1:2 + ntrain = size(data_train[1].data, 1) linestyle = ifil == 1 ? :solid : :dash - label = "CNN" + label = "CNN" label = label * (ifil == 1 ? " (FA)" : " (VA)") - scatterlines!(nles, e_cnn[:, ifil, iorder]; color = Cycled(3), linestyle, marker = :rect, label) + # ifil == 2 && (label = nothing) + scatterlines!( + nles[1:ntrain], + e_cnn[1:ntrain, ifil, iorder]; + color = Cycled(3), + linestyle, + marker = :rect, + label, + ) end # lines!( - # collect(extrema(nles[3:end])), + # collect(extrema(nles[4:end])), # n -> 2e4 * n^-2.0; # linestyle = :dash, # label = "n⁻²", + # color = Cycled(1), # ) - axislegend(; position = :lb) + axislegend(; position = :rt) + # iorder == 2 && limits!(ax, (T(60), T(1050)), (T(2e-2), T(1e1))) fig end -save("$output/convergence.pdf", current_figure()) +e_cnn +e_cnn_2 = copy(e_cnn); +e_cnn_2[isnan.(e_cnn_2)] .= 1e10; +e_cnn_2; + +save("$output/convergence_Lprior_prosecond.pdf", current_figure()) +save("$output/convergence_Lprior_profirst.pdf", current_figure()) markers_labels = [ (:circle, ":circle"), From 346ec8bd9e3c31a6d56aa2244cbaef08775a51b4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Mon, 29 Jan 2024 00:13:38 +0100 Subject: [PATCH 241/379] fix: Add missing zero in BC pullback and do chain --- scratch/testgrad.jl | 64 ++++++++++++++++++-------------------- src/boundary_conditions.jl | 57 ++++++++++++++++----------------- 2 files changed, 60 insertions(+), 61 deletions(-) diff --git a/scratch/testgrad.jl b/scratch/testgrad.jl index 08feec54a..1d9826e4e 100644 --- a/scratch/testgrad.jl +++ b/scratch/testgrad.jl @@ -5,29 +5,16 @@ if isdefined(@__MODULE__, :LanguageServer) #src using .IncompressibleNavierStokes #src end #src -# # Decaying Homogeneous Isotropic Turbulence - 2D -# -# In this example we consider decaying homogeneous isotropic turbulence, -# similar to the cases considered in [Kochkov2021](@cite) and -# [Kurz2022](@cite). The initial velocity field is created randomly, but with a -# specific energy spectrum. Due to viscous dissipation, the turbulent features -# eventually group to form larger visible eddies. - -# We start by loading packages. -# A [Makie](https://github.com/JuliaPlots/Makie.jl) plotting backend is needed -# for plotting. `GLMakie` creates an interactive window (useful for real-time -# plotting), but does not work when building this example on GitHub. -# `CairoMakie` makes high-quality static vector-graphics plots. - #md using CairoMakie using GLMakie #!md using IncompressibleNavierStokes +using KernelAbstractions +using Zygote +using LinearAlgebra +using Random set_theme!(; GLMakie = (; scalefactor = 1.5)) -# Output directory -output = "output/DecayingTurbulence2D" - # Floating point precision T = Float64 @@ -44,10 +31,11 @@ ArrayType = CuArray; CUDA.allowscalar(false); # Viscosity model -Re = T(10_000) +Re = T(100) # A 2D grid is a Cartesian product of two vectors -n = 8 +# n = 8 +n = 16 # n = 32 # n = 128 # n = 1024 @@ -61,23 +49,19 @@ setup = Setup(x...; Re, ArrayType); # Since the grid is uniform and identical for x and y, we may use a specialized # spectral pressure solver -pressure_solver = SpectralPressureSolver(setup); +psolver = SpectralPressureSolver(setup); -u₀ = random_field(setup, T(0); pressure_solver); +u₀ = random_field(setup, T(0); psolver); u = u₀ -using KernelAbstractions -using Zygote -using LinearAlgebra -using Random (; Iu, Ip) = setup.grid -function finitediff(f, u::Tuple, I; h = sqrt(eps(eltype(u[1])))) +function finitediff(f, u::Tuple, α, I; h = sqrt(eps(eltype(u[1])))) u1 = copy.(u) - CUDA.@allowscalar u1[1][I] -= h / 2 + CUDA.@allowscalar u1[α][I] -= h / 2 r1 = f(u1) u2 = copy.(u) - CUDA.@allowscalar u2[1][I] += h / 2 + CUDA.@allowscalar u2[α][I] += h / 2 r2 = f(u2) (r2 - r1) / h end @@ -130,33 +114,42 @@ function f(u, setup) φ = IncompressibleNavierStokes.momentum(u, T(0), setup) # φ = IncompressibleNavierStokes.diffusion(u, setup) # φ = IncompressibleNavierStokes.convection(u, setup) + # φ = u # dot(φ, φ) dot(getindex.(φ, Iu), getindex.(φ, Iu)) # sum(abs2, getindex.(φ, Iu)) + # u[1][1] end -solver = SpectralPressureSolver(setup) +I = CartesianIndex(1, 2) +α = 1 +CUDA.@allowscalar gradient(u -> f(u, setup), u)[1][α][I] +finitediff(u -> f(u, setup), u, α, I) + function f(u, setup) (; Ω, Iu) = setup.grid φ = u φ = IncompressibleNavierStokes.apply_bc_u(u, T(0), setup) φ = IncompressibleNavierStokes.momentum(φ, T(0), setup) φ = IncompressibleNavierStokes.apply_bc_u(φ, T(0), setup) - φ = IncompressibleNavierStokes.project(solver, φ, setup) + φ = IncompressibleNavierStokes.project(φ, setup; psolver) # dot(φ, φ) dot(getindex.(φ, Iu), getindex.(φ, Iu)) # sum(abs2, getindex.(φ, Iu)) end -pressure_solver = SpectralPressureSolver(setup) -# method = +I = CartesianIndex(8, 2) +α = 1 +CUDA.@allowscalar gradient(u -> f(u, setup), u)[1][α][I] +finitediff(u -> f(u, setup), u, α, I) + function f(u, setup) (; Ω, Iu) = setup.grid method = RK44(; T) stepper = IncompressibleNavierStokes.create_stepper( method; setup, - pressure_solver, + psolver, u, t = T(0), ) @@ -167,6 +160,11 @@ function f(u, setup) # sum(abs2, getindex.(φ, Iu)) end +I = CartesianIndex(5, 6) +α = 1 +CUDA.@allowscalar gradient(u -> f(u, setup), u)[1][α][I] +finitediff(u -> f(u, setup), u, α, I) + function f(u, setup) (; Iu) = setup.grid φ = IncompressibleNavierStokes.tupleadd(u, u) diff --git a/src/boundary_conditions.jl b/src/boundary_conditions.jl index 51e6cb70a..75d4cf383 100644 --- a/src/boundary_conditions.jl +++ b/src/boundary_conditions.jl @@ -113,7 +113,7 @@ ChainRulesCore.rrule(::typeof(apply_bc_u), u, t, setup; kwargs...) = ( # With respect to (apply_bc_u, u, t, setup) φbar -> ( NoTangent(), - apply_bc_u_pullback!(copy.((φbar...,)), (φbar...,), t, setup; kwargs...), + apply_bc_u_pullback!(copy.((φbar...,)), t, setup; kwargs...), NoTangent(), NoTangent(), ), @@ -124,7 +124,7 @@ ChainRulesCore.rrule(::typeof(apply_bc_p), p, t, setup) = ( # With respect to (apply_bc_p, p, t, setup) φbar -> ( NoTangent(), - apply_bc_p_pullback!(copy(φbar), φbar, t, setup), + apply_bc_p_pullback!(copy(φbar), t, setup), NoTangent(), NoTangent(), ), @@ -140,13 +140,13 @@ function apply_bc_u!(u, t, setup; kwargs...) u end -function apply_bc_u_pullback!(ubar, φbar, t, setup; kwargs...) - (; boundary_conditions) = setup - D = length(ubar) +function apply_bc_u_pullback!(φbar, t, setup; kwargs...) + (; grid, boundary_conditions) = setup + (; dimension) = grid + D = dimension() for β = 1:D apply_bc_u_pullback!( boundary_conditions[β][1], - ubar, φbar, β, t, @@ -156,7 +156,6 @@ function apply_bc_u_pullback!(ubar, φbar, t, setup; kwargs...) ) apply_bc_u_pullback!( boundary_conditions[β][2], - ubar, φbar, β, t, @@ -165,7 +164,7 @@ function apply_bc_u_pullback!(ubar, φbar, t, setup; kwargs...) kwargs..., ) end - ubar + φbar end function apply_bc_p!(p, t, setup; kwargs...) @@ -179,14 +178,13 @@ function apply_bc_p!(p, t, setup; kwargs...) p end -function apply_bc_p_pullback!(pbar, φbar, t, setup; kwargs...) +function apply_bc_p_pullback!(φbar, t, setup; kwargs...) (; grid, boundary_conditions) = setup (; dimension) = grid D = dimension() for β = 1:D apply_bc_p_pullback!( boundary_conditions[β][1], - pbar, φbar, β, t, @@ -195,7 +193,6 @@ function apply_bc_p_pullback!(pbar, φbar, t, setup; kwargs...) ) apply_bc_p_pullback!( boundary_conditions[β][2], - pbar, φbar, β, t, @@ -203,7 +200,7 @@ function apply_bc_p_pullback!(pbar, φbar, t, setup; kwargs...) atend = true, ) end - pbar + φbar end function apply_bc_u!(::PeriodicBC, u, β, t, setup; atend, kwargs...) @@ -230,28 +227,30 @@ function apply_bc_u!(::PeriodicBC, u, β, t, setup; atend, kwargs...) u end -function apply_bc_u_pullback!(::PeriodicBC, ubar, φbar, β, t, setup; atend, kwargs...) +function apply_bc_u_pullback!(::PeriodicBC, φbar, β, t, setup; atend, kwargs...) (; grid, workgroupsize) = setup (; dimension, N) = grid D = dimension() δ = Offset{D}() - @kernel function adj_a!(u, φ, ::Val{α}, ::Val{β}) where {α,β} + @kernel function adj_a!(φ, ::Val{α}, ::Val{β}) where {α,β} I = @index(Global, Cartesian) - u[α][I+(N[β]-2)*δ(β)] += φ[α][I] + φ[α][I+(N[β]-2)*δ(β)] += φ[α][I] + φ[α][I] = 0 end - @kernel function adj_b!(u, φ, ::Val{α}, ::Val{β}) where {α,β} + @kernel function adj_b!(φ, ::Val{α}, ::Val{β}) where {α,β} I = @index(Global, Cartesian) - u[α][I+δ(β)] += φ[α][I+(N[β]-1)*δ(β)] + φ[α][I+δ(β)] += φ[α][I+(N[β]-1)*δ(β)] + φ[α][I+(N[β]-1)*δ(β)] = 0 end ndrange = ntuple(γ -> γ == β ? 1 : N[γ], D) for α = 1:D if atend - adj_b!(get_backend(ubar[1]), workgroupsize)(ubar, φbar, Val(α), Val(β); ndrange) + adj_b!(get_backend(φbar[1]), workgroupsize)(φbar, Val(α), Val(β); ndrange) else - adj_a!(get_backend(ubar[1]), workgroupsize)(ubar, φbar, Val(α), Val(β); ndrange) + adj_a!(get_backend(φbar[1]), workgroupsize)(φbar, Val(α), Val(β); ndrange) end end - ubar + φbar end function apply_bc_p!(::PeriodicBC, p, β, t, setup; atend, kwargs...) @@ -276,26 +275,28 @@ function apply_bc_p!(::PeriodicBC, p, β, t, setup; atend, kwargs...) p end -function apply_bc_p_pullback!(::PeriodicBC, pbar, φbar, β, t, setup; atend, kwargs...) +function apply_bc_p_pullback!(::PeriodicBC, φbar, β, t, setup; atend, kwargs...) (; grid, workgroupsize) = setup (; dimension, N) = grid D = dimension() δ = Offset{D}() - @kernel function adj_a!(p, φ, ::Val{β}) where {β} + @kernel function adj_a!(φ, ::Val{β}) where {β} I = @index(Global, Cartesian) - p[I+(N[β]-2)*δ(β)] += φ[I] + φ[I+(N[β]-2)*δ(β)] += φ[I] + φ[I] = 0 end - @kernel function adj_b!(p, φ, ::Val{β}) where {β} + @kernel function adj_b!(φ, ::Val{β}) where {β} I = @index(Global, Cartesian) - p[I+δ(β)] += φ[I+(N[β]-1)*δ(β)] + φ[I+δ(β)] += φ[I+(N[β]-1)*δ(β)] + φ[I+(N[β]-1)*δ(β)] = 0 end ndrange = ntuple(γ -> γ == β ? 1 : N[γ], D) if atend - adj_b!(get_backend(pbar), workgroupsize)(pbar, φbar, Val(β); ndrange) + adj_b!(get_backend(φbar), workgroupsize)(φbar, Val(β); ndrange) else - adj_a!(get_backend(pbar), workgroupsize)(pbar, φbar, Val(β); ndrange) + adj_a!(get_backend(φbar), workgroupsize)(φbar, Val(β); ndrange) end - pbar + φbar end function apply_bc_u!(bc::DirichletBC, u, β, t, setup; atend, dudt = false, kwargs...) From 1e002227b2ef54d94bf686a23fb450d18998a9ee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Mon, 29 Jan 2024 21:17:23 +0100 Subject: [PATCH 242/379] Update files --- scratch/divfree.jl | 513 ++++++------------ src/closures/training.jl | 29 +- src/operators.jl | 20 +- src/processors/real_time_plot.jl | 6 +- src/setup.jl | 4 +- .../step_explicit_runge_kutta.jl | 53 +- 6 files changed, 228 insertions(+), 397 deletions(-) diff --git a/scratch/divfree.jl b/scratch/divfree.jl index 9ad692ec1..b69d7d9f7 100644 --- a/scratch/divfree.jl +++ b/scratch/divfree.jl @@ -26,6 +26,17 @@ using SparseArrays using KernelAbstractions using FFTW +getorder(i) = + if i == 1 + :first + elseif i == 2 + :second + elseif i == 3 + :last + else + error("Unknown order: $i") + end + GLMakie.activate!() set_theme!(; GLMakie = (; scalefactor = 1.5)) @@ -42,6 +53,7 @@ T = Float64 # Array type ArrayType = Array device = identity +clean() = nothing ## using CUDA; ArrayType = CuArray ## using AMDGPU; ArrayType = ROCArray ## using oneAPI; ArrayType = oneArray @@ -54,6 +66,7 @@ T = Float32; ArrayType = CuArray; CUDA.allowscalar(false); device = x -> adapt(CuArray, x) +clean() = (GC.gc(); CUDA.reclaim()) # Parameters get_params(nlesscalar) = (; @@ -88,10 +101,14 @@ data_valid = [create_les_data(; params_valid...) for _ = 1:1]; data_test = create_les_data(; params_test...); # Save filtered DNS data -jldsave("output/divfree/data.jld2"; data_train, data_valid, data_test) +jldsave("output/divfree/data_train.jld2"; data_train) +jldsave("output/divfree/data_valid.jld2"; data_valid) +jldsave("output/divfree/data_test.jld2"; data_test) -# # Load previous LES data -# data_train, data_valid, data_test = load("output/divfree/data.jld2", "data_train", "data_valid", "data_test"); +# Load filtered DNS data +data_train = load("output/divfree/data.jld2", "data_train"); +data_valid = load("output/divfree/data.jld2", "data_valid"); +data_test = load("output/divfree/data.jld2", "data_test"); # Build LES setup and assemble operators getsetups(params) = [ @@ -122,23 +139,25 @@ io_valid[1].u |> extrema io_valid[1].c |> extrema # Inspect data -ig = 2 -ifil = 1 -field, setup = data_train[1].data[ig, ifil].u, setups_train[ig]; -# field, setup = data_valid[1].data[ig, ifil].u, setups_valid[ig]; -# field, setup = data_test.data[ig, ifil], setups_test[ig]; -u = device.(field[1]); -o = Observable((; u, t = nothing)); -# energy_spectrum_plot(o; setup) -fieldplot( - o; - setup, - # fieldname = :velocity, - # fieldname = 1, -) -for i = 1:length(field) - o[] = (; o[]..., u = device(field[i])) - sleep(0.001) +let + ig = 2 + ifil = 1 + field, setup = data_train[1].data[ig, ifil].u, setups_train[ig] + # field, setup = data_valid[1].data[ig, ifil].u, setups_valid[ig]; + # field, setup = data_test.data[ig, ifil], setups_test[ig]; + u = device.(field[1]) + o = Observable((; u, t = nothing)) + # energy_spectrum_plot(o; setup) |> display + fieldplot( + o; + setup, + # fieldname = :velocity, + # fieldname = 1, + ) |> display + for i = 1:length(field) + o[] = (; o[]..., u = device(field[i])) + sleep(0.001) + end end GLMakie.activate!() @@ -226,7 +245,7 @@ closure.chain @show ig ifil d = create_dataloader_prior(io_train[ig, ifil]; batchsize = 50, device) θ = T(1.0e0) * device(θ₀) - loss = createloss(mean_squared_error, closure); + loss = createloss(mean_squared_error, closure) opt = Optimisers.setup(Adam(T(1.0e-3)), θ) callbackstate = Point2f[] it = rand(1:size(io_valid[ig, ifil].u, 4), 50) @@ -243,21 +262,20 @@ closure.chain ) θ end -GC.gc() -CUDA.reclaim() +clean() -# # Save trained parameters -# jldsave("output/divfree/theta_prior.jld2"; theta = Array.(θ_cnn)) +# Save trained parameters +jldsave("output/divfree/theta_prior.jld2"; theta = Array.(θ_cnn)) -# # Load trained parameters -# θ_cnn = [device(θ₀) for _ in CartesianIndices(size(data_train[1].data))]; -# θθ = load("output/divfree/theta_prior.jld2"); -# copyto!.(θ_cnn, θθ["theta"]); +# Load trained parameters +θ_cnn = [device(θ₀) for _ in CartesianIndices(size(data_train[1].data))]; +θθ = load("output/divfree/theta_prior.jld2"); +copyto!.(θ_cnn, θθ["theta"]); -# θ_post = map(CartesianIndices(size(io_train))) do I -# ig, ifil = I.I -θ_post = let ig = 2, ifil = 1 - iorder = 1 +# θ_cnn_post = let ig = 3, ifil = 2, iorder = 2 +θ_cnn_post = map(CartesianIndices((size(io_train)..., 2))) do I + ig, ifil, iorder = I.I + println("iorder = $iorder, ifil = $ifil, ig = $ig") setup = setups_train[ig] psolver = SpectralPressureSolver(setup) loss = IncompressibleNavierStokes.create_loss_post(; @@ -265,179 +283,152 @@ CUDA.reclaim() psolver, closure, nupdate = 4, - unproject_closure = iorder == 2, + projectorder = getorder(iorder), ) data = [(; u = d.data[ig, ifil].u, d.t) for d in data_train] - d = create_dataloader_post(data; device, nunroll = 10) - # θ = copy(θ_cnn[ig, ifil]) - θ = device(θ₀) + d = create_dataloader_post(data; device, nunroll = 20) + # θ = T(1e-1) * copy(θ_cnn[ig, ifil]) + θ = copy(θ_cnn[ig, ifil]) + # θ = copy(θ_cnn_post) + # θ = device(θ₀) opt = Optimisers.setup(Adam(T(1.0e-3)), θ) callbackstate = Point2f[] - it = rand(1:size(io_valid[ig, ifil].u, 4), 50) + it = 1:20 data = data_valid[1] - data = (; u = device.(data.data[ig, ifil].u), data.t) + data = (; u = device.(data.data[ig, ifil].u[it]), t = data.t[it]) (; opt, θ, callbackstate) = train( [d], loss, opt, θ; - niter = 100, - ncallback = 1, + niter = 200, + ncallback = 10, callbackstate, callback = create_callback( create_relerr_post(; data, setup, psolver, - closure, - unproject_closure = iorder == 2, + closure_model = wrappedclosure(closure, setup), + projectorder = getorder(iorder), + nupdate = 8, ); state = callbackstate, displayref = false, ), ) + jldsave( + "output/divfree/theta_post_iorder$(iorder)_ifil$(ifil)_ig$(ig).jld2"; + theta = Array(θ), + ) + clean() θ +end; +clean() + +θ_cnn[2, 2] |> extrema + +Array(θ_post) + +# Train Smagorinsky model with Lpost (grid search) +θ_smag = map(CartesianIndices((size(io_train, 2), 2))) do I + ifil, iorder = I.I + ngrid = size(io_train, 1) + θmin = T(0) + emin = T(Inf) + isample = 1 + it = 1:50 + for θ in LinRange(T(0), T(0.5), 51) + e = T(0) + for igrid = 1:ngrid + println("iorder = $iorder, ifil = $ifil, θ = $θ, igrid = $igrid") + projectorder = getorder(iorder) + setup = setups_train[igrid] + psolver = SpectralPressureSolver(setup) + d = data_train[isample] + data = (; u = device.(d.data[igrid, ifil].u[it]), t = d.t[it]) + nupdate = 4 + err = create_relerr_post(; + data, + setup, + psolver, + closure_model = smagorinsky_closure(setup), + projectorder, + nupdate, + ) + e += err(θ) + end + e /= ngrid + if e < emin + emin = e + θmin = θ + end + end + θmin end +clean() +θ_smag() -# Train Smagorinsky closure model -ig = 2; -setup = setups_train[ig]; -sample = data_train[1]; -m = smagorinsky_closure(setup); -θ = T(0.05) -e_smag = sum(2:length(sample.t)) do it - It = setup.grid.Ip - u = sample.u[ig][it] |> device - c = sample.c[ig][it] |> device - mu = m(u, θ) - e = zero(eltype(u[1])) - for α = 1:D - # e += sum(abs2, mu[α][Ip] .- c[α][Ip]) / sum(abs2, c[α][Ip]) - e += norm(mu[α][Ip] .- c[α][Ip]) / norm(c[α][Ip]) - end - e / D -end / length(sample.t) -# for θ in LinRange(T(0), T(1), 100)]; +# Save trained parameters +jldsave("output/divfree/theta_smag.jld2"; theta = θ_smag); + +# Load trained parameters +θ_smag = load("output/divfree/theta_smag.jld2")["theta"]; +θ_smag # lines(LinRange(T(0), T(1), 100), e_smag) # Errors for grid-specialized closure models -e_smag, e_cnn = let +e_nm, e_smag, e_cnn, e_cnn_post = let + e_nm = zeros(T, size(data_test.data)...) e_smag = zeros(T, size(data_test.data)..., 2) e_cnn = zeros(T, size(data_test.data)..., 2) + e_cnn_post = zeros(T, size(data_test.data)..., 2) for iorder = 1:2, ifil = 1:2, ig = 1:size(data_test.data, 1) - @show iorder ifil ig + println("iorder = $iorder, ifil = $ifil, ig = $ig") + projectorder = getorder(iorder) setup = setups_test[ig] - params = params_test[ig] psolver = SpectralPressureSolver(setup) - u = device.(data_test.data[ig, ifil].u) - u₀ = device(data_test.data[ig, ifil].u[1]) - Δt = params_test.Δt * params_test.savefreq - tlims = extrema(data_test.t) - nupdate = 20 - Δt /= nupdate - processors = (; relerr = relerr_trajectory(u, setup; nupdate)) + data = (; u = device.(data_test.data[ig, ifil].u), t = data_test.t) + nupdate = 50 + # No model + # Only for closurefirst, since projectfirst is the same + if iorder == 1 + err = create_relerr_post(; data, setup, psolver, closure_model = nothing, nupdate) + e_nm[ig, ifil] = err(nothing) + end # Smagorinsky - closedsetup = (; - setup..., + err = create_relerr_post(; + data, + setup, + psolver, closure_model = smagorinsky_closure(setup), - unproject_closure = iorder == 2, + projectorder, + nupdate, ) - _, outputs = - solve_unsteady(closedsetup, u₀, tlims; Δt, psolver, processors, θ = T(0.1)) - e_smag[ig, ifil, iorder] = outputs.relerr[] + e_smag[ig, ifil, iorder] = err(θ_smag[ifil, iorder]) # CNN - # The first grids are trained for + # Only the first grids are trained for if ig ≤ size(data_train[1].data, 1) - closedsetup = (; - setup..., - closure_model = wrappedclosure(closure, setup), - unproject_closure = iorder == 2, - ) - _, outputs = solve_unsteady( - closedsetup, - u₀, - tlims; - Δt, + err = create_relerr_post(; + data, + setup, psolver, - processors, - θ = θ_cnn[ig, ifil], + closure_model = wrappedclosure(closure, setup), + projectorder, + nupdate, ) - e_cnn[ig, ifil, iorder] = outputs.relerr[] + e_cnn[ig, ifil, iorder] = err(θ_cnn[ig, ifil]) + e_cnn_post[ig, ifil, iorder] = err(θ_cnn_post[ig, ifil, iorder]) end end - e_smag, e_cnn -end -e_smag -e_cnn - -# julia> e_smag julia> e_smag -# 5×2×2 Array{Float32, 3}: 5×2×2 Array{Float32, 3}: -# [:, :, 1] = [:, :, 1] = -# 0.781462 0.778116 0.781462 0.778104 -# 0.505524 0.505589 0.505524 0.505565 -# 0.30132 0.301625 0.30132 0.301601 -# 0.143863 0.144141 0.143863 0.144118 -# 0.0457445 0.0459661 0.0457429 0.0459541 -# -# [:, :, 2] = [:, :, 2] = -# 0.813456 0.826474 0.813457 0.826474 -# 0.527202 0.534727 0.527203 0.534726 -# 0.318339 0.32539 0.318339 0.325389 -# 0.152291 0.157393 0.152292 0.157392 -# 0.0482445 0.051005 0.0482446 0.0510034 -# -# julia> e_cnn julia> e_cnn -# 5×2×2 Array{Float32, 3}: 5×2×2 Array{Float32, 3}: -# [:, :, 1] = [:, :, 1] = -# 0.585886 0.5875 0.585887 0.5875 -# 0.311889 0.277596 0.311892 0.277624 -# 0.0725899 0.114744 0.0725901 0.114847 -# 0.0 0.0 0.0 0.0 -# 0.0 0.0 0.0 0.0 -# -# [:, :, 2] = [:, :, 2] = -# 0.783691 0.859665 0.783688 0.859664 -# 1.81263 NaN 1.78943 6.52473 -# NaN NaN NaN NaN -# 0.0 0.0 0.0 0.0 -# 0.0 0.0 0.0 0.0 - -# No model -e_nm = let - e_nm = zeros(T, size(data_test.data)...) - for ifil = 1:2, ig = 1:size(data_test.data, 1) - @show ifil ig - setup = setups_test[ig] - params = params_test[ig] - psolver = SpectralPressureSolver(setup) - u = device.(data_test.data[ig, ifil].u) - u₀ = device(data_test.data[ig, ifil].u[1]) - Δt = params_test.Δt * params_test.savefreq - tlims = extrema(data_test.t) - nupdate = 20 - Δt /= nupdate - processors = (; relerr = relerr_trajectory(u, setup; nupdate)) - _, outputs = solve_unsteady( - (; setup..., unproject_closure = false), - u₀, - tlims; - Δt, - psolver, - processors, - ) - e_nm[ig, ifil] = outputs.relerr[] - end - e_nm + e_nm, e_smag, e_cnn, e_cnn_post end -e_nm - -GC.gc() -CUDA.reclaim() - +clean() e_nm e_smag e_cnn +e_cnn_post CairoMakie.activate!() GLMakie.activate!() @@ -450,7 +441,7 @@ with_theme(; # fontsize = 20, palette = (; color = ["#3366cc", "#cc0000", "#669900", "#ffcc00"]), ) do - iorder = 2 + iorder = 1 lesmodel = iorder == 1 ? "closure-then-project" : "project-then-closure" nles = [n[1] for n in params_test.nles] fig = Figure(; size = (500, 400)) @@ -506,6 +497,21 @@ with_theme(; label, ) end + for ifil = 1:2 + ntrain = size(data_train[1].data, 1) + linestyle = ifil == 1 ? :solid : :dash + label = "CNN (post)" + label = label * (ifil == 1 ? " (FA)" : " (VA)") + # ifil == 2 && (label = nothing) + scatterlines!( + nles[1:ntrain], + e_cnn_post[1:ntrain, ifil, iorder]; + color = Cycled(4), + linestyle, + marker = :diamond, + label, + ) + end # lines!( # collect(extrema(nles[4:end])), # n -> 2e4 * n^-2.0; @@ -518,11 +524,6 @@ with_theme(; fig end -e_cnn -e_cnn_2 = copy(e_cnn); -e_cnn_2[isnan.(e_cnn_2)] .= 1e10; -e_cnn_2; - save("$output/convergence_Lprior_prosecond.pdf", current_figure()) save("$output/convergence_Lprior_profirst.pdf", current_figure()) @@ -551,186 +552,6 @@ markers_labels = [ ('✈', "'\\:airplane:'"), ] -# Final spectra -ig = 4 -setup = setups_test[ig]; -params = params_test -pressure_solver = SpectralPressureSolver(setup); -uref = device(data_test.u[ig][end]); -u₀ = device(data_test.u[ig][1]); -Δt = params_test.Δt * params_test.savefreq; -tlims = extrema(data_test.t); -nupdate = 4; -Δt /= nupdate; -state_nm, outputs = solve_unsteady(setup, u₀, tlims; Δt, pressure_solver); -m = smagorinsky_closure(setup); -closedsetup = (; setup..., closure_model = u -> m(u, T(0.1))); -state_smag, outputs = solve_unsteady(closedsetup, u₀, tlims; Δt, pressure_solver); -# closedsetup = (; setup..., closure_model = wrappedclosure(closure, θ_cnn_shared, setup)); -closedsetup = (; setup..., closure_model = wrappedclosure(closure, θ_cnn[ig-1], setup)); -state_cnn, outputs = solve_unsteady(closedsetup, u₀, tlims; Δt, pressure_solver); - -# Plot predicted spectra -fig = with_theme(; palette = (; color = ["#3366cc", "#cc0000", "#669900", "#ffcc00"])) do - (; xp, Ip) = setup.grid - D = params.D - K = size(Ip) .÷ 2 - kx = ntuple(α -> 0:K[α]-1, D) - k = fill!(similar(xp[1], length.(kx)), 0) - for α = 1:D - kα = reshape(kx[α], ntuple(Returns(1), α - 1)..., :, ntuple(Returns(1), D - α)...) - k .+= kα .^ 2 - end - k .= sqrt.(k) - k = reshape(k, :) - # Sum or average wavenumbers between k and k+1 - nk = ceil(Int, maximum(k)) - kmax = nk - 1 - # kmax = minimum(K) - 1 - kint = 1:kmax - ia = similar(xp[1], Int, 0) - ib = sortperm(k) - vals = similar(xp[1], 0) - ksort = k[ib] - jprev = 2 # Do not include constant mode - for ki = 1:kmax - j = findfirst(>(ki + 1), ksort) - isnothing(j) && (j = length(k) + 1) - ia = [ia; fill!(similar(ia, j - jprev), ki)] - # val = doaverage ? T(1) / (j - jprev) : T(1) - val = T(π) * ((ki + 1)^2 - ki^2) / (j - jprev) - # val = T(1) / (j - jprev) - vals = [vals; fill!(similar(vals, j - jprev), val)] - jprev = j - end - ib = ib[2:jprev-1] - A = sparse(ia, ib, vals, kmax, length(k)) - # Build inertial slope above energy - # krange = [cbrt(T(kmax)), T(kmax)] - krange = [T(kmax)^(T(2) / 3), T(kmax)] - # slope, slopelabel = D == 2 ? (-T(3), L"k^{-3}") : (-T(5 / 3), L"k^{-5/3}") - slope, slopelabel = D == 2 ? (-T(3), "|k|⁻³") : (-T(5 / 3), "|k|⁻⁵³") - slopeconst = T(0) - # Nice ticks - logmax = round(Int, log2(kmax + 1)) - xticks = T(2) .^ (0:logmax) - # Make plot - fig = Figure(; size = (500, 400)) - ax = Axis( - fig[1, 1]; - xticks, - xlabel = "|k|", - ylabel = "e(|k|)", - # title = "Kinetic energy (n = $(params.nles[ig])) at time t = $(round(data_test.t[end]; digits = 1))", - title = "Kinetic energy (n = $(params.nles[ig]))", - xscale = log10, - yscale = log10, - limits = (extrema(kint)..., T(1e-8), T(1)), - ) - for (u, label) in ( - # (uref, "Reference"), - (state_nm.u, "No closure"), - (state_smag.u, "Smagorinsky"), - (state_cnn.u, "CNN (specialized)"), - (uref, "Reference"), - ) - ke = IncompressibleNavierStokes.kinetic_energy(u, setup) - e = ke[Ip] - e = fft(e)[ntuple(α -> kx[α] .+ 1, D)...] - e = abs.(e) ./ size(e, 1) - e = A * reshape(e, :) - ehat = max.(e, eps(T)) # Avoid log(0) - slopeconst = max(slopeconst, maximum(ehat ./ kint .^ slope)) - lines!(ax, kint, Array(ehat); label) - end - inertia = 2 .* slopeconst .* krange .^ slope - lines!(ax, krange, inertia; linestyle = :dash, label = slopelabel) - axislegend(ax; position = :lb) - autolimits!(ax) - fig -end - save("predicted_spectra.pdf", fig) -# Plot spectrum errors -fig = with_theme(; palette = (; color = ["#3366cc", "#cc0000", "#669900", "#ffcc00"])) do - (; xp, Ip) = setup.grid - D = params.D - K = size(Ip) .÷ 2 - kx = ntuple(α -> 0:K[α]-1, D) - k = fill!(similar(xp[1], length.(kx)), 0) - for α = 1:D - kα = reshape(kx[α], ntuple(Returns(1), α - 1)..., :, ntuple(Returns(1), D - α)...) - k .+= kα .^ 2 - end - k .= sqrt.(k) - k = reshape(k, :) - # Sum or average wavenumbers between k and k+1 - nk = ceil(Int, maximum(k)) - # kmax = nk - 1 - kmax = minimum(K) - 1 - kint = 1:kmax - ia = similar(xp[1], Int, 0) - ib = sortperm(k) - vals = similar(xp[1], 0) - ksort = k[ib] - jprev = 2 # Do not include constant mode - for ki = 1:kmax - j = findfirst(>(ki + 1), ksort) - isnothing(j) && (j = length(k) + 1) - ia = [ia; fill!(similar(ia, j - jprev), ki)] - # val = doaverage ? T(1) / (j - jprev) : T(1) - val = T(π) * ((ki + 1)^2 - ki^2) / (j - jprev) - vals = [vals; fill!(similar(vals, j - jprev), val)] - jprev = j - end - ib = ib[2:jprev-1] - A = sparse(ia, ib, vals, kmax, length(k)) - # Build inertial slope above energy - # krange = [cbrt(T(kmax)), T(kmax)] - krange = [T(kmax)^(T(2) / 3), T(kmax)] - # slope, slopelabel = D == 2 ? (-T(3), L"k^{-3}") : (-T(5 / 3), L"k^{-5/3}") - slope, slopelabel = D == 2 ? (-T(3), "|k|⁻³") : (-T(5 / 3), "|k|⁻⁵³") - slopeconst = T(0) - # Nice ticks - logmax = round(Int, log2(kmax + 1)) - xticks = T(2) .^ (0:logmax) - # Make plot - fig = Figure(; size = (500, 400)) - ax = Axis( - fig[1, 1]; - xticks, - xlabel = "|k|", - ylabel = "e(|k|)", - # title = "Kinetic energy (n = $(params.nles[ig])) at time t = $(round(data_test.t[end]; digits = 1))", - title = "Relative energy error (n = $(params.nles[ig]))", - xscale = log10, - yscale = log10, - limits = (extrema(kint)..., T(1e-8), T(1)), - ) - ke = IncompressibleNavierStokes.kinetic_energy(uref, setup) - e = ke[Ip] - e = fft(e)[ntuple(α -> kx[α] .+ 1, D)...] - e = abs.(e) ./ size(e, 1) - e = A * reshape(e, :) - eref = max.(e, eps(T)) # Avoid log(0) - for (u, label) in ( - (state_nm.u, "No closure"), - (state_smag.u, "Smagorinsky"), - (state_cnn.u, "CNN (specialized)"), - ) - ke = IncompressibleNavierStokes.kinetic_energy(u, setup) - e = ke[Ip] - e = fft(e)[ntuple(α -> kx[α] .+ 1, D)...] - e = abs.(e) ./ size(e, 1) - e = A * reshape(e, :) - ehat = max.(e, eps(T)) # Avoid log(0) - ee = @. abs(ehat - eref) / abs(eref) - lines!(ax, kint, Array(ee); label) - end - axislegend(ax; position = :lt) - autolimits!(ax) - fig -end - save("spectrum_error.pdf", fig) diff --git a/src/closures/training.jl b/src/closures/training.jl index 6c97f6bf2..464fc68ed 100644 --- a/src/closures/training.jl +++ b/src/closures/training.jl @@ -92,10 +92,10 @@ function create_loss_post(; psolver, closure, nupdate = 1, - unproject_closure = false, + projectorder = :last, ) closure_model = wrappedclosure(closure, setup) - setup = (; setup..., closure_model, unproject_closure) + setup = (; setup..., closure_model, projectorder) (; dimension, Iu) = setup.grid D = dimension() function loss_post(data, θ) @@ -125,29 +125,32 @@ function create_relerr_post(; setup, method = RK44(; T = eltype(setup.grid.x[1])), psolver, - closure, + closure_model, nupdate = 1, - unproject_closure = false, + projectorder = :last, ) - closure_model = wrappedclosure(closure, setup) - setup = (; setup..., closure_model, unproject_closure) + setup = (; setup..., closure_model, projectorder) (; dimension, Iu) = setup.grid D = dimension() + (; u, t) = data + v = copy.(u[1]) + cache = ode_method_cache(method, setup, v) function relerr_post(θ) - T = eltype(θ) - (; u, t) = data - v = u[1] + T = eltype(u[1][1]) + copyto!.(v, u[1]) stepper = create_stepper(method; setup, psolver, u = v, t = t[1]) - e = zero(eltype(v[1])) + e = zero(T) for it = 2:length(t) Δt = (t[it] - t[it-1]) / nupdate for isub = 1:nupdate - stepper = timestep(method, stepper, Δt; θ) + stepper = timestep!(method, stepper, Δt; θ, cache) end a, b = T(0), T(0) for α = 1:D - a += sum(abs2, (stepper.u[α]-u[it][α])[Iu[α]]) - b += sum(abs2, u[it][α][Iu[α]]) + # a += sum(abs2, (stepper.u[α]-u[it][α])[Iu[α]]) + # b += sum(abs2, u[it][α][Iu[α]]) + a += sum(abs2, view(stepper.u[α]-u[it][α], Iu[α])) + b += sum(abs2, view(u[it][α], Iu[α])) end e += sqrt(a) / sqrt(b) end diff --git a/src/operators.jl b/src/operators.jl index 27137fe5b..6fd4048cf 100644 --- a/src/operators.jl +++ b/src/operators.jl @@ -466,8 +466,8 @@ ChainRulesCore.rrule(::typeof(bodyforce), u, t, setup) = Right hand side of momentum equations, excluding pressure gradient. Put the result in ``F``. """ -function momentum!(F, u, t, setup; θ = nothing) - (; grid, closure_model, unproject_closure) = setup +function momentum!(F, u, t, setup) + (; grid, closure_model) = setup (; dimension) = grid D = dimension() for α = 1:D @@ -477,13 +477,6 @@ function momentum!(F, u, t, setup; θ = nothing) # convection!(F, u, setup) convectiondiffusion!(F, u, setup) bodyforce!(F, u, t, setup) - if !isnothing(closure_model) && !unproject_closure - m = closure_model - mu = m(u, θ) - for α = 1:D - F[α] .+= mu[α] - end - end F end @@ -501,8 +494,8 @@ end Right hand side of momentum equations, excluding pressure gradient. """ # momentum(u, t, setup) = momentum!(zero.(u), u, t, setup) -function momentum(u, t, setup; θ = nothing) - (; grid, closure_model, unproject_closure) = setup +function momentum(u, t, setup) + (; grid, closure_model) = setup (; dimension) = grid D = dimension() d = diffusion(u, setup) @@ -513,11 +506,6 @@ function momentum(u, t, setup; θ = nothing) # end F = @. d + c + f # F = tupleadd(d, c, f) - if !isnothing(closure_model) && !unproject_closure - m = closure_model - F = F .+ m(u, θ) - # F = tupleadd(F, m) - end F end diff --git a/src/processors/real_time_plot.jl b/src/processors/real_time_plot.jl index d42638be6..0db108d21 100644 --- a/src/processors/real_time_plot.jl +++ b/src/processors/real_time_plot.jl @@ -307,9 +307,13 @@ Create energy history plot. """ function energy_history_plot(state; setup) @assert state isa Observable "Energy history requires observable state." + (; Ω, Ip) = setup.grid + e = zero(state[].u[1]) _points = Point2f[] points = lift(state) do (; u, t) - E = total_kinetic_energy(u, setup) + kinetic_energy!(e, u, setup) + e .*= Ω + E = sum(e[Ip]) push!(_points, Point2f(t, E)) end fig = lines(points; axis = (; xlabel = "t", ylabel = "Kinetic energy")) diff --git a/src/setup.jl b/src/setup.jl index fc3dd3938..6a28f2197 100644 --- a/src/setup.jl +++ b/src/setup.jl @@ -21,7 +21,7 @@ function Setup( bodyforce = nothing, issteadybodyforce = true, closure_model = nothing, - unproject_closure = false, + projectorder = :last, ArrayType = Array, workgroupsize = 64, ) @@ -33,7 +33,7 @@ function Setup( bodyforce, issteadybodyforce = false, closure_model, - unproject_closure, + projectorder, ArrayType, T = eltype(x[1]), workgroupsize, diff --git a/src/time_steppers/step_explicit_runge_kutta.jl b/src/time_steppers/step_explicit_runge_kutta.jl index b8df934f9..21f5f5ba4 100644 --- a/src/time_steppers/step_explicit_runge_kutta.jl +++ b/src/time_steppers/step_explicit_runge_kutta.jl @@ -3,13 +3,14 @@ create_stepper(::ExplicitRungeKuttaMethod; setup, psolver, u, t, n = 0) = function timestep!(method::ExplicitRungeKuttaMethod, stepper, Δt; θ = nothing, cache) (; setup, psolver, u, t, n) = stepper - (; grid, closure_model, unproject_closure) = setup + (; grid, closure_model, projectorder) = setup (; dimension, Iu) = grid (; A, b, c) = method (; u₀, ku, div, p) = cache D = dimension() nstage = length(b) m = closure_model + projectorder ∈ (:first, :second, :last) || error("Unknown projectorder: $projectorder") # Update current solution t₀ = t @@ -18,16 +19,21 @@ function timestep!(method::ExplicitRungeKuttaMethod, stepper, Δt; θ = nothing, for i = 1:nstage # Compute force at current stage i apply_bc_u!(u, t, setup) - momentum!(ku[i], u, t, setup; θ) + momentum!(ku[i], u, t, setup) - if unproject_closure - # Project F first, add closure second + # Project F first + if projectorder == :first + apply_bc_u!(ku[i], t, setup; dudt = true) + project!(ku[i], setup; psolver, div, p) + end + + # Add closure term + isnothing(m) || map((k, m) -> k .+= m, ku[i], m(u, θ)) + + # Project F second + if projectorder == :second apply_bc_u!(ku[i], t, setup; dudt = true) project!(ku[i], setup; psolver, div, p) - mu = m(u, θ) - for α = 1:D - ku[i][α] .+= mu[α] - end end # Intermediate time step @@ -42,10 +48,10 @@ function timestep!(method::ExplicitRungeKuttaMethod, stepper, Δt; θ = nothing, end end + # Project stage u directly # Make velocity divergence free at time t - apply_bc_u!(u, t, setup) - if !unproject_closure - # Project stage u directly, closure is already included in momentum + if projectorder == :last + apply_bc_u!(u, t, setup) project!(u, setup; psolver, div, p) end end @@ -60,12 +66,13 @@ end function timestep(method::ExplicitRungeKuttaMethod, stepper, Δt; θ = nothing) (; setup, psolver, u, t, n) = stepper - (; grid, closure_model, unproject_closure) = setup + (; grid, closure_model, projectorder) = setup (; dimension) = grid (; A, b, c) = method D = dimension() nstage = length(b) m = closure_model + projectorder ∈ (:first, :second, :last) || error("Unknown projectorder: $projectorder") # Update current solution (does not depend on previous step size) t₀ = t @@ -75,13 +82,21 @@ function timestep(method::ExplicitRungeKuttaMethod, stepper, Δt; θ = nothing) for i = 1:nstage # Compute force at current stage i u = apply_bc_u(u, t, setup) - F = momentum(u, t, setup; θ) + F = momentum(u, t, setup) - if unproject_closure - # Project F first, add closure second + # Project F first + if projectorder == :first + F = apply_bc_u(F, t, setup; dudt = true) + F = project(F, setup; psolver) + end + + # Add closure term + isnothing(m) || (F = F .+ m(u, θ)) + + # Project F second + if projectorder == :second F = apply_bc_u(F, t, setup; dudt = true) F = project(F, setup; psolver) - F = F .+ m(u, θ) end # Store right-hand side of stage i @@ -97,10 +112,10 @@ function timestep(method::ExplicitRungeKuttaMethod, stepper, Δt; θ = nothing) # u = tupleadd(u, @.(Δt * A[i, j] * ku[j])) end + # Project stage u directly # Make velocity divergence free at time t - u = apply_bc_u(u, t, setup) - if !unproject_closure - # Project stage u directly, closure is already included in momentum + if projectorder == :last + u = apply_bc_u(u, t, setup) u = project(u, setup; psolver) end end From 1750abf87b38774e8a5d3d48f420c044e7a008c2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Tue, 30 Jan 2024 14:05:09 +0100 Subject: [PATCH 243/379] fix: Possible bug in IC --- src/create_initial_conditions.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/create_initial_conditions.jl b/src/create_initial_conditions.jl index 5c9e4b524..e0446c8de 100644 --- a/src/create_initial_conditions.jl +++ b/src/create_initial_conditions.jl @@ -210,6 +210,6 @@ function random_field( # Make velocity field divergence free on staggered grid # (it is already diergence free on the "spectral grid") apply_bc_u!(u, t, setup) - project(u, setup; psolver) + u = project(u, setup; psolver) apply_bc_u!(u, t, setup) end From 248dce6f12e5364148b81026922d7555ef96d31e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Thu, 8 Feb 2024 01:43:01 +0100 Subject: [PATCH 244/379] Update files --- scratch/divergence.jl | 8 +- scratch/divfree.jl | 543 ++++++++++++++++++++++++++----- src/closures/closure.jl | 4 +- src/closures/create_les_data.jl | 27 +- src/closures/training.jl | 18 +- src/processors/real_time_plot.jl | 3 +- src/utils/spectral_stuff.jl | 4 +- 7 files changed, 503 insertions(+), 104 deletions(-) diff --git a/scratch/divergence.jl b/scratch/divergence.jl index 3a45694d5..fd2f4370f 100644 --- a/scratch/divergence.jl +++ b/scratch/divergence.jl @@ -175,7 +175,7 @@ fieldplot( @time state, outputs = solve_unsteady( dns, u₀, - (T(0), T(1e-1)); + (T(0), T(5e-1)); Δt, # Δt = T(1e-4), # Δt = T(1e-5), @@ -191,7 +191,7 @@ fieldplot( # # levels = 5, # docolorbar = false, # ), - # obs = observe_u(dns, psolver_dns, filters; nupdate = 20), + obs = observe_u(dns, psolver_dns, filters; nupdate = 20), # ehist = realtimeplotter(; # setup, # plot = energy_history_plot, @@ -247,8 +247,8 @@ fieldplot( save("$output/vorticity_end_$(filters[i].compression).png", current_figure()) fieldplot( - # (; u = u₀, t = T(0)); - state; + (; u = u₀, t = T(0)); + # state; setup = dns, fieldname = :eig2field, levels = LinRange(T(2), T(10), 10), diff --git a/scratch/divfree.jl b/scratch/divfree.jl index b69d7d9f7..396316f4e 100644 --- a/scratch/divfree.jl +++ b/scratch/divfree.jl @@ -30,8 +30,6 @@ getorder(i) = if i == 1 :first elseif i == 2 - :second - elseif i == 3 :last else error("Unknown order: $i") @@ -93,22 +91,25 @@ get_params(nlesscalar) = (; params_train = (; get_params([64, 128, 256])..., savefreq = 10); params_valid = (; get_params([64, 128, 256])..., tsim = T(0.1), savefreq = 40); -params_test = (; get_params([64, 128, 256, 512, 1024])..., tsim = T(0.2), savefreq = 20); +params_test = (; get_params([64, 128, 256, 512, 1024])..., tsim = T(0.1), savefreq = 10); # Create LES data from DNS data_train = [create_les_data(; params_train...) for _ = 1:5]; data_valid = [create_les_data(; params_valid...) for _ = 1:1]; data_test = create_les_data(; params_test...); +data_train[1].data[1].u[1][1] +data_test.data[6].u[1][1] + # Save filtered DNS data jldsave("output/divfree/data_train.jld2"; data_train) jldsave("output/divfree/data_valid.jld2"; data_valid) jldsave("output/divfree/data_test.jld2"; data_test) # Load filtered DNS data -data_train = load("output/divfree/data.jld2", "data_train"); -data_valid = load("output/divfree/data.jld2", "data_valid"); -data_test = load("output/divfree/data.jld2", "data_test"); +data_train = load("output/divfree/data_train.jld2", "data_train"); +data_valid = load("output/divfree/data_valid.jld2", "data_valid"); +data_test = load("output/divfree/data_test.jld2", "data_test"); # Build LES setup and assemble operators getsetups(params) = [ @@ -144,7 +145,7 @@ let ifil = 1 field, setup = data_train[1].data[ig, ifil].u, setups_train[ig] # field, setup = data_valid[1].data[ig, ifil].u, setups_valid[ig]; - # field, setup = data_test.data[ig, ifil], setups_test[ig]; + # field, setup = data_test.data[ig, ifil].u, setups_test[ig]; u = device.(field[1]) o = Observable((; u, t = nothing)) # energy_spectrum_plot(o; setup) |> display @@ -238,42 +239,52 @@ closure, θ₀ = cnn(; ); closure.chain +closure(device(io_train[1, 1].u[:, :, :, 1:50]), device(θ₀)); + # Train grid-specialized closure models -θ_cnn = map(CartesianIndices(size(io_train))) do I +prior = map(CartesianIndices(size(io_train))) do I # Prepare training + starttime = time() ig, ifil = I.I - @show ig ifil + println("ig = $ig, ifil = $ifil") d = create_dataloader_prior(io_train[ig, ifil]; batchsize = 50, device) θ = T(1.0e0) * device(θ₀) - loss = createloss(mean_squared_error, closure) + loss = create_loss_prior(mean_squared_error, closure) opt = Optimisers.setup(Adam(T(1.0e-3)), θ) callbackstate = Point2f[] it = rand(1:size(io_valid[ig, ifil].u, 4), 50) - validset = map(v -> v[:, :, :, it], io_valid[ig, ifil]) - (; opt, θ, callbackstate) = train( - [d], - loss, - opt, - θ; - niter = 10000, - ncallback = 20, - callbackstate, - callback = create_relerr_prior(closure, validset), + validset = device(map(v -> v[:, :, :, it], io_valid[ig, ifil])) + (; callbackstate, callback) = create_callback( + create_relerr_prior(closure, validset...); + θ, + displayref = true, + display_each_iteration = false, ) - θ + (; opt, θ, callbackstate) = + train([d], loss, opt, θ; niter = 10000, ncallback = 20, callbackstate, callback) + θ = callbackstate.θmin # Use best θ instead of last θ + comptime = time() - starttime + (; Array(θ), comptime) end clean() +prior = [(; θ = Array(θ), comptime = 0.0) for θ ∈ θ_cnn] + # Save trained parameters -jldsave("output/divfree/theta_prior.jld2"; theta = Array.(θ_cnn)) +jldsave("output/divfree/prior.jld2"; prior) # Load trained parameters -θ_cnn = [device(θ₀) for _ in CartesianIndices(size(data_train[1].data))]; -θθ = load("output/divfree/theta_prior.jld2"); -copyto!.(θ_cnn, θθ["theta"]); +prior = load("output/divfree/prior.jld2")["prior"]; -# θ_cnn_post = let ig = 3, ifil = 2, iorder = 2 -θ_cnn_post = map(CartesianIndices((size(io_train)..., 2))) do I +# Extract component array +θ_cnn = [copyto!(device(θ₀), p.θ) for p in prior]; + +ngrid, nfilter = size(io_train) +# post = map(CartesianIndices((ngrid, nfilter, 2))) do I + +let I = CartesianIndex((3, 2, 1)) + clean() + starttime = time() ig, ifil, iorder = I.I println("iorder = $iorder, ifil = $ifil, ig = $ig") setup = setups_train[ig] @@ -286,52 +297,51 @@ copyto!.(θ_cnn, θθ["theta"]); projectorder = getorder(iorder), ) data = [(; u = d.data[ig, ifil].u, d.t) for d in data_train] - d = create_dataloader_post(data; device, nunroll = 20) + d = create_dataloader_post(data; device, nunroll = 10) # θ = T(1e-1) * copy(θ_cnn[ig, ifil]) - θ = copy(θ_cnn[ig, ifil]) + # θ = copy(θ_cnn[ig, ifil]) # θ = copy(θ_cnn_post) + θ = copy(θ_cnn_post[ig, ifil, iorder]) # θ = device(θ₀) opt = Optimisers.setup(Adam(T(1.0e-3)), θ) - callbackstate = Point2f[] it = 1:20 data = data_valid[1] data = (; u = device.(data.data[ig, ifil].u[it]), t = data.t[it]) - (; opt, θ, callbackstate) = train( - [d], - loss, - opt, - θ; - niter = 200, - ncallback = 10, - callbackstate, - callback = create_callback( - create_relerr_post(; - data, - setup, - psolver, - closure_model = wrappedclosure(closure, setup), - projectorder = getorder(iorder), - nupdate = 8, - ); - state = callbackstate, - displayref = false, - ), - ) - jldsave( - "output/divfree/theta_post_iorder$(iorder)_ifil$(ifil)_ig$(ig).jld2"; - theta = Array(θ), + (; callbackstate, callback) = create_callback( + create_relerr_post(; + data, + setup, + psolver, + closure_model = wrappedclosure(closure, setup), + projectorder = getorder(iorder), + nupdate = 8, + ); + θ, + displayref = false, ) - clean() - θ + (; opt, θ, callbackstate) = + train([d], loss, opt, θ; niter = 1000, ncallback = 10, callbackstate, callback) + θ = callbackstate.θmin # Use best θ instead of last θ + post = (; θ = Array(θ), comptime = time() - starttime) + jldsave("output/divfree/post_iorder$(iorder)_ifil$(ifil)_ig$(ig).jld2"; post) + post end; clean() -θ_cnn[2, 2] |> extrema +post = map(CartesianIndices((size(io_train)..., 2))) do I + ig, ifil, iorder = I.I + name = "output/divfree/post_iorder$(iorder)_ifil$(ifil)_ig$(ig).jld2" + load(name)["post"] +end +θ_cnn_post = [copyto!(device(θ₀), p.θ) for p in post]; + +θ_cnn_post .|> extrema -Array(θ_post) +map(p -> p.comptime, post) # Train Smagorinsky model with Lpost (grid search) -θ_smag = map(CartesianIndices((size(io_train, 2), 2))) do I +smag = map(CartesianIndices((size(io_train, 2), 2))) do I + starttime = time() ifil, iorder = I.I ngrid = size(io_train, 1) θmin = T(0) @@ -364,36 +374,40 @@ Array(θ_post) θmin = θ end end - θmin + (; θ = θmin, comptime = time() - starttime) end clean() -θ_smag() + +smag # Save trained parameters -jldsave("output/divfree/theta_smag.jld2"; theta = θ_smag); +jldsave("output/divfree/smag.jld2"; smag); # Load trained parameters -θ_smag = load("output/divfree/theta_smag.jld2")["theta"]; -θ_smag +smag = load("output/divfree/smag.jld2")["smag"]; + +# Extract coefficients +θ_smag = map(s -> s.θ, smag) # lines(LinRange(T(0), T(1), 100), e_smag) -# Errors for grid-specialized closure models +# Compute posterior errors e_nm, e_smag, e_cnn, e_cnn_post = let e_nm = zeros(T, size(data_test.data)...) e_smag = zeros(T, size(data_test.data)..., 2) e_cnn = zeros(T, size(data_test.data)..., 2) e_cnn_post = zeros(T, size(data_test.data)..., 2) + # for iorder = 1:1, ifil = 1:1, ig in 3 for iorder = 1:2, ifil = 1:2, ig = 1:size(data_test.data, 1) println("iorder = $iorder, ifil = $ifil, ig = $ig") projectorder = getorder(iorder) setup = setups_test[ig] psolver = SpectralPressureSolver(setup) data = (; u = device.(data_test.data[ig, ifil].u), t = data_test.t) - nupdate = 50 + nupdate = 20 # No model # Only for closurefirst, since projectfirst is the same - if iorder == 1 + if iorder == 2 err = create_relerr_post(; data, setup, psolver, closure_model = nothing, nupdate) e_nm[ig, ifil] = err(nothing) end @@ -407,8 +421,8 @@ e_nm, e_smag, e_cnn, e_cnn_post = let nupdate, ) e_smag[ig, ifil, iorder] = err(θ_smag[ifil, iorder]) - # CNN - # Only the first grids are trained for + CNN + Only the first grids are trained for if ig ≤ size(data_train[1].data, 1) err = create_relerr_post(; data, @@ -417,6 +431,7 @@ e_nm, e_smag, e_cnn, e_cnn_post = let closure_model = wrappedclosure(closure, setup), projectorder, nupdate, + # nupdate = 50, ) e_cnn[ig, ifil, iorder] = err(θ_cnn[ig, ifil]) e_cnn_post[ig, ifil, iorder] = err(θ_cnn_post[ig, ifil, iorder]) @@ -442,7 +457,8 @@ with_theme(; palette = (; color = ["#3366cc", "#cc0000", "#669900", "#ffcc00"]), ) do iorder = 1 - lesmodel = iorder == 1 ? "closure-then-project" : "project-then-closure" + # lesmodel = iorder == 1 ? "project-then-closure" : "closure-then-project" + lesmodel = iorder == 1 ? "project-then-closure" : "closure-then-project" nles = [n[1] for n in params_test.nles] fig = Figure(; size = (500, 400)) ax = Axis( @@ -459,7 +475,8 @@ with_theme(; for ifil = 1:2 linestyle = ifil == 1 ? :solid : :dash label = "No closure" - label = label * (ifil == 1 ? " (FA)" : " (VA)") + # label = label * (ifil == 1 ? " (FA)" : " (VA)") + ifil == 2 && (label = nothing) scatterlines!( nles, e_nm[:, ifil]; @@ -472,7 +489,8 @@ with_theme(; for ifil = 1:2 linestyle = ifil == 1 ? :solid : :dash label = "Smagorinsky" - label = label * (ifil == 1 ? " (FA)" : " (VA)") + # label = label * (ifil == 1 ? " (FA)" : " (VA)") + ifil == 2 && (label = nothing) scatterlines!( nles, e_smag[:, ifil, iorder]; @@ -485,9 +503,9 @@ with_theme(; for ifil = 1:2 ntrain = size(data_train[1].data, 1) linestyle = ifil == 1 ? :solid : :dash - label = "CNN" - label = label * (ifil == 1 ? " (FA)" : " (VA)") - # ifil == 2 && (label = nothing) + label = "CNN (prior)" + # label = label * (ifil == 1 ? " (FA)" : " (VA)") + ifil == 2 && (label = nothing) scatterlines!( nles[1:ntrain], e_cnn[1:ntrain, ifil, iorder]; @@ -501,8 +519,8 @@ with_theme(; ntrain = size(data_train[1].data, 1) linestyle = ifil == 1 ? :solid : :dash label = "CNN (post)" - label = label * (ifil == 1 ? " (FA)" : " (VA)") - # ifil == 2 && (label = nothing) + # label = label * (ifil == 1 ? " (FA)" : " (VA)") + ifil == 2 && (label = nothing) scatterlines!( nles[1:ntrain], e_cnn_post[1:ntrain, ifil, iorder]; @@ -527,6 +545,258 @@ end save("$output/convergence_Lprior_prosecond.pdf", current_figure()) save("$output/convergence_Lprior_profirst.pdf", current_figure()) +# Energy evolution ########################################################### + +kineticenergy = let + clean() + ngrid, nfilter = size(io_train) + ke_ref = fill(zeros(T, 0), ngrid, nfilter) + ke_nomodel = fill(zeros(T, 0), ngrid, nfilter) + ke_smag = fill(zeros(T, 0), ngrid, nfilter, 2) + ke_cnn_prior = fill(zeros(T, 0), ngrid, nfilter, 2) + ke_cnn_post = fill(zeros(T, 0), ngrid, nfilter, 2) + for iorder = 1:2, ifil = 1:nfilter, ig = 1:ngrid + # for iorder = 1:1, ifil = 1:1, ig = 1:1 + println("iorder = $iorder, ifil = $ifil, ig = $ig") + setup = setups_test[ig] + psolver = SpectralPressureSolver(setup) + t = data_test.t + u₀ = data_test.data[ig, ifil].u[1] |> device + tlims = (t[1], t[end]) + nupdate = 50 + Δt = (t[2] - t[1]) / nupdate + T = eltype(u₀[1]) + ewriter = processor() do state + ehist = zeros(T, 0) + on(state) do (; u, n) + if n % nupdate == 0 + e = IncompressibleNavierStokes.total_kinetic_energy(u, setup) + push!(ehist, e) + end + end + state[] = state[] # Compute initial energy + ehist + end + processors = (; ewriter) + if iorder == 1 + # Does not depend on projection order + ke_ref[ig, ifil] = map( + u -> IncompressibleNavierStokes.total_kinetic_energy(device(u), setup), + data_test.data[ig, ifil].u, + ) + ke_nomodel[ig, ifil] = + solve_unsteady( + setup, + u₀, + tlims; + Δt, + processors, + psolver, + )[2].ewriter + end + ke_smag[ig, ifil, iorder] = + solve_unsteady( + (; + setup..., + projectorder = getorder(iorder), + closure_model = smagorinsky_closure(setup), + ), + u₀, + tlims; + Δt, + processors, + psolver, + θ = θ_smag[ifil, iorder], + )[2].ewriter + ke_cnn_prior[ig, ifil, iorder] = + solve_unsteady( + (; + setup..., + projectorder = getorder(iorder), + closure_model = wrappedclosure(closure, setup), + ), + u₀, + tlims; + Δt, + processors, + psolver, + θ = θ_cnn[ig, ifil], + )[2].ewriter + ke_cnn_post[ig, ifil, iorder] = + solve_unsteady( + (; + setup..., + projectorder = getorder(iorder), + closure_model = wrappedclosure(closure, setup), + ), + u₀, + tlims; + Δt, + processors, + psolver, + θ = θ_cnn_post[ig, ifil, iorder], + )[2].ewriter + end + (; ke_ref, ke_nomodel, ke_smag, ke_cnn_prior, ke_cnn_post) +end; +clean(); + +kineticenergy.ke_ref[1] +kineticenergy.ke_nomodel[1] +kineticenergy.ke_smag[1] +kineticenergy.ke_cnn_prior[1] +kineticenergy.ke_cnn_post[1] + +CairoMakie.activate!() + +with_theme(; palette = (; color = ["#3366cc", "#cc0000", "#669900", "#ffcc00"])) do + t = data_test.t[2:end] + for iorder = 1:2, ifil = 1:2, igrid = 1:3 + println("iorder = $iorder, ifil = $ifil, igrid = $igrid") + reflevel = kineticenergy.ke_ref[igrid, ifil][2:end] + reflevel = fill!(reflevel, 1) + lesmodel = iorder == 1 ? "P-first" : "P-last" + fil = ifil == 1 ? "FA" : "VA" + nles = params_test.nles[igrid] + fig = Figure(; size = (500, 400)) + ax = Axis( + fig[1, 1]; + xlabel = "t", + ylabel = "E(t)", + # title = "Kinetic energy: $lesmodel, $fil, $nles", + title = "Kinetic energy: $lesmodel, $fil", + ) + lines!( + ax, + t, + kineticenergy.ke_ref[igrid, ifil][2:end] ./ reflevel; + color = Cycled(1), + linestyle = :dash, + label = "Reference", + ) + lines!( + ax, + t, + kineticenergy.ke_nomodel[igrid, ifil] ./ reflevel; + color = Cycled(1), + label = "No closure", + ) + lines!( + ax, + t, + kineticenergy.ke_smag[igrid, ifil, iorder] ./ reflevel; + color = Cycled(2), + label = "Smagorinsky", + ) + lines!( + ax, + t, + kineticenergy.ke_cnn_prior[igrid, ifil, iorder] ./ reflevel; + color = Cycled(3), + label = "CNN (prior)", + ) + lines!( + ax, + t, + kineticenergy.ke_cnn_post[igrid, ifil, iorder] ./ reflevel; + color = Cycled(4), + label = "CNN (post)", + ) + iorder == 1 && axislegend(; position = :lt) + iorder == 2 && axislegend(; position = :lb) + # axislegend(; position = :lb) + # axislegend() + name = "$output/energy_evolution" + save("$(name)_iorder$(iorder)_ifilter$(ifil)_igrid$(igrid).pdf", fig) + end +end + +# Solutions at final time #################################################### + +ufinal = let + ngrid, nfilter = size(io_train) + temp = ntuple(α -> zeros(T, 0, 0), 2) + u_ref = fill(temp, ngrid, nfilter) + u_nomodel = fill(temp, ngrid, nfilter) + u_smag = fill(temp, ngrid, nfilter, 2) + u_cnn_prior = fill(temp, ngrid, nfilter, 2) + u_cnn_post = fill(temp, ngrid, nfilter, 2) + # for iorder = 1:1, ifil = 2:2, igrid = 3:3 + for iorder = 1:2, ifil = 1:nfilter, igrid = 1:ngrid + clean() + println("iorder = $iorder, ifil = $ifil, igrid = $igrid") + t = data_test.t + setup = setups_test[igrid] + psolver = SpectralPressureSolver(setup) + u₀ = data_test.data[igrid, ifil].u[1] |> device + tlims = (t[1], t[end]) + nupdate = 50 + Δt = (t[2] - t[1]) / nupdate + T = eltype(u₀[1]) + if iorder == 1 + # Does not depend on projection order + u_ref[igrid, ifil] = data_test.data[igrid, ifil].u[end] + u_nomodel[igrid, ifil] = + solve_unsteady( + setup, + u₀, + tlims; + Δt, + psolver, + )[1].u .|> Array + end + u_smag[igrid, ifil, iorder] = + solve_unsteady( + (; + setup..., + projectorder = getorder(iorder), + closure_model = smagorinsky_closure(setup), + ), + u₀, + tlims; + Δt, + psolver, + θ = θ_smag[ifil, iorder], + )[1].u .|> Array + u_cnn_prior[igrid, ifil, iorder] = + solve_unsteady( + (; + setup..., + projectorder = getorder(iorder), + closure_model = wrappedclosure(closure, setup), + ), + u₀, + tlims; + Δt, + psolver, + θ = θ_cnn[igrid, ifil], + )[1].u .|> Array + u_cnn_post[igrid, ifil, iorder] = + solve_unsteady( + (; + setup..., + projectorder = getorder(iorder), + closure_model = wrappedclosure(closure, setup), + ), + u₀, + tlims; + Δt, + psolver, + θ = θ_cnn_post[igrid, ifil, iorder], + )[1].u .|> Array + end + (; u_ref, u_nomodel, u_smag, u_cnn_prior, u_cnn_post) +end; +clean(); + +ufinal.u_ref[1][2] +ufinal.u_cnn_prior[3, 1, 1][1] +ufinal.u_cnn_post[3, 1, 1][1] + +jldsave("output/divfree/ufinal.jld2"; ufinal) + +ufinal = load("output/divfree/ufinal.jld2")["ufinal"]; + markers_labels = [ (:circle, ":circle"), (:rect, ":rect"), @@ -552,6 +822,121 @@ markers_labels = [ ('✈', "'\\:airplane:'"), ] -save("predicted_spectra.pdf", fig) +# Plot spectra ############################################################### -save("spectrum_error.pdf", fig) +fig = with_theme(; palette = (; color = ["#3366cc", "#cc0000", "#669900", "#ffcc00"])) do + for iorder = 1:2, ifil = 1:2, igrid = 1:3 + println("iorder = $iorder, ifil = $ifil, igrid = $igrid") + lesmodel = iorder == 1 ? "P-first" : "P-last" + fil = ifil == 1 ? "FA" : "VA" + nles = params_test.nles[igrid] + setup = setups_test[igrid] + fields = [ + ufinal.u_ref[igrid, ifil], + ufinal.u_nomodel[igrid, ifil], + ufinal.u_smag[igrid, ifil, iorder], + ufinal.u_cnn_prior[igrid, ifil, iorder], + ufinal.u_cnn_post[igrid, ifil, iorder], + ] .|> device + (; Ip) = setup.grid + (; A, κ, K) = IncompressibleNavierStokes.spectral_stuff(setup) + specs = map(fields) do u + # up = interpolate_u_p(u, setup) + up = u + e = sum(up) do u + u = u[Ip] + uhat = fft(u)[ntuple(α -> 1:K[α], 2)...] + # abs2.(uhat) + abs2.(uhat) ./ (2 * prod(size(u))^2) + # abs2.(uhat) ./ size(u, 1) + end + e = A * reshape(e, :) + # e = max.(e, eps(T)) # Avoid log(0) + ehat = Array(e) + end + kmax = maximum(κ) + # Build inertial slope above energy + krange = [T(16), T(κ[end])] + slope, slopelabel = -T(3), L"$\kappa^{-3}" + slopeconst = maximum(specs[1] ./ κ .^ slope) + offset = 3 + inertia = offset .* slopeconst .* krange .^ slope + # Nice ticks + logmax = round(Int, log2(kmax + 1)) + xticks = T(2) .^ (0:logmax) + # Make plot + fig = Figure(; size = (500, 400)) + ax = Axis( + fig[1, 1]; + xticks, + # xlabel = "k", + xlabel = "κ", + # ylabel = "e(κ)", + xscale = log10, + yscale = log10, + limits = (1, kmax, T(1e-8), T(1)), + title = "Kinetic energy: $lesmodel, $fil", + ) + lines!(ax, κ, specs[2]; color = Cycled(1), label = "No model") + lines!(ax, κ, specs[3]; color = Cycled(2), label = "Smagorinsky") + lines!(ax, κ, specs[4]; color = Cycled(3), label = "CNN (prior)") + lines!(ax, κ, specs[5]; color = Cycled(4), label = "CNN (post)") + lines!(ax, κ, specs[1]; color = Cycled(1), linestyle = :dash, label = "Reference") + lines!(ax, krange, inertia; color = Cycled(1), label = slopelabel, linestyle = :dot) + # axislegend(ax; position = :lb) + axislegend(ax; position = :cb) + autolimits!(ax) + # limits!(ax, (T(0.8), T(800)), (T(1e-10), T(1))) + name = "$output/energy_spectra" + save("$(name)_iorder$(iorder)_ifilter$(ifil)_igrid$(igrid).pdf", fig) + end +end +clean(); + +# Plot fields ################################################################ + +GLMakie.activate!() + +with_theme(; fontsize = 25, palette = (; color = ["#3366cc", "#cc0000", "#669900", "#ffcc00"])) do + x1 = 0.3 + x2 = 0.5 + y1 = 0.5 + y2 = 0.7 + box = [ + Point2f(x1, y1), + Point2f(x2, y1), + Point2f(x2, y2), + Point2f(x1, y2), + Point2f(x1, y1), + # Point2f(x2, y1), + ] + path = "$output/les_fields" + ispath(path) || mkpath(path) + for iorder = 1:2, ifil = 1:2, igrid = 1:3 + setup = setups_test[igrid] + name = "$(path)/iorder$(iorder)_ifilter$(ifil)_igrid$(igrid)" + lesmodel = iorder == 1 ? "P-first" : "P-last" + fil = ifil == 1 ? "FA" : "VA" + nles = params_test.nles[igrid] + function makeplot(u, title, suffix) + fig = fieldplot( + (; u, t = T(0)); + setup, + title, + # type = image, + # colormap = :viridis, + docolorbar = false, + size = (500, 500), + ) + lines!(box; linewidth = 5, color = Cycled(2)) + fname = "$(name)_$(suffix).png" + save(fname, fig) + # run(`convert $fname -trim $fname`) # Requires imagemagick + end + iorder == 2 && makeplot(device(ufinal.u_ref[igrid, ifil]), "Reference, $fil, $nles", "ref") + iorder == 2 && makeplot(device(ufinal.u_nomodel[igrid, ifil]), "No closure, $fil, $nles", "nomodel") + makeplot(device(ufinal.u_smag[igrid, ifil, iorder]), "Smagorinsky, $lesmodel, $fil, $nles", "smag") + makeplot(device(ufinal.u_cnn_prior[igrid, ifil, iorder]), "CNN (prior), $lesmodel, $fil, $nles", "prior") + makeplot(device(ufinal.u_cnn_post[igrid, ifil, iorder]), "CNN (post), $lesmodel, $fil, $nles", "post") + end +end diff --git a/src/closures/closure.jl b/src/closures/closure.jl index 3b4a09eea..e99b6b04a 100644 --- a/src/closures/closure.jl +++ b/src/closures/closure.jl @@ -15,7 +15,7 @@ function wrappedclosure(m, setup) # i = ntuple(Returns(:), D) # mu = ntuple(α -> mu[i..., α, 1], D) # end - function neuralclosure(u, θ) + neuralclosure(u, θ) = if D == 2 u = cat(u[1][Iu[1]], u[2][Iu[2]]; dims = 3) u = reshape(u, size(u)..., 1) # One sample @@ -29,8 +29,6 @@ function wrappedclosure(m, setup) mu = pad_circular(mu, 1) mu = (mu[:, :, :, 1, 1], mu[:, :, :, 2, 1], mu[:, :, :, 3, 1]) end - mu - end end """ diff --git a/src/closures/create_les_data.jl b/src/closures/create_les_data.jl index 9d456f936..e1c5be373 100644 --- a/src/closures/create_les_data.jl +++ b/src/closures/create_les_data.jl @@ -39,10 +39,7 @@ function lesdatagen(dnsobs, Φ, les, compression, psolver) ΦF = zero.(Φu) FΦ = zero.(Φu) c = zero.(Φu) - results = (; - u = fill(Array.(dnsobs[].u), 0), - c = fill(Array.(dnsobs[].u), 0), - ) + results = (; u = fill(Array.(dnsobs[].u), 0), c = fill(Array.(dnsobs[].u), 0)) on(dnsobs) do (; u, F, t) Φ(Φu, u, les, compression) apply_bc_u!(Φu, t, les) @@ -60,7 +57,10 @@ function lesdatagen(dnsobs, Φ, les, compression, psolver) end filtersaver(dns, les, filters, compression, psolver_dns, psolver_les; nupdate = 1) = - processor() do state + processor( + (results, state) -> (; results..., comptime = time() - results.comptime), + ) do state + comptime = time() (; x) = dns.grid T = eltype(x[1]) F = zero.(state[].u) @@ -71,10 +71,7 @@ filtersaver(dns, les, filters, compression, psolver_dns, psolver_les; nupdate = lesdatagen(dnsobs, Φ, les[i], compression[i], psolver_les[i]) for i = 1:length(les), Φ in filters ] - results = (; - data, - t = zeros(T, 0), - ) + results = (; data, t = zeros(T, 0), comptime) on(state) do (; u, t, n) n % nupdate == 0 || return momentum!(F, u, t, dns) @@ -165,6 +162,8 @@ function create_les_data(; # Initial conditions u₀ = icfunc(dns, psolver) + any(u -> any(isnan, u), u₀) && @warn "Initial conditions contain NaNs" + # Random body force # force_dns = # gaussian_force(xdns...) + @@ -227,8 +226,14 @@ function create_io_arrays(data, setups) c = zeros(T, (N .- 2)..., D, nt + 1, nsample) ifield = ntuple(Returns(:), D) for is = 1:nsample, it = 1:nt+1, α = 1:D - copyto!(view(u, ifield..., α, it, is), view(data[is].data[ig, ifil].u[it][α], Iu[α])) - copyto!(view(c, ifield..., α, it, is), view(data[is].data[ig, ifil].c[it][α], Iu[α])) + copyto!( + view(u, ifield..., α, it, is), + view(data[is].data[ig, ifil].u[it][α], Iu[α]), + ) + copyto!( + view(c, ifield..., α, it, is), + view(data[is].data[ig, ifil].c[it][α], Iu[α]), + ) end (; u = reshape(u, (N .- 2)..., D, :), c = reshape(c, (N .- 2)..., D, :)) end diff --git a/src/closures/training.jl b/src/closures/training.jl index 464fc68ed..b1b91ebe0 100644 --- a/src/closures/training.jl +++ b/src/closures/training.jl @@ -177,24 +177,32 @@ If not using interactive GLMakie window, set `display_each_iteration` to """ function create_callback( err; - state = Point2f[], + θ, + callbackstate = (; + θmin = θ, + emin = eltype(θ)(Inf), + hist = Point2f[], + ), displayref = true, display_each_iteration = false, ) - istart = isempty(state) ? 0 : Int(first(state[end])) + istart = isempty(callbackstate.hist) ? 0 : Int(callbackstate.hist[end][1]) obs = Observable([Point2f(0, 0)]) fig = lines(obs; axis = (; title = "Relative prediction error", xlabel = "Iteration")) displayref && hlines!([1.0f0]; linestyle = :dash) - obs[] = state + obs[] = callbackstate.hist display(fig) function callback(state, i, θ) e = err(θ) @info "Iteration $i \trelative error: $e" - state = push!(copy(state), Point2f(istart + i, e)) - obs[] = state + hist = push!(copy(state.hist), Point2f(istart + i, e)) + obs[] = hist # i < 30 || autolimits!(fig.axis) autolimits!(fig.axis) display_each_iteration && display(fig) + state = (; state..., hist) + e < state.emin && (state = (; state..., θmin = θ, emin = e)) state end + (; callbackstate, callback) end diff --git a/src/processors/real_time_plot.jl b/src/processors/real_time_plot.jl index 0db108d21..20380d217 100644 --- a/src/processors/real_time_plot.jl +++ b/src/processors/real_time_plot.jl @@ -87,6 +87,7 @@ function fieldplot( equal_axis = true, docolorbar = true, size = nothing, + title = nothing, kwargs..., ) (; boundary_conditions, grid) = setup @@ -169,9 +170,9 @@ function fieldplot( end axis = (; - title = titlecase(string(fieldname)), xlabel = "x", ylabel = "y", + title = isnothing(title) ? titlecase(string(fieldname)) : title, limits = (xlims[1]..., xlims[2]...), ) equal_axis && (axis = (axis..., aspect = DataAspect())) diff --git a/src/utils/spectral_stuff.jl b/src/utils/spectral_stuff.jl index 9b56bd9d6..376538f7b 100644 --- a/src/utils/spectral_stuff.jl +++ b/src/utils/spectral_stuff.jl @@ -22,7 +22,9 @@ function spectral_stuff(setup; npoint = 100, a = typeof(setup.Re)(1 + sqrt(5)) / vals = zeros(T, 0) # Output query points (evenly log-spaced, but only integer wavenumbers) - logκ = LinRange(T(0), log(T(kmax)), npoint) + logκ = LinRange(T(0), log(T(kmax) / a), npoint) + # logκ = LinRange(log(a), log(T(kmax) / a), npoint) + # logκ = LinRange(T(0), log(T(kmax)), npoint) κ = exp.(logκ) κ = sort(unique(round.(Int, κ))) npoint = length(κ) From d3c743e7418eeeef28c4992aa97b0e95e278bfcf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Thu, 22 Feb 2024 08:15:16 +0100 Subject: [PATCH 245/379] Add linear stuff --- src/utils/spectral_stuff.jl | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/utils/spectral_stuff.jl b/src/utils/spectral_stuff.jl index 376538f7b..ef4b1f81e 100644 --- a/src/utils/spectral_stuff.jl +++ b/src/utils/spectral_stuff.jl @@ -22,6 +22,7 @@ function spectral_stuff(setup; npoint = 100, a = typeof(setup.Re)(1 + sqrt(5)) / vals = zeros(T, 0) # Output query points (evenly log-spaced, but only integer wavenumbers) + # logκ = LinRange(T(0), log(T(kmax) - 1), npoint) logκ = LinRange(T(0), log(T(kmax) / a), npoint) # logκ = LinRange(log(a), log(T(kmax) / a), npoint) # logκ = LinRange(T(0), log(T(kmax)), npoint) @@ -32,6 +33,8 @@ function spectral_stuff(setup; npoint = 100, a = typeof(setup.Re)(1 + sqrt(5)) / for i = 1:npoint jstart = findfirst(≥(κ[i] / a), ksort) jstop = findfirst(≥(κ[i] * a), ksort) + # jstart = findfirst(≥(κ[i] - T(1.01)), ksort) + # jstop = findfirst(≥(κ[i] + T(1.01)), ksort) isnothing(jstop) && (jstop = length(ksort) + 1) jstop -= 1 nk = jstop - jstart + 1 From 3c6dfa8f5ab1a1c54c2802c5bcfc2833a422df58 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Thu, 22 Feb 2024 08:15:48 +0100 Subject: [PATCH 246/379] Update script --- scratch/divergence.jl | 217 +++++++++++++++++++++++++++++++----------- 1 file changed, 160 insertions(+), 57 deletions(-) diff --git a/scratch/divergence.jl b/scratch/divergence.jl index fd2f4370f..6b5c3cad3 100644 --- a/scratch/divergence.jl +++ b/scratch/divergence.jl @@ -8,7 +8,14 @@ end #src using GLMakie using IncompressibleNavierStokes using IncompressibleNavierStokes: - momentum!, divergence!, project!, apply_bc_u!, spectral_stuff, kinetic_energy, + momentum, + momentum!, + divergence!, + project, + project!, + apply_bc_u!, + spectral_stuff, + kinetic_energy, interpolate_u_p using LinearAlgebra using Printf @@ -114,7 +121,7 @@ set_theme!(; GLMakie = (; scalefactor = 1.5)) # 3D T = Float32 -Re = T(10_000) +Re = T(2_000) ndns = 512 D = 3 kp = 5 @@ -157,6 +164,7 @@ psolver_dns = SpectralPressureSolver(dns); # Create random initial conditions u₀ = random_field(dns, T(0); kp, psolver = psolver_dns); + state = (; u = u₀, t = T(0)); GC.gc() @@ -175,7 +183,8 @@ fieldplot( @time state, outputs = solve_unsteady( dns, u₀, - (T(0), T(5e-1)); + # state.u, + (T(0), T(1e-1)); Δt, # Δt = T(1e-4), # Δt = T(1e-5), @@ -191,7 +200,7 @@ fieldplot( # # levels = 5, # docolorbar = false, # ), - obs = observe_u(dns, psolver_dns, filters; nupdate = 20), + # obs = observe_u(dns, psolver_dns, filters; nupdate = 20), # ehist = realtimeplotter(; # setup, # plot = energy_history_plot, @@ -199,7 +208,18 @@ fieldplot( # displayfig = false, # ), # espec = realtimeplotter(; setup, plot = energy_spectrum_plot, nupdate = 10), - # anim = animator(; setup, path = "$output/solution.mkv", nupdate = 20), + # anim = animator(; path = "$output/solution_Re$(Int(Re)).mkv", nupdate = 10, + # setup = dns, + # fieldname = :eig2field, + # levels = LinRange(T(2), T(10), 10), + # # levels = LinRange(T(4), T(12), 10), + # # levels = LinRange(-1.0f0, 3.0f0, 5), + # # levels = LinRange(-2.0f0, 2.0f0, 5), + # # levels = 5, + # docolorbar = false, + # # size = (800, 800), + # size = (600, 600), + # ), # vtk = vtk_writer(; setup, nupdate = 10, dir = output, filename = "solution"), # field = fieldsaver(; setup, nupdate = 10), log = timelogger(; nupdate = 5), @@ -210,49 +230,118 @@ CUDA.reclaim() # 103.5320324 -state.u[1] - -fieldplot( - (; u = u₀, t = T(0)); - setup = dns, - # type = image, - # colormap = :viridis, - docolorbar = false, - size = (500, 500), -) - -save("$output/vorticity_start.png", current_figure()) - -fieldplot( - state, - setup = dns, - # type = image, - # colormap = :viridis, - docolorbar = false, - size = (500, 500), -) - -save("$output/vorticity_end.png", current_figure()) +state.u -i = 1 -fieldplot( - (; u = filters[i].Φ(state.u, filters[i].setup, filters[i].compression), t = T(0)); - setup = filters[i].setup, - # type = image, - # colormap = :viridis, - docolorbar = false, - size = (500, 500), -) +state.u[2] + +fil = filters[2]; +apply_bc_u!(state.u, T(0), dns); +v = fil.Φ(state.u, fil.setup, fil.compression); +apply_bc_u!(v, T(0), fil.setup); +Fv = momentum(v, T(0), fil.setup); +apply_bc_u!(Fv, T(0), fil.setup); +PFv = project(Fv, fil.setup; psolver = fil.psolver); +apply_bc_u!(PFv, T(0), fil.setup); +F = momentum(state.u, T(0), dns); +apply_bc_u!(F, T(0), dns); +PF = project(F, dns; psolver = psolver_dns); +apply_bc_u!(PF, T(0), dns); +ΦPF = fil.Φ(PF, fil.setup, fil.compression); +apply_bc_u!(ΦPF, T(0), fil.setup); +c = ΦPF .- PFv +apply_bc_u!(c, T(0), fil.setup) + +with_theme(; fontsize = 25) do + fig = fieldplot( + (; u = u₀, t = T(0)); + setup = dns, + # type = image, + # colormap = :viridis, + docolorbar = false, + size = (500, 500), + title = "u₀" + ) + save("$output/priorfields/ustart.png", fig) + fig = fieldplot( + state, + setup = dns, + # type = image, + # colormap = :viridis, + docolorbar = false, + size = (500, 500), + title = "u", + ) + save("$output/priorfields/u.png", fig) + fig = fieldplot( + (; u = v, t = T(0)); + fil.setup, + # type = image, + # colormap = :viridis, + # fieldname = 1, + docolorbar = false, + size = (500, 500), + # title = "ubar" + title = "ū" + ) + save("$output/priorfields/v.png", fig) + fig = fieldplot( + (; u = PF, t = T(0)); + setup = dns, + # type = image, + # colormap = :viridis, + # fieldname = 1, + docolorbar = false, + size = (500, 500), + title = "PF(u)" + ) + save("$output/priorfields/PFu.png", fig) + fig = fieldplot( + (; u = PFv, t = T(0)); + fil.setup, + # type = image, + # colormap = :viridis, + # fieldname = 1, + docolorbar = false, + size = (500, 500), + # title = "PF(ubar)" + title = "P̄F̄(ū)" + ) + save("$output/priorfields/PFv.png", fig) + fig = fieldplot( + (; u = ΦPF, t = T(0)); + fil.setup, + # type = image, + # colormap = :viridis, + # fieldname = 1, + docolorbar = false, + size = (500, 500), + title = "ΦPF(u)" + ) + save("$output/priorfields/PhiPFu.png", fig) + fig = fieldplot( + (; u = c, t = T(0)); + fil.setup, + # type = image, + # colormap = :viridis, + # fieldname = 1, + # fieldname = :velocity, + docolorbar = false, + size = (500, 500), + # title = "c(u, ubar)" + title = "c(u, ū)" + ) + save("$output/priorfields/c.png", fig) +end -save("$output/vorticity_end_$(filters[i].compression).png", current_figure()) +#################################################################### fieldplot( - (; u = u₀, t = T(0)); - # state; + # (; u = u₀, t = T(0)); + state; setup = dns, fieldname = :eig2field, - levels = LinRange(T(2), T(10), 10), - # levels = LinRange(T(4), T(12), 10), + # levels = LinRange(T(2), T(10), 10), + levels = LinRange(T(4), T(12), 10), # levels = LinRange(-1.0f0, 3.0f0, 5), # levels = LinRange(-2.0f0, 2.0f0, 5), # levels = 5, @@ -261,18 +350,18 @@ fieldplot( size = (600, 600), ) -fname = "$output/lambda2_start.png" -fname = "$output/lambda2_end.png" +fname = "$output/prioranalysis/lambda2/Re$(Int(Re))_start.png" +fname = "$output/prioranalysis/lambda2/Re$(Int(Re))_end.png" save(fname, current_figure()) run(`convert $fname -trim $fname`) # Requires imagemagick -i = 2 +i = 3 fieldplot( (; u = filters[i].Φ(state.u, filters[i].setup, filters[i].compression), t = T(0)); setup = filters[i].setup, fieldname = :eig2field, - levels = LinRange(T(2), T(10), 10), - # levels = LinRange(T(4), T(12), 10), + # levels = LinRange(T(2), T(10), 10), + levels = LinRange(T(4), T(12), 10), # levels = LinRange(-1.0f0, 3.0f0, 5), # levels = LinRange(-2.0f0, 2.0f0, 5), # levels = 5, @@ -281,7 +370,7 @@ fieldplot( size = (600, 600), ) -fname = "$output/lambda2_end_filtered.png" +fname = "$output/prioranalysis/lambda2/Re$(Int(Re))_end_filtered.png" save(fname, current_figure()) run(`convert $fname -trim $fname`) # Requires imagemagick @@ -347,8 +436,8 @@ begin Pv = sum(o.Pv) / nt c = sum(o.c) / nt @printf( - # "%s\t%d^%d\t%.2g\t%.2g\t%.2g\t%.2g\n", - "%s &\t\$%d^%d\$ &\t\$%.2g\$ &\t\$%.2g\$ &\t\$%.2g\$ &\t\$%.2g\$\n", + "%s\t%d^%d\t%.2g\t%.2g\t%.2g\t%.2g\n", + # "%s &\t\$%d^%d\$ &\t\$%.2g\$ &\t\$%.2g\$ &\t\$%.2g\$ &\t\$%.2g\$\n", typeof(o.Φ), o.Mα, D, @@ -383,9 +472,19 @@ norm(ubar[1][les.grid.Iu[1]]) GLMakie.activate!() +# To free up memory +psolver_dns = nothing +fig = lines([1, 2, 3]) +GC.gc() +CUDA.reclaim() + using CairoMakie CairoMakie.activate!() +filters = map(filters) do f + (; f.Φ, f.setup, f.compression) +end + # Plot predicted spectra fig = with_theme(; palette = (; color = ["#3366cc", "#cc0000", "#669900", "#ffcc00"])) do fields = [state.u, u₀, (f.Φ(state.u, f.setup, f.compression) for f in filters)...] @@ -420,7 +519,9 @@ fig = with_theme(; palette = (; color = ["#3366cc", "#cc0000", "#669900", "#ffcc krange = [T(16), T(128)] elseif D == 3 # krange = [T(kmax)^(T(1.5) / 3), T(kmax)] - krange = [T(64), T(256)] + # krange = [T(64), T(256)] + # krange = [T(8), T(64)] + krange = [T(16), T(100)] end slope, slopelabel = D == 2 ? (-T(3), L"$\kappa^{-3}") : (-T(5 / 3), L"$\kappa^{-5/3}") @@ -452,15 +553,16 @@ fig = with_theme(; palette = (; color = ["#3366cc", "#cc0000", "#669900", "#ffcc # lines!(ax, 1:specs[2].kmax, specs[2].ehat; color = Cycled(4), label = "DNS, t = 0") lines!(ax, plotparts(1)...; color = Cycled(1), label = "DNS") lines!(ax, plotparts(2)...; color = Cycled(4), label = "DNS, t = 0") - lines!(ax, plotparts(3)...; color = Cycled(2), label = "Face average") + lines!(ax, plotparts(3)...; color = Cycled(2), label = "Filtered DNS (FA)") lines!(ax, plotparts(4)...; color = Cycled(2)) lines!(ax, plotparts(5)...; color = Cycled(2)) - lines!(ax, plotparts(6)...; color = Cycled(3), label = "Volume average") + lines!(ax, plotparts(6)...; color = Cycled(3), label = "Filtered DNS (VA)") lines!(ax, plotparts(7)...; color = Cycled(3)) lines!(ax, plotparts(8)...; color = Cycled(3)) lines!(ax, krange, inertia; color = Cycled(1), label = slopelabel, linestyle = :dash) D == 2 && axislegend(ax; position = :lb) - D == 3 && axislegend(ax; position = :rt) + # D == 3 && axislegend(ax; position = :rt) + D == 3 && axislegend(ax; position = :lb) autolimits!(ax) if D == 2 # limits!(ax, (T(0.68), T(520)), (T(1e-3), T(3e1))) @@ -470,12 +572,13 @@ fig = with_theme(; palette = (; color = ["#3366cc", "#cc0000", "#669900", "#ffcc # limits!(ax, ax.xaxis.attributes.limits[], (T(1e-1), T(3e1))) # limits!(ax, ax.xaxis.attributes.limits[], (T(1e-3), ax.yaxis.attributes.limits[][2])) # limits!(ax, ax.xaxis.attributes.limits[], (T(3e-3), T(2e0))) - limits!(ax, (T(8e-1), T(400)), (T(2e-3), T(1.5e0))) + # limits!(ax, (T(8e-1), T(400)), (T(2e-3), T(1.5e0))) + limits!(ax, (T(8e-1), T(200)), (T(4e-5), T(1.5e0))) end fig end GC.gc() CUDA.reclaim() -# save("$output/priorspectra_$(D)D_linear.pdf", fig) -save("$output/priorspectra_$(D)D_dyadic.pdf", fig) +# save("$output/prioranalysis/spectra_$(D)D_linear_Re$(Int(Re)).pdf", fig) +save("$output/prioranalysis/spectra_$(D)D_dyadic_Re$(Int(Re)).pdf", fig) From 8c1b4ab2fc6ffd179cfe1b6da4967df87571876e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Thu, 22 Feb 2024 08:16:59 +0100 Subject: [PATCH 247/379] Update script --- scratch/divfree.jl | 536 ++++++++++++++++++++++++++++++++++----------- 1 file changed, 412 insertions(+), 124 deletions(-) diff --git a/scratch/divfree.jl b/scratch/divfree.jl index 396316f4e..bca77d07d 100644 --- a/scratch/divfree.jl +++ b/scratch/divfree.jl @@ -31,6 +31,8 @@ getorder(i) = :first elseif i == 2 :last + elseif i == 3 + :second else error("Unknown order: $i") end @@ -98,9 +100,6 @@ data_train = [create_les_data(; params_train...) for _ = 1:5]; data_valid = [create_les_data(; params_valid...) for _ = 1:1]; data_test = create_les_data(; params_test...); -data_train[1].data[1].u[1][1] -data_test.data[6].u[1][1] - # Save filtered DNS data jldsave("output/divfree/data_train.jld2"; data_train) jldsave("output/divfree/data_valid.jld2"; data_valid) @@ -111,6 +110,21 @@ data_train = load("output/divfree/data_train.jld2", "data_train"); data_valid = load("output/divfree/data_valid.jld2", "data_valid"); data_test = load("output/divfree/data_test.jld2", "data_test"); +data_train[5].comptime +data_valid[1].comptime +data_test.comptime + +map(d -> d.comptime, data_train) + +sum(d -> d.comptime, data_train) / 60 + +data_test.comptime / 60 + +(sum(d -> d.comptime, data_train) + sum(d -> d.comptime, data_valid) + data_test.comptime) + +data_train[1].data[1].u[1][1] +data_test.data[6].u[1][1] + # Build LES setup and assemble operators getsetups(params) = [ Setup( @@ -229,6 +243,8 @@ end save("$output/training_data.pdf", fig) +# Architecture 1 +mname="balzac" closure, θ₀ = cnn(; setup = setups_train[1], radii = [2, 2, 2, 2], @@ -239,9 +255,23 @@ closure, θ₀ = cnn(; ); closure.chain +# Architecture 2 +mname="rimbaud" +closure, θ₀ = cnn(; + setup = setups_train[1], + radii = [2, 2, 2, 2, 2], + channels = [24, 24, 24, 24, params_train.D], + # activations = [leakyrelu, leakyrelu, leakyrelu, leakyrelu, identity], + activations = [tanh, tanh, tanh, tanh, identity], + use_bias = [true, true, true, true, false], + rng, +); +closure.chain +mkpath("output/divfree/$mname") + closure(device(io_train[1, 1].u[:, :, :, 1:50]), device(θ₀)); -# Train grid-specialized closure models +# A-priori training prior = map(CartesianIndices(size(io_train))) do I # Prepare training starttime = time() @@ -251,7 +281,6 @@ prior = map(CartesianIndices(size(io_train))) do I θ = T(1.0e0) * device(θ₀) loss = create_loss_prior(mean_squared_error, closure) opt = Optimisers.setup(Adam(T(1.0e-3)), θ) - callbackstate = Point2f[] it = rand(1:size(io_valid[ig, ifil].u, 4), 50) validset = device(map(v -> v[:, :, :, it], io_valid[ig, ifil])) (; callbackstate, callback) = create_callback( @@ -260,84 +289,106 @@ prior = map(CartesianIndices(size(io_train))) do I displayref = true, display_each_iteration = false, ) - (; opt, θ, callbackstate) = - train([d], loss, opt, θ; niter = 10000, ncallback = 20, callbackstate, callback) + (; opt, θ, callbackstate) = train( + [d], + loss, + opt, + θ; + niter = 10_000, + ncallback = 20, + callbackstate, + callback, + ) θ = callbackstate.θmin # Use best θ instead of last θ - comptime = time() - starttime - (; Array(θ), comptime) + prior = (; θ = Array(θ), comptime = time() - starttime, callbackstate.hist) + jldsave("output/divfree/$mname/prior_ifilter$(ifil)_igrid$(ig).jld2"; prior) + prior end clean() -prior = [(; θ = Array(θ), comptime = 0.0) for θ ∈ θ_cnn] - -# Save trained parameters -jldsave("output/divfree/prior.jld2"; prior) - # Load trained parameters -prior = load("output/divfree/prior.jld2")["prior"]; +prior = map(CartesianIndices(size(io_train))) do I + ig, ifil = I.I + name = "output/divfree/$mname/prior_ifilter$(ifil)_igrid$(ig).jld2" + load(name)["prior"] +end +θ_cnn_prior = [copyto!(device(θ₀), p.θ) for p in prior]; -# Extract component array -θ_cnn = [copyto!(device(θ₀), p.θ) for p in prior]; +θ_cnn_prior .|> extrema -ngrid, nfilter = size(io_train) -# post = map(CartesianIndices((ngrid, nfilter, 2))) do I +map(p -> p.comptime, prior) +map(p -> p.comptime, prior) |> vec +map(p -> p.comptime, prior) |> sum +map(p -> p.comptime, prior) |> sum |> x -> x / 60 +map(p -> p.comptime, prior) |> sum |> x -> x / 3600 -let I = CartesianIndex((3, 2, 1)) - clean() - starttime = time() - ig, ifil, iorder = I.I - println("iorder = $iorder, ifil = $ifil, ig = $ig") - setup = setups_train[ig] - psolver = SpectralPressureSolver(setup) - loss = IncompressibleNavierStokes.create_loss_post(; - setup, - psolver, - closure, - nupdate = 4, - projectorder = getorder(iorder), - ) - data = [(; u = d.data[ig, ifil].u, d.t) for d in data_train] - d = create_dataloader_post(data; device, nunroll = 10) - # θ = T(1e-1) * copy(θ_cnn[ig, ifil]) - # θ = copy(θ_cnn[ig, ifil]) - # θ = copy(θ_cnn_post) - θ = copy(θ_cnn_post[ig, ifil, iorder]) - # θ = device(θ₀) - opt = Optimisers.setup(Adam(T(1.0e-3)), θ) - it = 1:20 - data = data_valid[1] - data = (; u = device.(data.data[ig, ifil].u[it]), t = data.t[it]) - (; callbackstate, callback) = create_callback( - create_relerr_post(; - data, +# A-posteriori training ###################################################### + +let + ngrid, nfilter = size(io_train) + for iorder = 1:2, ifil = 1:nfilter, ig = 1:ngrid + clean() + starttime = time() + # (ig, ifil, iorder) == (3, 1, 1) || continue + # (ifil, iorder) == (1, 1) && continue + # (ifil, iorder) == (2, 2) || continue + println("iorder = $iorder, ifil = $ifil, ig = $ig") + setup = setups_train[ig] + psolver = SpectralPressureSolver(setup) + loss = IncompressibleNavierStokes.create_loss_post(; setup, psolver, - closure_model = wrappedclosure(closure, setup), + closure, + nupdate = 2, projectorder = getorder(iorder), - nupdate = 8, - ); - θ, - displayref = false, - ) - (; opt, θ, callbackstate) = - train([d], loss, opt, θ; niter = 1000, ncallback = 10, callbackstate, callback) - θ = callbackstate.θmin # Use best θ instead of last θ - post = (; θ = Array(θ), comptime = time() - starttime) - jldsave("output/divfree/post_iorder$(iorder)_ifil$(ifil)_ig$(ig).jld2"; post) - post -end; -clean() + ) + data = [(; u = d.data[ig, ifil].u, d.t) for d in data_train] + d = create_dataloader_post(data; device, nunroll = 20) + # θ = T(1e-1) * copy(θ_cnn_prior[ig, ifil]) + θ = copy(θ_cnn_prior[ig, ifil]) + # θ = copy(θ_cnn_post) + # θ = copy(θ_cnn_post[ig, ifil, iorder]) + # θ = device(θ₀) + opt = Optimisers.setup(Adam(T(1.0e-3)), θ) + it = 1:30 + data = data_valid[1] + data = (; u = device.(data.data[ig, ifil].u[it]), t = data.t[it]) + (; callbackstate, callback) = create_callback( + create_relerr_post(; + data, + setup, + psolver, + closure_model = wrappedclosure(closure, setup), + projectorder = getorder(iorder), + nupdate = 2, + ); + θ, + displayref = false, + ) + (; opt, θ, callbackstate) = + train([d], loss, opt, θ; niter = 2000, ncallback = 10, callbackstate, callback) + θ = callbackstate.θmin # Use best θ instead of last θ + post = (; θ = Array(θ), comptime = time() - starttime) + jldsave("output/divfree/$mname/post_iorder$(iorder)_ifil$(ifil)_ig$(ig).jld2"; post) + end; + clean() +end post = map(CartesianIndices((size(io_train)..., 2))) do I ig, ifil, iorder = I.I - name = "output/divfree/post_iorder$(iorder)_ifil$(ifil)_ig$(ig).jld2" + name = "output/divfree/$mname/post_iorder$(iorder)_ifil$(ifil)_ig$(ig).jld2" load(name)["post"] -end +end; θ_cnn_post = [copyto!(device(θ₀), p.θ) for p in post]; θ_cnn_post .|> extrema map(p -> p.comptime, post) +map(p -> p.comptime, post) |> x -> reshape(x, 6, 2) +map(p -> p.comptime, post) ./ 60 +map(p -> p.comptime, post) |> sum +map(p -> p.comptime, post) |> sum |> x -> x / 60 +map(p -> p.comptime, post) |> sum |> x -> x / 3600 # Train Smagorinsky model with Lpost (grid search) smag = map(CartesianIndices((size(io_train, 2), 2))) do I @@ -348,7 +399,7 @@ smag = map(CartesianIndices((size(io_train, 2), 2))) do I emin = T(Inf) isample = 1 it = 1:50 - for θ in LinRange(T(0), T(0.5), 51) + for θ in LinRange(T(0), T(0.5), 501) e = T(0) for igrid = 1:ngrid println("iorder = $iorder, ifil = $ifil, θ = $θ, igrid = $igrid") @@ -389,6 +440,9 @@ smag = load("output/divfree/smag.jld2")["smag"]; # Extract coefficients θ_smag = map(s -> s.θ, smag) +map(s -> s.comptime, smag) +map(s -> s.comptime, smag) |> sum + # lines(LinRange(T(0), T(1), 100), e_smag) # Compute posterior errors @@ -397,14 +451,15 @@ e_nm, e_smag, e_cnn, e_cnn_post = let e_smag = zeros(T, size(data_test.data)..., 2) e_cnn = zeros(T, size(data_test.data)..., 2) e_cnn_post = zeros(T, size(data_test.data)..., 2) - # for iorder = 1:1, ifil = 1:1, ig in 3 for iorder = 1:2, ifil = 1:2, ig = 1:size(data_test.data, 1) + # (ig, ifil, iorder) == (2, 2, 2) || continue println("iorder = $iorder, ifil = $ifil, ig = $ig") projectorder = getorder(iorder) setup = setups_test[ig] psolver = SpectralPressureSolver(setup) data = (; u = device.(data_test.data[ig, ifil].u), t = data_test.t) - nupdate = 20 + # nupdate = ig > 3 ? 4 : 2 + nupdate = 2 # No model # Only for closurefirst, since projectfirst is the same if iorder == 2 @@ -421,8 +476,8 @@ e_nm, e_smag, e_cnn, e_cnn_post = let nupdate, ) e_smag[ig, ifil, iorder] = err(θ_smag[ifil, iorder]) - CNN - Only the first grids are trained for + # CNN + # Only the first grids are trained for if ig ≤ size(data_train[1].data, 1) err = create_relerr_post(; data, @@ -433,7 +488,7 @@ e_nm, e_smag, e_cnn, e_cnn_post = let nupdate, # nupdate = 50, ) - e_cnn[ig, ifil, iorder] = err(θ_cnn[ig, ifil]) + e_cnn[ig, ifil, iorder] = err(θ_cnn_prior[ig, ifil]) e_cnn_post[ig, ifil, iorder] = err(θ_cnn_post[ig, ifil, iorder]) end end @@ -445,6 +500,9 @@ e_smag e_cnn e_cnn_post +data_train[1].t[2] - data_train[1].t[1] +data_test.t[2] - data_test.t[1] + CairoMakie.activate!() GLMakie.activate!() @@ -456,9 +514,10 @@ with_theme(; # fontsize = 20, palette = (; color = ["#3366cc", "#cc0000", "#669900", "#ffcc00"]), ) do - iorder = 1 + iorder = 2 # lesmodel = iorder == 1 ? "project-then-closure" : "closure-then-project" - lesmodel = iorder == 1 ? "project-then-closure" : "closure-then-project" + # lesmodel = iorder == 1 ? "project-then-closure" : "closure-then-project" + lesmodel = iorder == 1 ? "Gen" : "DFC" nles = [n[1] for n in params_test.nles] fig = Figure(; size = (500, 400)) ax = Axis( @@ -542,8 +601,10 @@ with_theme(; fig end -save("$output/convergence_Lprior_prosecond.pdf", current_figure()) -save("$output/convergence_Lprior_profirst.pdf", current_figure()) +name = "$output/convergence" +ispath(name) || mkpath(name) +save("$name/$(mname)_gen.pdf", current_figure()) +save("$name/$(mname)_dfc.pdf", current_figure()) # Energy evolution ########################################################### @@ -556,14 +617,14 @@ kineticenergy = let ke_cnn_prior = fill(zeros(T, 0), ngrid, nfilter, 2) ke_cnn_post = fill(zeros(T, 0), ngrid, nfilter, 2) for iorder = 1:2, ifil = 1:nfilter, ig = 1:ngrid - # for iorder = 1:1, ifil = 1:1, ig = 1:1 println("iorder = $iorder, ifil = $ifil, ig = $ig") setup = setups_test[ig] psolver = SpectralPressureSolver(setup) t = data_test.t u₀ = data_test.data[ig, ifil].u[1] |> device tlims = (t[1], t[end]) - nupdate = 50 + # nupdate = 50 + nupdate = 2 Δt = (t[2] - t[1]) / nupdate T = eltype(u₀[1]) ewriter = processor() do state @@ -585,14 +646,7 @@ kineticenergy = let data_test.data[ig, ifil].u, ) ke_nomodel[ig, ifil] = - solve_unsteady( - setup, - u₀, - tlims; - Δt, - processors, - psolver, - )[2].ewriter + solve_unsteady(setup, u₀, tlims; Δt, processors, psolver)[2].ewriter end ke_smag[ig, ifil, iorder] = solve_unsteady( @@ -620,7 +674,7 @@ kineticenergy = let Δt, processors, psolver, - θ = θ_cnn[ig, ifil], + θ = θ_cnn_prior[ig, ifil], )[2].ewriter ke_cnn_post[ig, ifil, iorder] = solve_unsteady( @@ -650,12 +704,14 @@ kineticenergy.ke_cnn_post[1] CairoMakie.activate!() with_theme(; palette = (; color = ["#3366cc", "#cc0000", "#669900", "#ffcc00"])) do - t = data_test.t[2:end] + # t = data_test.t[2:end] + t = data_test.t for iorder = 1:2, ifil = 1:2, igrid = 1:3 println("iorder = $iorder, ifil = $ifil, igrid = $igrid") - reflevel = kineticenergy.ke_ref[igrid, ifil][2:end] + # reflevel = kineticenergy.ke_ref[igrid, ifil][2:end] + reflevel = copy(kineticenergy.ke_ref[igrid, ifil]) reflevel = fill!(reflevel, 1) - lesmodel = iorder == 1 ? "P-first" : "P-last" + lesmodel = iorder == 1 ? "Gen" : "DCF" fil = ifil == 1 ? "FA" : "VA" nles = params_test.nles[igrid] fig = Figure(; size = (500, 400)) @@ -669,7 +725,8 @@ with_theme(; palette = (; color = ["#3366cc", "#cc0000", "#669900", "#ffcc00"])) lines!( ax, t, - kineticenergy.ke_ref[igrid, ifil][2:end] ./ reflevel; + # kineticenergy.ke_ref[igrid, ifil][2:end] ./ reflevel; + kineticenergy.ke_ref[igrid, ifil] ./ reflevel; color = Cycled(1), linestyle = :dash, label = "Reference", @@ -706,8 +763,224 @@ with_theme(; palette = (; color = ["#3366cc", "#cc0000", "#669900", "#ffcc00"])) iorder == 2 && axislegend(; position = :lb) # axislegend(; position = :lb) # axislegend() - name = "$output/energy_evolution" - save("$(name)_iorder$(iorder)_ifilter$(ifil)_igrid$(igrid).pdf", fig) + name = "$output/energy_evolution/$mname/" + ispath(name) || mkpath(name) + save("$(name)/iorder$(iorder)_ifilter$(ifil)_igrid$(igrid).pdf", fig) + end +end + +# Divergence ################################################################# + +divs = let + clean() + ngrid, nfilter = size(io_train) + d_ref = fill(zeros(T, 0), ngrid, nfilter) + d_nomodel = fill(zeros(T, 0), ngrid, nfilter, 3) + d_smag = fill(zeros(T, 0), ngrid, nfilter, 3) + d_cnn_prior = fill(zeros(T, 0), ngrid, nfilter, 3) + d_cnn_post = fill(zeros(T, 0), ngrid, nfilter, 3) + for iorder = 1:3, ifil = 1:nfilter, ig = 1:ngrid + println("iorder = $iorder, ifil = $ifil, ig = $ig") + setup = setups_test[ig] + psolver = SpectralPressureSolver(setup) + t = data_test.t + u₀ = data_test.data[ig, ifil].u[1] |> device + tlims = (t[1], t[end]) + # nupdate = 50 + nupdate = 2 + Δt = (t[2] - t[1]) / nupdate + T = eltype(u₀[1]) + dwriter = processor() do state + div = fill!(similar(setup.grid.x[1], setup.grid.N), 0) + dhist = zeros(T, 0) + on(state) do (; u, n) + if n % nupdate == 0 + IncompressibleNavierStokes.divergence!(div, u, setup) + d = view(div, setup.grid.Ip) + d = sum(abs2, d) / length(d) + d = sqrt(d) + push!(dhist, d) + end + end + state[] = state[] # Compute initial divergence + dhist + end + processors = (; dwriter) + if iorder == 1 + # Does not depend on projection order + d_ref[ig, ifil] = map(data_test.data[ig, ifil].u) do u + u = device(u) + div = IncompressibleNavierStokes.divergence(u, setup) + d = view(div, setup.grid.Ip) + d = sum(abs2, d) / length(d) + d = sqrt(d) + end + end + iorder_use = iorder == 3 ? 2 : iorder + d_nomodel[ig, ifil, iorder] = + solve_unsteady( + ( + setup..., + projectorder = getorder(iorder), + ), + u₀, tlims; Δt, processors, psolver)[2].dwriter + d_smag[ig, ifil, iorder] = + solve_unsteady( + (; + setup..., + projectorder = getorder(iorder), + closure_model = smagorinsky_closure(setup), + ), + u₀, + tlims; + Δt, + processors, + psolver, + θ = θ_smag[ifil, iorder_use], + )[2].dwriter + d_cnn_prior[ig, ifil, iorder] = + solve_unsteady( + (; + setup..., + projectorder = getorder(iorder), + closure_model = wrappedclosure(closure, setup), + ), + u₀, + tlims; + Δt, + processors, + psolver, + θ = θ_cnn_prior[ig, ifil], + )[2].dwriter + d_cnn_post[ig, ifil, iorder] = + solve_unsteady( + (; + setup..., + projectorder = getorder(iorder), + closure_model = wrappedclosure(closure, setup), + ), + u₀, + tlims; + Δt, + processors, + psolver, + θ = θ_cnn_post[ig, ifil, iorder_use], + )[2].dwriter + end + (; + d_ref, + d_nomodel, + d_smag, + d_cnn_prior, + d_cnn_post) +end; +clean(); + +# Save +jldsave("output/divfree/$(mname)_divs.jld2"; divs) + +# Load +divs = load("output/divfree/$(mname)_divs.jld2")["divs"] + +divs.d_ref .|> extrema +divs.d_nomodel .|> extrema +divs.d_smag .|> extrema +divs.d_cnn_prior .|> extrema +divs.d_cnn_post .|> extrema + +divs.d_ref[1, 1] |> lines +divs.d_ref[1, 2] |> lines! + +divs.d_nomodel[1, 1] |> lines +divs.d_nomodel[1, 2] |> lines! + +divs.d_cnn_post[1, 1, 3] |> lines +divs.d_cnn_post[1, 2, 3] |> lines! + +divs.d_smag[1, 2, 3] +divs.d_smag[1, 1, 3] +divs.d_smag[1, 2, 3] + +CairoMakie.activate!() + +with_theme(; + fontsize = 20, + palette = (; color = ["#3366cc", "#cc0000", "#669900", "#ffcc00"]), +) do + t = data_test.t + for islog in (true, false) + for iorder = 1:3, ifil = 1:2, igrid = 1:3 + # println("iorder = $iorder, igrid = $igrid") + println("iorder = $iorder, ifil = $ifil, igrid = $igrid") + lesmodel = + if iorder == 1 + "Gen" + elseif iorder == 2 + "DCF" + else + "DCF-RHS" + end + fil = ifil == 1 ? "FA" : "VA" + nles = params_test.nles[igrid] + fig = Figure(; size = (500, 400)) + yscale = islog ? log10 : identity + ax = Axis( + fig[1, 1]; + yscale, + xlabel = "t", + # ylabel = "Dv", + # title = "Divergence: $lesmodel, $nles", + title = "Divergence: $lesmodel, $fil, $nles", + # title = "Divergence: $lesmodel, $fil", + ) + linestyle = ifil == 1 ? :solid : :dash + lines!( + ax, + t, + divs.d_ref[igrid, ifil]; + color = Cycled(1), + linestyle = :dash, + label = "Reference", + ) + lines!( + ax, + t, + divs.d_nomodel[igrid, ifil, iorder]; + color = Cycled(1), + label = "No closure", + ) + lines!( + ax, + t, + divs.d_smag[igrid, ifil, iorder]; + color = Cycled(2), + label = "Smagorinsky", + ) + lines!( + ax, + t, + divs.d_cnn_prior[igrid, ifil, iorder]; + color = Cycled(3), + label = "CNN (prior)", + ) + lines!( + ax, + t, + divs.d_cnn_post[igrid, ifil, iorder]; + color = Cycled(4), + label = "CNN (post)", + ) + # axislegend() + # iorder == 1 && axislegend(; position = :lt) + # iorder == 2 && axislegend(; position = :lb) + iorder == 2 && ifil == 1 && axislegend(; position = :rt) + # axislegend() + islog && ylims!(ax, (T(1e-6), T(1e3))) + name = "$output/divergence/$mname/$(islog ? "log" : "lin")" + ispath(name) || mkpath(name) + # save("$(name)/iorder$(iorder)_igrid$(igrid).pdf", fig) + save("$(name)/iorder$(iorder)_ifilter$(ifil)_igrid$(igrid).pdf", fig) + end end end @@ -721,7 +994,6 @@ ufinal = let u_smag = fill(temp, ngrid, nfilter, 2) u_cnn_prior = fill(temp, ngrid, nfilter, 2) u_cnn_post = fill(temp, ngrid, nfilter, 2) - # for iorder = 1:1, ifil = 2:2, igrid = 3:3 for iorder = 1:2, ifil = 1:nfilter, igrid = 1:ngrid clean() println("iorder = $iorder, ifil = $ifil, igrid = $igrid") @@ -730,20 +1002,14 @@ ufinal = let psolver = SpectralPressureSolver(setup) u₀ = data_test.data[igrid, ifil].u[1] |> device tlims = (t[1], t[end]) - nupdate = 50 + nupdate = 2 Δt = (t[2] - t[1]) / nupdate T = eltype(u₀[1]) if iorder == 1 # Does not depend on projection order u_ref[igrid, ifil] = data_test.data[igrid, ifil].u[end] u_nomodel[igrid, ifil] = - solve_unsteady( - setup, - u₀, - tlims; - Δt, - psolver, - )[1].u .|> Array + solve_unsteady(setup, u₀, tlims; Δt, psolver)[1].u .|> Array end u_smag[igrid, ifil, iorder] = solve_unsteady( @@ -757,7 +1023,7 @@ ufinal = let Δt, psolver, θ = θ_smag[ifil, iorder], - )[1].u .|> Array + )[1].u .|> Array u_cnn_prior[igrid, ifil, iorder] = solve_unsteady( (; @@ -769,8 +1035,8 @@ ufinal = let tlims; Δt, psolver, - θ = θ_cnn[igrid, ifil], - )[1].u .|> Array + θ = θ_cnn_prior[igrid, ifil], + )[1].u .|> Array u_cnn_post[igrid, ifil, iorder] = solve_unsteady( (; @@ -783,7 +1049,7 @@ ufinal = let Δt, psolver, θ = θ_cnn_post[igrid, ifil, iorder], - )[1].u .|> Array + )[1].u .|> Array end (; u_ref, u_nomodel, u_smag, u_cnn_prior, u_cnn_post) end; @@ -793,9 +1059,9 @@ ufinal.u_ref[1][2] ufinal.u_cnn_prior[3, 1, 1][1] ufinal.u_cnn_post[3, 1, 1][1] -jldsave("output/divfree/ufinal.jld2"; ufinal) +jldsave("output/divfree/ufinal_$mname.jld2"; ufinal) -ufinal = load("output/divfree/ufinal.jld2")["ufinal"]; +ufinal = load("output/divfree/ufinal_$mname.jld2")["ufinal"]; markers_labels = [ (:circle, ":circle"), @@ -827,17 +1093,18 @@ markers_labels = [ fig = with_theme(; palette = (; color = ["#3366cc", "#cc0000", "#669900", "#ffcc00"])) do for iorder = 1:2, ifil = 1:2, igrid = 1:3 println("iorder = $iorder, ifil = $ifil, igrid = $igrid") - lesmodel = iorder == 1 ? "P-first" : "P-last" + lesmodel = iorder == 1 ? "Gen" : "DCF" fil = ifil == 1 ? "FA" : "VA" nles = params_test.nles[igrid] setup = setups_test[igrid] - fields = [ - ufinal.u_ref[igrid, ifil], - ufinal.u_nomodel[igrid, ifil], - ufinal.u_smag[igrid, ifil, iorder], - ufinal.u_cnn_prior[igrid, ifil, iorder], - ufinal.u_cnn_post[igrid, ifil, iorder], - ] .|> device + fields = + [ + ufinal.u_ref[igrid, ifil], + ufinal.u_nomodel[igrid, ifil], + ufinal.u_smag[igrid, ifil, iorder], + ufinal.u_cnn_prior[igrid, ifil, iorder], + ufinal.u_cnn_post[igrid, ifil, iorder], + ] .|> device (; Ip) = setup.grid (; A, κ, K) = IncompressibleNavierStokes.spectral_stuff(setup) specs = map(fields) do u @@ -887,8 +1154,9 @@ fig = with_theme(; palette = (; color = ["#3366cc", "#cc0000", "#669900", "#ffcc axislegend(ax; position = :cb) autolimits!(ax) # limits!(ax, (T(0.8), T(800)), (T(1e-10), T(1))) - name = "$output/energy_spectra" - save("$(name)_iorder$(iorder)_ifilter$(ifil)_igrid$(igrid).pdf", fig) + name = "$output/energy_spectra/$mname" + ispath(name) || mkpath(name) + save("$(name)/iorder$(iorder)_ifilter$(ifil)_igrid$(igrid).pdf", fig) end end clean(); @@ -897,7 +1165,10 @@ clean(); GLMakie.activate!() -with_theme(; fontsize = 25, palette = (; color = ["#3366cc", "#cc0000", "#669900", "#ffcc00"])) do +with_theme(; + fontsize = 25, + palette = (; color = ["#3366cc", "#cc0000", "#669900", "#ffcc00"]), +) do x1 = 0.3 x2 = 0.5 y1 = 0.5 @@ -910,12 +1181,12 @@ with_theme(; fontsize = 25, palette = (; color = ["#3366cc", "#cc0000", "#669900 Point2f(x1, y1), # Point2f(x2, y1), ] - path = "$output/les_fields" + path = "$output/les_fields/$mname" ispath(path) || mkpath(path) for iorder = 1:2, ifil = 1:2, igrid = 1:3 setup = setups_test[igrid] - name = "$(path)/iorder$(iorder)_ifilter$(ifil)_igrid$(igrid)" - lesmodel = iorder == 1 ? "P-first" : "P-last" + name = "$path/iorder$(iorder)_ifilter$(ifil)_igrid$(igrid)" + lesmodel = iorder == 1 ? "Gen" : "DCF" fil = ifil == 1 ? "FA" : "VA" nles = params_test.nles[igrid] function makeplot(u, title, suffix) @@ -933,10 +1204,27 @@ with_theme(; fontsize = 25, palette = (; color = ["#3366cc", "#cc0000", "#669900 save(fname, fig) # run(`convert $fname -trim $fname`) # Requires imagemagick end - iorder == 2 && makeplot(device(ufinal.u_ref[igrid, ifil]), "Reference, $fil, $nles", "ref") - iorder == 2 && makeplot(device(ufinal.u_nomodel[igrid, ifil]), "No closure, $fil, $nles", "nomodel") - makeplot(device(ufinal.u_smag[igrid, ifil, iorder]), "Smagorinsky, $lesmodel, $fil, $nles", "smag") - makeplot(device(ufinal.u_cnn_prior[igrid, ifil, iorder]), "CNN (prior), $lesmodel, $fil, $nles", "prior") - makeplot(device(ufinal.u_cnn_post[igrid, ifil, iorder]), "CNN (post), $lesmodel, $fil, $nles", "post") + iorder == 2 && + makeplot(device(ufinal.u_ref[igrid, ifil]), "Reference, $fil, $nles", "ref") + iorder == 2 && makeplot( + device(ufinal.u_nomodel[igrid, ifil]), + "No closure, $fil, $nles", + "nomodel", + ) + makeplot( + device(ufinal.u_smag[igrid, ifil, iorder]), + "Smagorinsky, $lesmodel, $fil, $nles", + "smag", + ) + makeplot( + device(ufinal.u_cnn_prior[igrid, ifil, iorder]), + "CNN (prior), $lesmodel, $fil, $nles", + "prior", + ) + makeplot( + device(ufinal.u_cnn_post[igrid, ifil, iorder]), + "CNN (post), $lesmodel, $fil, $nles", + "post", + ) end end From 6963f1f908621a01eadeaedbdc066e5c49fa2666 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Fri, 23 Feb 2024 20:22:58 +0100 Subject: [PATCH 248/379] Format --- scratch/divergence.jl | 12 +-- scratch/divfree.jl | 174 +++++++++++++++++++++--------------------- 2 files changed, 91 insertions(+), 95 deletions(-) diff --git a/scratch/divergence.jl b/scratch/divergence.jl index 6b5c3cad3..8dc6a39b0 100644 --- a/scratch/divergence.jl +++ b/scratch/divergence.jl @@ -259,7 +259,7 @@ with_theme(; fontsize = 25) do # colormap = :viridis, docolorbar = false, size = (500, 500), - title = "u₀" + title = "u₀", ) save("$output/priorfields/ustart.png", fig) fig = fieldplot( @@ -281,7 +281,7 @@ with_theme(; fontsize = 25) do docolorbar = false, size = (500, 500), # title = "ubar" - title = "ū" + title = "ū", ) save("$output/priorfields/v.png", fig) fig = fieldplot( @@ -292,7 +292,7 @@ with_theme(; fontsize = 25) do # fieldname = 1, docolorbar = false, size = (500, 500), - title = "PF(u)" + title = "PF(u)", ) save("$output/priorfields/PFu.png", fig) fig = fieldplot( @@ -304,7 +304,7 @@ with_theme(; fontsize = 25) do docolorbar = false, size = (500, 500), # title = "PF(ubar)" - title = "P̄F̄(ū)" + title = "P̄F̄(ū)", ) save("$output/priorfields/PFv.png", fig) fig = fieldplot( @@ -315,7 +315,7 @@ with_theme(; fontsize = 25) do # fieldname = 1, docolorbar = false, size = (500, 500), - title = "ΦPF(u)" + title = "ΦPF(u)", ) save("$output/priorfields/PhiPFu.png", fig) fig = fieldplot( @@ -328,7 +328,7 @@ with_theme(; fontsize = 25) do docolorbar = false, size = (500, 500), # title = "c(u, ubar)" - title = "c(u, ū)" + title = "c(u, ū)", ) save("$output/priorfields/c.png", fig) end diff --git a/scratch/divfree.jl b/scratch/divfree.jl index bca77d07d..6cc715fcf 100644 --- a/scratch/divfree.jl +++ b/scratch/divfree.jl @@ -244,7 +244,7 @@ end save("$output/training_data.pdf", fig) # Architecture 1 -mname="balzac" +mname = "balzac" closure, θ₀ = cnn(; setup = setups_train[1], radii = [2, 2, 2, 2], @@ -256,7 +256,7 @@ closure, θ₀ = cnn(; closure.chain # Architecture 2 -mname="rimbaud" +mname = "rimbaud" closure, θ₀ = cnn(; setup = setups_train[1], radii = [2, 2, 2, 2, 2], @@ -370,7 +370,7 @@ let θ = callbackstate.θmin # Use best θ instead of last θ post = (; θ = Array(θ), comptime = time() - starttime) jldsave("output/divfree/$mname/post_iorder$(iorder)_ifil$(ifil)_ig$(ig).jld2"; post) - end; + end clean() end @@ -809,21 +809,23 @@ divs = let if iorder == 1 # Does not depend on projection order d_ref[ig, ifil] = map(data_test.data[ig, ifil].u) do u - u = device(u) - div = IncompressibleNavierStokes.divergence(u, setup) - d = view(div, setup.grid.Ip) - d = sum(abs2, d) / length(d) - d = sqrt(d) - end + u = device(u) + div = IncompressibleNavierStokes.divergence(u, setup) + d = view(div, setup.grid.Ip) + d = sum(abs2, d) / length(d) + d = sqrt(d) + end end iorder_use = iorder == 3 ? 2 : iorder d_nomodel[ig, ifil, iorder] = solve_unsteady( - ( - setup..., - projectorder = getorder(iorder), - ), - u₀, tlims; Δt, processors, psolver)[2].dwriter + (setup..., projectorder = getorder(iorder)), + u₀, + tlims; + Δt, + processors, + psolver, + )[2].dwriter d_smag[ig, ifil, iorder] = solve_unsteady( (; @@ -867,12 +869,7 @@ divs = let θ = θ_cnn_post[ig, ifil, iorder_use], )[2].dwriter end - (; - d_ref, - d_nomodel, - d_smag, - d_cnn_prior, - d_cnn_post) + (; d_ref, d_nomodel, d_smag, d_cnn_prior, d_cnn_post) end; clean(); @@ -903,84 +900,83 @@ divs.d_smag[1, 2, 3] CairoMakie.activate!() -with_theme(; +with_theme(; fontsize = 20, palette = (; color = ["#3366cc", "#cc0000", "#669900", "#ffcc00"]), ) do t = data_test.t for islog in (true, false) - for iorder = 1:3, ifil = 1:2, igrid = 1:3 - # println("iorder = $iorder, igrid = $igrid") - println("iorder = $iorder, ifil = $ifil, igrid = $igrid") - lesmodel = - if iorder == 1 + for iorder = 1:3, ifil = 1:2, igrid = 1:3 + # println("iorder = $iorder, igrid = $igrid") + println("iorder = $iorder, ifil = $ifil, igrid = $igrid") + lesmodel = if iorder == 1 "Gen" - elseif iorder == 2 + elseif iorder == 2 "DCF" - else + else "DCF-RHS" end - fil = ifil == 1 ? "FA" : "VA" - nles = params_test.nles[igrid] - fig = Figure(; size = (500, 400)) - yscale = islog ? log10 : identity - ax = Axis( - fig[1, 1]; - yscale, - xlabel = "t", - # ylabel = "Dv", - # title = "Divergence: $lesmodel, $nles", - title = "Divergence: $lesmodel, $fil, $nles", - # title = "Divergence: $lesmodel, $fil", - ) - linestyle = ifil == 1 ? :solid : :dash - lines!( - ax, - t, - divs.d_ref[igrid, ifil]; - color = Cycled(1), - linestyle = :dash, - label = "Reference", - ) - lines!( - ax, - t, - divs.d_nomodel[igrid, ifil, iorder]; - color = Cycled(1), - label = "No closure", - ) - lines!( - ax, - t, - divs.d_smag[igrid, ifil, iorder]; - color = Cycled(2), - label = "Smagorinsky", - ) - lines!( - ax, - t, - divs.d_cnn_prior[igrid, ifil, iorder]; - color = Cycled(3), - label = "CNN (prior)", - ) - lines!( - ax, - t, - divs.d_cnn_post[igrid, ifil, iorder]; - color = Cycled(4), - label = "CNN (post)", - ) - # axislegend() - # iorder == 1 && axislegend(; position = :lt) - # iorder == 2 && axislegend(; position = :lb) - iorder == 2 && ifil == 1 && axislegend(; position = :rt) - # axislegend() - islog && ylims!(ax, (T(1e-6), T(1e3))) - name = "$output/divergence/$mname/$(islog ? "log" : "lin")" - ispath(name) || mkpath(name) - # save("$(name)/iorder$(iorder)_igrid$(igrid).pdf", fig) - save("$(name)/iorder$(iorder)_ifilter$(ifil)_igrid$(igrid).pdf", fig) - end + fil = ifil == 1 ? "FA" : "VA" + nles = params_test.nles[igrid] + fig = Figure(; size = (500, 400)) + yscale = islog ? log10 : identity + ax = Axis( + fig[1, 1]; + yscale, + xlabel = "t", + # ylabel = "Dv", + # title = "Divergence: $lesmodel, $nles", + title = "Divergence: $lesmodel, $fil, $nles", + # title = "Divergence: $lesmodel, $fil", + ) + linestyle = ifil == 1 ? :solid : :dash + lines!( + ax, + t, + divs.d_ref[igrid, ifil]; + color = Cycled(1), + linestyle = :dash, + label = "Reference", + ) + lines!( + ax, + t, + divs.d_nomodel[igrid, ifil, iorder]; + color = Cycled(1), + label = "No closure", + ) + lines!( + ax, + t, + divs.d_smag[igrid, ifil, iorder]; + color = Cycled(2), + label = "Smagorinsky", + ) + lines!( + ax, + t, + divs.d_cnn_prior[igrid, ifil, iorder]; + color = Cycled(3), + label = "CNN (prior)", + ) + lines!( + ax, + t, + divs.d_cnn_post[igrid, ifil, iorder]; + color = Cycled(4), + label = "CNN (post)", + ) + # axislegend() + # iorder == 1 && axislegend(; position = :lt) + # iorder == 2 && axislegend(; position = :lb) + iorder == 2 && ifil == 1 && axislegend(; position = :rt) + # axislegend() + islog && ylims!(ax, (T(1e-6), T(1e3))) + name = "$output/divergence/$mname/$(islog ? "log" : "lin")" + ispath(name) || mkpath(name) + # save("$(name)/iorder$(iorder)_igrid$(igrid).pdf", fig) + save("$(name)/iorder$(iorder)_ifilter$(ifil)_igrid$(igrid).pdf", fig) + end end end From 559496049c792c41aa8fcac739c4e3b441c210b8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Fri, 23 Feb 2024 20:24:17 +0100 Subject: [PATCH 249/379] Rename scripts --- scratch/{divfree.jl => postanalysis.jl} | 0 scratch/{divergence.jl => prioranalysis.jl} | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename scratch/{divfree.jl => postanalysis.jl} (100%) rename scratch/{divergence.jl => prioranalysis.jl} (100%) diff --git a/scratch/divfree.jl b/scratch/postanalysis.jl similarity index 100% rename from scratch/divfree.jl rename to scratch/postanalysis.jl diff --git a/scratch/divergence.jl b/scratch/prioranalysis.jl similarity index 100% rename from scratch/divergence.jl rename to scratch/prioranalysis.jl From 680a6359c6bca7e92ca5cefd842e839dd11de476 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Fri, 23 Feb 2024 21:37:58 +0100 Subject: [PATCH 250/379] Add low memory Poisson solver --- src/IncompressibleNavierStokes.jl | 3 +- src/solvers/pressure/poisson.jl | 38 ++++++++++++++++++ src/solvers/pressure/solvers.jl | 64 ++++++++++++++++++++++++++----- 3 files changed, 95 insertions(+), 10 deletions(-) diff --git a/src/IncompressibleNavierStokes.jl b/src/IncompressibleNavierStokes.jl index 82111a4c7..17aa76f41 100644 --- a/src/IncompressibleNavierStokes.jl +++ b/src/IncompressibleNavierStokes.jl @@ -113,7 +113,8 @@ export Setup export stretched_grid, cosine_grid # Pressure solvers -export DirectPressureSolver, CGPressureSolver, SpectralPressureSolver +export DirectPressureSolver, + CGPressureSolver, SpectralPressureSolver, LowMemorySpectralPressureSolver # Solvers export solve_unsteady, solve_steady_state diff --git a/src/solvers/pressure/poisson.jl b/src/solvers/pressure/poisson.jl index c46371a66..ee06fc42a 100644 --- a/src/solvers/pressure/poisson.jl +++ b/src/solvers/pressure/poisson.jl @@ -170,3 +170,41 @@ function poisson!(solver::SpectralPressureSolver, p, f) p end + +function poisson!(solver::LowMemorySpectralPressureSolver, p, f) + (; setup, ahat, phat) = solver + (; dimension, Ip) = setup.grid + D = dimension() + + f = view(f, Ip) + + phat .= complex.(f) + + # Fourier transform of right hand side + fft!(phat) + + # Solve for coefficients in Fourier space + if D == 2 + ax = ahat + ay = reshape(ahat, 1, :) + @. phat = -phat / (ax + ay) + + else + ax = ahat + ay = reshape(ahat, 1, :) + az = reshape(ahat, 1, 1, :) + @. phat = -phat / (ax + ay + az) + end + + # Pressure is determined up to constant. We set this to 0 (instead of + # phat[1] / 0 = Inf) + # Note use of singleton range 1:1 instead of scalar index 1 + # (otherwise CUDA gets annoyed) + phat[1:1] .= 0 + + # Transform back + ifft!(phat) + @. p[Ip] = real(phat) + + p +end diff --git a/src/solvers/pressure/solvers.jl b/src/solvers/pressure/solvers.jl index c8b96e78c..45b94f2d9 100644 --- a/src/solvers/pressure/solvers.jl +++ b/src/solvers/pressure/solvers.jl @@ -196,15 +196,6 @@ struct SpectralPressureSolver{T,A<:AbstractArray{Complex{T}},S,P} <: plan::P end -# This moves all the inner arrays to the GPU when calling -# `cu(::SpectralPressureSolver)` from CUDA.jl -Adapt.adapt_structure(to, s::SpectralPressureSolver) = SpectralPressureSolver( - adapt(to, s.Ahat), - adapt(to, s.phat), - adapt(to, s.fhat), - adapt(to, s.plan), -) - """ SpectralPressureSolver(setup) @@ -253,6 +244,8 @@ function SpectralPressureSolver(setup) # Pressure is determined up to constant. By setting the constant # scaling factor to 1, we preserve the average. + # Note use of singleton range 1:1 instead of scalar index 1 + # (otherwise CUDA gets annoyed) Ahat[1:1] .= 1 # Placeholders for intermediate results @@ -268,3 +261,56 @@ function SpectralPressureSolver(setup) plan, ) end + +struct LowMemorySpectralPressureSolver{ + T, + A<:AbstractArray{T}, + P<:AbstractArray{Complex{T}}, + S, +} <: AbstractPressureSolver{T} + setup::S + ahat::A + phat::P +end + +""" + LowMemorySpectralPressureSolver(setup) + +Build spectral pressure solver from setup. +This one is slower than `SpectralPressureSolver` but occupies less memory. +""" +function LowMemorySpectralPressureSolver(setup) + (; grid, boundary_conditions) = setup + (; dimension, Δ, Np, x) = grid + + D = dimension() + T = eltype(Δ[1]) + + Δx = Array(Δ[1])[1] + + @assert( + all(bc -> bc[1] isa PeriodicBC && bc[2] isa PeriodicBC, boundary_conditions), + "SpectralPressureSolver only implemented for periodic boundary conditions", + ) + + @assert( + all(α -> all(≈(Δx), Δ[α]), 1:D), + "SpectralPressureSolver requires uniform grid along each dimension", + ) + + @assert all(n -> n == Np[1], Np) + + # Fourier transform of the discretization + # Assuming uniform grid, although Δx[1] and Δx[2] do not need to be the same + + k = 0:Np[1]-1 + + Tπ = T(π) # CUDA doesn't like pi + ahat = fill!(similar(x[1], Np[1]), 0) + @. ahat = 4 * Δx^D * sin(k * Tπ / Np[1])^2 / Δx^2 + + # Placeholders for intermediate results + phat = fill!(similar(x[1], Complex{T}, Np), 0) + + LowMemorySpectralPressureSolver{T,typeof(ahat),typeof(phat),typeof(setup)}(setup, ahat, phat) +end From 5fd38c3ec0fe449b56df35e5401533da2f0aabca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Thu, 29 Feb 2024 09:44:27 +0100 Subject: [PATCH 251/379] Add prior errors --- scratch/postanalysis.jl | 143 +++++++++++++++++++++++++++++++++++++++- 1 file changed, 140 insertions(+), 3 deletions(-) diff --git a/scratch/postanalysis.jl b/scratch/postanalysis.jl index 6cc715fcf..b4ce584ff 100644 --- a/scratch/postanalysis.jl +++ b/scratch/postanalysis.jl @@ -144,6 +144,7 @@ data_train[1].data[1, 1].u[end][1] # Create input/output arrays io_train = create_io_arrays(data_train, setups_train); io_valid = create_io_arrays(data_valid, setups_valid); +io_test = create_io_arrays([data_test], setups_test); # jldsave("output/divfree/io_train.jld2"; io_train) # jldsave("output/divfree/io_train.jld2"; io_valid) @@ -445,7 +446,32 @@ map(s -> s.comptime, smag) |> sum # lines(LinRange(T(0), T(1), 100), e_smag) -# Compute posterior errors +# Compute a-priori errors ################################################### + +eprior = let + prior = zeros(T, 3, 2) + post = zeros(T, 3, 2, 2) + for ig = 1:3, ifil = 1:2 + println("ig = $ig, ifil = $ifil") + testset = device(io_test[ig, ifil]) + err = create_relerr_prior(closure, testset...) + prior[ig, ifil] = err(θ_cnn_prior[ig, ifil]) + for iorder = 1:2 + post[ig, ifil, iorder] = err(θ_cnn_post[ig, ifil, iorder]) + end + end + (; prior, post) +end +clean() + +eprior.prior +eprior.post + +eprior.prior |> x -> reshape(x, :) |> x -> round.(x; digits = 2) +eprior.post |> x -> reshape(x, :, 2) |> x -> round.(x; digits = 2) + +# Compute posterior errors #################################################### + e_nm, e_smag, e_cnn, e_cnn_post = let e_nm = zeros(T, size(data_test.data)...) e_smag = zeros(T, size(data_test.data)..., 2) @@ -500,13 +526,120 @@ e_smag e_cnn e_cnn_post +round.([e_nm[:] reshape(e_smag, :, 2) reshape(e_cnn, :, 2) reshape(e_cnn_post, :, 2)][[1:3; 6:8], :]; sigdigits = 2) + data_train[1].t[2] - data_train[1].t[1] data_test.t[2] - data_test.t[1] CairoMakie.activate!() GLMakie.activate!() -# Plot convergence +# Plot a-priori errors ######################################################## + +fig = with_theme(; + # linewidth = 5, + # markersize = 10, + # markersize = 20, + # fontsize = 20, + palette = (; color = ["#3366cc", "#cc0000", "#669900", "#ffcc00"]), +) do + nles = [n[1] for n in params_test.nles][1:3] + fig = Figure(; size = (500, 400)) + ax = Axis( + fig[1, 1]; + xscale = log10, + # yscale = log10, + xticks = nles, + xlabel = "Resolution", + title = "Relative a-priori error", + ) + for ifil = 1:2 + linestyle = ifil == 1 ? :solid : :dash + label = "No closure" + # label = label * (ifil == 1 ? " (FA)" : " (VA)") + ifil == 2 && (label = nothing) + scatterlines!( + nles, + ones(T, length(nles)); + color = Cycled(1), + linestyle, + marker = :circle, + label, + ) + end + # for ifil = 1:2 + # linestyle = ifil == 1 ? :solid : :dash + # label = "Smagorinsky" + # # label = label * (ifil == 1 ? " (FA)" : " (VA)") + # ifil == 2 && (label = nothing) + # scatterlines!( + # nles, + # eprior.smag[:, ifil, iorder]; + # color = Cycled(2), + # linestyle, + # marker = :utriangle, + # label, + # ) + # end + for ifil = 1:2 + linestyle = ifil == 1 ? :solid : :dash + label = "CNN (Lprior)" + # label = label * (ifil == 1 ? " (FA)" : " (VA)") + ifil == 2 && (label = nothing) + scatterlines!( + nles, + eprior.prior[:, ifil]; + color = Cycled(2), + linestyle, + marker = :utriangle, + label, + ) + end + for ifil = 1:2 + linestyle = ifil == 1 ? :solid : :dash + label = "CNN (Lpost, Gen)" + # label = label * (ifil == 1 ? " (FA)" : " (VA)") + ifil == 2 && (label = nothing) + scatterlines!( + nles, + eprior.post[:, ifil, 1]; + color = Cycled(3), + linestyle, + marker = :rect, + label, + ) + end + for ifil = 1:2 + linestyle = ifil == 1 ? :solid : :dash + label = "CNN (Lpost, DFC)" + # label = label * (ifil == 1 ? " (FA)" : " (VA)") + ifil == 2 && (label = nothing) + scatterlines!( + nles, + eprior.post[:, ifil, 2]; + color = Cycled(4), + linestyle, + marker = :diamond, + label, + ) + end + # lines!( + # collect(extrema(nles[4:end])), + # n -> 2e4 * n^-2.0; + # linestyle = :dash, + # label = "n⁻²", + # color = Cycled(1), + # ) + axislegend(; position = :lb) + ylims!(ax, (T(-0.05), T(1.05))) + # iorder == 2 && limits!(ax, (T(60), T(1050)), (T(2e-2), T(1e1))) + fig +end + +save("$output/convergence/$(mname)_prior.pdf", fig) + +# Plot convergence ############################################################ + with_theme(; # linewidth = 5, # markersize = 10, @@ -703,6 +836,8 @@ kineticenergy.ke_cnn_post[1] CairoMakie.activate!() +# Plot energy evolution ######################################################## + with_theme(; palette = (; color = ["#3366cc", "#cc0000", "#669900", "#ffcc00"])) do # t = data_test.t[2:end] t = data_test.t @@ -769,7 +904,7 @@ with_theme(; palette = (; color = ["#3366cc", "#cc0000", "#669900", "#ffcc00"])) end end -# Divergence ################################################################# +# Compute Divergence ########################################################## divs = let clean() @@ -900,6 +1035,8 @@ divs.d_smag[1, 2, 3] CairoMakie.activate!() +# Plot Divergence ############################################################# + with_theme(; fontsize = 20, palette = (; color = ["#3366cc", "#cc0000", "#669900", "#ffcc00"]), From e9b8857a6323b5a56ebfe00de2455422434454ff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Thu, 29 Feb 2024 10:20:48 +0100 Subject: [PATCH 252/379] Fix typos --- scratch/postanalysis.jl | 72 +++++++++++++++++++---------------------- 1 file changed, 34 insertions(+), 38 deletions(-) diff --git a/scratch/postanalysis.jl b/scratch/postanalysis.jl index b4ce584ff..a4637f26a 100644 --- a/scratch/postanalysis.jl +++ b/scratch/postanalysis.jl @@ -262,13 +262,13 @@ closure, θ₀ = cnn(; setup = setups_train[1], radii = [2, 2, 2, 2, 2], channels = [24, 24, 24, 24, params_train.D], - # activations = [leakyrelu, leakyrelu, leakyrelu, leakyrelu, identity], activations = [tanh, tanh, tanh, tanh, identity], use_bias = [true, true, true, true, false], rng, ); closure.chain -mkpath("output/divfree/$mname") +savepath = "output/divfree/$mname" +ispath(savepath) || mkpath(savepath) closure(device(io_train[1, 1].u[:, :, :, 1:50]), device(θ₀)); @@ -302,7 +302,7 @@ prior = map(CartesianIndices(size(io_train))) do I ) θ = callbackstate.θmin # Use best θ instead of last θ prior = (; θ = Array(θ), comptime = time() - starttime, callbackstate.hist) - jldsave("output/divfree/$mname/prior_ifilter$(ifil)_igrid$(ig).jld2"; prior) + jldsave("$savepath/prior_ifilter$(ifil)_igrid$(ig).jld2"; prior) prior end clean() @@ -310,9 +310,9 @@ clean() # Load trained parameters prior = map(CartesianIndices(size(io_train))) do I ig, ifil = I.I - name = "output/divfree/$mname/prior_ifilter$(ifil)_igrid$(ig).jld2" + name = "$savepath/prior_ifilter$(ifil)_igrid$(ig).jld2" load(name)["prior"] -end +end; θ_cnn_prior = [copyto!(device(θ₀), p.θ) for p in prior]; θ_cnn_prior .|> extrema @@ -370,14 +370,14 @@ let train([d], loss, opt, θ; niter = 2000, ncallback = 10, callbackstate, callback) θ = callbackstate.θmin # Use best θ instead of last θ post = (; θ = Array(θ), comptime = time() - starttime) - jldsave("output/divfree/$mname/post_iorder$(iorder)_ifil$(ifil)_ig$(ig).jld2"; post) + jldsave("$savepath/post_iorder$(iorder)_ifil$(ifil)_ig$(ig).jld2"; post) end clean() end post = map(CartesianIndices((size(io_train)..., 2))) do I ig, ifil, iorder = I.I - name = "output/divfree/$mname/post_iorder$(iorder)_ifil$(ifil)_ig$(ig).jld2" + name = "$savepath/post_iorder$(iorder)_ifil$(ifil)_ig$(ig).jld2" load(name)["post"] end; θ_cnn_post = [copyto!(device(θ₀), p.θ) for p in post]; @@ -544,6 +544,7 @@ fig = with_theme(; palette = (; color = ["#3366cc", "#cc0000", "#669900", "#ffcc00"]), ) do nles = [n[1] for n in params_test.nles][1:3] + ifil = 2 fig = Figure(; size = (500, 400)) ax = Axis( fig[1, 1]; @@ -551,13 +552,15 @@ fig = with_theme(; # yscale = log10, xticks = nles, xlabel = "Resolution", - title = "Relative a-priori error", + # title = "Relative a-priori error", + title = "Relative a-priori error $(ifil == 1 ? " (FA)" : " (VA)")", ) - for ifil = 1:2 - linestyle = ifil == 1 ? :solid : :dash + linestyle = :solid + # for ifil = 1:2 + # linestyle = ifil == 1 ? :solid : :dash label = "No closure" # label = label * (ifil == 1 ? " (FA)" : " (VA)") - ifil == 2 && (label = nothing) + # ifil == 2 && (label = nothing) scatterlines!( nles, ones(T, length(nles)); @@ -566,7 +569,7 @@ fig = with_theme(; marker = :circle, label, ) - end + # end # for ifil = 1:2 # linestyle = ifil == 1 ? :solid : :dash # label = "Smagorinsky" @@ -581,11 +584,11 @@ fig = with_theme(; # label, # ) # end - for ifil = 1:2 - linestyle = ifil == 1 ? :solid : :dash + # for ifil = 1:2 + # linestyle = ifil == 1 ? :solid : :dash label = "CNN (Lprior)" # label = label * (ifil == 1 ? " (FA)" : " (VA)") - ifil == 2 && (label = nothing) + # ifil == 2 && (label = nothing) scatterlines!( nles, eprior.prior[:, ifil]; @@ -594,12 +597,12 @@ fig = with_theme(; marker = :utriangle, label, ) - end - for ifil = 1:2 - linestyle = ifil == 1 ? :solid : :dash + # end + # for ifil = 1:2 + # linestyle = ifil == 1 ? :solid : :dash label = "CNN (Lpost, Gen)" # label = label * (ifil == 1 ? " (FA)" : " (VA)") - ifil == 2 && (label = nothing) + # ifil == 2 && (label = nothing) scatterlines!( nles, eprior.post[:, ifil, 1]; @@ -608,12 +611,12 @@ fig = with_theme(; marker = :rect, label, ) - end - for ifil = 1:2 - linestyle = ifil == 1 ? :solid : :dash - label = "CNN (Lpost, DFC)" + # end + # for ifil = 1:2 + # linestyle = ifil == 1 ? :solid : :dash + label = "CNN (Lpost, DCF)" # label = label * (ifil == 1 ? " (FA)" : " (VA)") - ifil == 2 && (label = nothing) + # ifil == 2 && (label = nothing) scatterlines!( nles, eprior.post[:, ifil, 2]; @@ -622,7 +625,7 @@ fig = with_theme(; marker = :diamond, label, ) - end + # end # lines!( # collect(extrema(nles[4:end])), # n -> 2e4 * n^-2.0; @@ -633,11 +636,10 @@ fig = with_theme(; axislegend(; position = :lb) ylims!(ax, (T(-0.05), T(1.05))) # iorder == 2 && limits!(ax, (T(60), T(1050)), (T(2e-2), T(1e1))) + save("$output/convergence/$(mname)_prior_ifilter$ifil.pdf", fig) fig end -save("$output/convergence/$(mname)_prior.pdf", fig) - # Plot convergence ############################################################ with_theme(; @@ -647,10 +649,10 @@ with_theme(; # fontsize = 20, palette = (; color = ["#3366cc", "#cc0000", "#669900", "#ffcc00"]), ) do - iorder = 2 + iorder = 1 # lesmodel = iorder == 1 ? "project-then-closure" : "closure-then-project" # lesmodel = iorder == 1 ? "project-then-closure" : "closure-then-project" - lesmodel = iorder == 1 ? "Gen" : "DFC" + lesmodel = iorder == 1 ? "Gen" : "DCF" nles = [n[1] for n in params_test.nles] fig = Figure(; size = (500, 400)) ax = Axis( @@ -737,7 +739,7 @@ end name = "$output/convergence" ispath(name) || mkpath(name) save("$name/$(mname)_gen.pdf", current_figure()) -save("$name/$(mname)_dfc.pdf", current_figure()) +save("$name/$(mname)_dcf.pdf", current_figure()) # Energy evolution ########################################################### @@ -1008,12 +1010,6 @@ divs = let end; clean(); -# Save -jldsave("output/divfree/$(mname)_divs.jld2"; divs) - -# Load -divs = load("output/divfree/$(mname)_divs.jld2")["divs"] - divs.d_ref .|> extrema divs.d_nomodel .|> extrema divs.d_smag .|> extrema @@ -1192,9 +1188,9 @@ ufinal.u_ref[1][2] ufinal.u_cnn_prior[3, 1, 1][1] ufinal.u_cnn_post[3, 1, 1][1] -jldsave("output/divfree/ufinal_$mname.jld2"; ufinal) +jldsave("$savepath/ufinal.jld2"; ufinal) -ufinal = load("output/divfree/ufinal_$mname.jld2")["ufinal"]; +ufinal = load("$savepath/ufinal.jld2")["ufinal"]; markers_labels = [ (:circle, ":circle"), From f7eda50fbb94cefdba49717a3fdc93824dc8349f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Thu, 7 Mar 2024 16:53:51 +0100 Subject: [PATCH 253/379] Up --- jobs/prioranalysis.jl | 454 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 454 insertions(+) create mode 100644 jobs/prioranalysis.jl diff --git a/jobs/prioranalysis.jl b/jobs/prioranalysis.jl new file mode 100644 index 000000000..a64abf54d --- /dev/null +++ b/jobs/prioranalysis.jl @@ -0,0 +1,454 @@ +# Little LSP hack to get function signatures, go #src +# to definition etc. #src +if isdefined(@__MODULE__, :LanguageServer) #src + include("../src/IncompressibleNavierStokes.jl") #src + using .IncompressibleNavierStokes #src +end #src + +using CairoMakie +using IncompressibleNavierStokes +using IncompressibleNavierStokes: + momentum, + momentum!, + divergence!, + project, + project!, + apply_bc_u!, + spectral_stuff, + kinetic_energy, + interpolate_u_p +using LinearAlgebra +using Printf +using FFTW + +function observe_v(dnsobs, Φ, les, compression, psolver) + (; grid) = les + (; dimension, N, Iu, Ip) = grid + D = dimension() + Mα = N[1] - 2 + v = zero.(Φ(dnsobs[].u, les, compression)) + Pv = zero.(v) + p = zero(v[1]) + div = zero(p) + ΦPF = zero.(v) + PFΦ = zero.(v) + c = zero.(v) + T = eltype(v[1]) + results = (; + Φ, + Mα, + t = zeros(T, 0), + Dv = zeros(T, 0), + Pv = zeros(T, 0), + Pc = zeros(T, 0), + c = zeros(T, 0), + ) + on(dnsobs) do (; u, PF, t) + push!(results.t, t) + + Φ(v, u, les, compression) + apply_bc_u!(v, t, les) + Φ(ΦPF, PF, les, compression) + momentum!(PFΦ, v, t, les) + apply_bc_u!(PFΦ, t, les; dudt = true) + project!(PFΦ, les; psolver, div, p) + foreach(α -> c[α] .= ΦPF[α] .- PFΦ[α], 1:D) + apply_bc_u!(c, t, les) + divergence!(div, v, les) + norm_Du = norm(div[Ip]) + norm_v = sqrt(sum(α -> sum(abs2, v[α][Iu[α]]), 1:D)) + push!(results.Dv, norm_Du / norm_v) + + copyto!.(Pv, v) + project!(Pv, les; psolver, div, p) + foreach(α -> Pv[α] .= Pv[α] .- v[α], 1:D) + norm_vmPv = sqrt(sum(α -> sum(abs2, Pv[α][Iu[α]]), 1:D)) + push!(results.Pv, norm_vmPv / norm_v) + + Pc = Pv + copyto!.(Pc, c) + project!(Pc, les; psolver, div, p) + foreach(α -> Pc[α] .= Pc[α] .- c[α], 1:D) + norm_cmPc = sqrt(sum(α -> sum(abs2, Pc[α][Iu[α]]), 1:D)) + norm_c = sqrt(sum(α -> sum(abs2, c[α][Iu[α]]), 1:D)) + push!(results.Pc, norm_cmPc / norm_c) + + norm_ΦPF = sqrt(sum(α -> sum(abs2, ΦPF[α][Iu[α]]), 1:D)) + push!(results.c, norm_c / norm_ΦPF) + end + results +end + +observe_u(dns, psolver_dns, filters; nupdate = 1) = + processor() do state + PF = zero.(state[].u) + div = zero(state[].u[1]) + p = zero(state[].u[1]) + dnsobs = Observable((; state[].u, PF, state[].t)) + results = [ + observe_v(dnsobs, Φ, setup, compression, psolver) for + (; setup, Φ, compression, psolver) in filters + ] + on(state) do (; u, t, n) + n % nupdate == 0 || return + apply_bc_u!(u, t, dns) + momentum!(PF, u, t, dns) + apply_bc_u!(PF, t, dns; dudt = true) + project!(PF, dns; psolver = psolver_dns, div, p) + dnsobs[] = (; u, PF, t) + end + # state[] = state[] # Save initial conditions + results + end + +# Output directory +output = "output/prioranalysis" +ispath(output) || mkpath(output) + +# Array type +ArrayType = Array +# using CUDA; ArrayType = CuArray; +## using AMDGPU; ArrayType = ROCArray +## using oneAPI; ArrayType = oneArray +## using Metal; ArrayType = MtlArray + +using CUDA; +ArrayType = CuArray; +CUDA.allowscalar(false); + +# 3D +# T = Float32 +T = Float64 +Re = T(2_000) +ndns = 512 +D = 3 +kp = 5 +Δt = T(1e-4) +filterdefs = [ + (FaceAverage(), 32), + (FaceAverage(), 64), + (FaceAverage(), 128), + (VolumeAverage(), 32), + (VolumeAverage(), 64), + (VolumeAverage(), 128), +] + +# # 2D +# T = Float64; +# Re = T(10_000) +# ndns = 4096 +# D = 2 +# kp = 20 +# Δt = T(5e-5) +# filterdefs = [ +# (FaceAverage(), 64), +# (FaceAverage(), 128), +# (FaceAverage(), 256), +# (VolumeAverage(), 64), +# (VolumeAverage(), 128), +# (VolumeAverage(), 256), +# ] + +# Setup +lims = T(0), T(1) +dns = Setup(ntuple(α -> LinRange(lims..., ndns + 1), D)...; Re, ArrayType); +filters = map(filterdefs) do (Φ, nles) + compression = ndns ÷ nles + setup = Setup(ntuple(α -> LinRange(lims..., nles + 1), D)...; Re, ArrayType) + psolver = SpectralPressureSolver(setup) + (; setup, Φ, compression, psolver) +end; +psolver_dns = SpectralPressureSolver(dns); + +# Create random initial conditions +u₀ = random_field(dns, T(0); kp, psolver = psolver_dns); + +state = (; u = u₀, t = T(0)); + +GC.gc() +CUDA.reclaim() + +# Solve unsteady problem +@time state, outputs = solve_unsteady( + dns, + u₀, + (T(0), T(1e-1)); + Δt, + docopy = true, + psolver = psolver_dns, + processors = ( + obs = observe_u(dns, psolver_dns, filters; nupdate = 20), + log = timelogger(; nupdate = 20), + ), +); +GC.gc() +CUDA.reclaim() + +# Save final solution +jldsave("$output/finalsolution.jld", u = Array.(state.u)) + +# fil = filters[2]; +# apply_bc_u!(state.u, T(0), dns); +# v = fil.Φ(state.u, fil.setup, fil.compression); +# apply_bc_u!(v, T(0), fil.setup); +# Fv = momentum(v, T(0), fil.setup); +# apply_bc_u!(Fv, T(0), fil.setup); +# PFv = project(Fv, fil.setup; psolver = fil.psolver); +# apply_bc_u!(PFv, T(0), fil.setup); +# F = momentum(state.u, T(0), dns); +# apply_bc_u!(F, T(0), dns); +# PF = project(F, dns; psolver = psolver_dns); +# apply_bc_u!(PF, T(0), dns); +# ΦPF = fil.Φ(PF, fil.setup, fil.compression); +# apply_bc_u!(ΦPF, T(0), fil.setup); +# c = ΦPF .- PFv +# apply_bc_u!(c, T(0), fil.setup) + +# with_theme(; fontsize = 25) do +# fig = fieldplot( +# (; u = u₀, t = T(0)); +# setup = dns, +# # type = image, +# # colormap = :viridis, +# docolorbar = false, +# size = (500, 500), +# title = "u₀", +# ) +# save("$output/ustart.png", fig) +# fig = fieldplot( +# state, +# setup = dns, +# # type = image, +# # colormap = :viridis, +# docolorbar = false, +# size = (500, 500), +# title = "u", +# ) +# save("$output/u.png", fig) +# fig = fieldplot( +# (; u = v, t = T(0)); +# fil.setup, +# # type = image, +# # colormap = :viridis, +# # fieldname = 1, +# docolorbar = false, +# size = (500, 500), +# # title = "ubar" +# title = "ū", +# ) +# save("$output/v.png", fig) +# fig = fieldplot( +# (; u = PF, t = T(0)); +# setup = dns, +# # type = image, +# # colormap = :viridis, +# # fieldname = 1, +# docolorbar = false, +# size = (500, 500), +# title = "PF(u)", +# ) +# save("$output/PFu.png", fig) +# fig = fieldplot( +# (; u = PFv, t = T(0)); +# fil.setup, +# # type = image, +# # colormap = :viridis, +# # fieldname = 1, +# docolorbar = false, +# size = (500, 500), +# # title = "PF(ubar)" +# title = "P̄F̄(ū)", +# ) +# save("$output/PFv.png", fig) +# fig = fieldplot( +# (; u = ΦPF, t = T(0)); +# fil.setup, +# # type = image, +# # colormap = :viridis, +# # fieldname = 1, +# docolorbar = false, +# size = (500, 500), +# title = "ΦPF(u)", +# ) +# save("$output/PhiPFu.png", fig) +# fig = fieldplot( +# (; u = c, t = T(0)); +# fil.setup, +# # type = image, +# # colormap = :viridis, +# # fieldname = 1, +# # fieldname = :velocity, +# docolorbar = false, +# size = (500, 500), +# # title = "c(u, ubar)" +# title = "c(u, ū)", +# ) +# save("$output/c.png", fig) +# end + +# 3D fieldplot ################################################################# + +# fieldplot( +# # (; u = u₀, t = T(0)); +# state; +# setup = dns, +# fieldname = :eig2field, +# # levels = LinRange(T(2), T(10), 10), +# levels = LinRange(T(4), T(12), 10), +# # levels = LinRange(-1.0f0, 3.0f0, 5), +# # levels = LinRange(-2.0f0, 2.0f0, 5), +# # levels = 5, +# docolorbar = false, +# # size = (800, 800), +# size = (600, 600), +# ) +# +# fname = "$output/lambda2/Re$(Int(Re))_start.png" +# fname = "$output/lambda2/Re$(Int(Re))_end.png" +# save(fname, current_figure()) +# run(`convert $fname -trim $fname`) # Requires imagemagick + +# i = 3 +# fieldplot( +# (; u = filters[i].Φ(state.u, filters[i].setup, filters[i].compression), t = T(0)); +# setup = filters[i].setup, +# fieldname = :eig2field, +# # levels = LinRange(T(2), T(10), 10), +# levels = LinRange(T(4), T(12), 10), +# # levels = LinRange(-1.0f0, 3.0f0, 5), +# # levels = LinRange(-2.0f0, 2.0f0, 5), +# # levels = 5, +# docolorbar = false, +# # size = (800, 800), +# size = (600, 600), +# ) +# +# fname = "$output/lambda2/Re$(Int(Re))_end_filtered.png" +# save(fname, current_figure()) +# run(`convert $fname -trim $fname`) # Requires imagemagick + +begin + println("Φ\t\tM\tDu\tPv\tPc\tc") + for o in outputs.obs + nt = length(o.t) + Dv = sum(o.Dv) / nt + Pc = sum(o.Pc) / nt + Pv = sum(o.Pv) / nt + c = sum(o.c) / nt + @printf( + "%s\t%d^%d\t%.2g\t%.2g\t%.2g\t%.2g\n", + # "%s &\t\$%d^%d\$ &\t\$%.2g\$ &\t\$%.2g\$ &\t\$%.2g\$ &\t\$%.2g\$\n", + typeof(o.Φ), + o.Mα, + D, + Dv, + Pv, + Pc, + c + ) + end +end; + +# # To free up memory +# psolver_dns = nothing +# fig = lines([1, 2, 3]) +# GC.gc() +# CUDA.reclaim() + +# Plot predicted spectra +fig = with_theme(; palette = (; color = ["#3366cc", "#cc0000", "#669900", "#ffcc00"])) do + fields = [state.u, u₀, (f.Φ(state.u, f.setup, f.compression) for f in filters)...] + setups = [dns, dns, (f.setup for f in filters)...] + specs = map(fields, setups) do u, setup + GC.gc() + CUDA.reclaim() + (; dimension, xp, Ip) = setup.grid + T = eltype(xp[1]) + D = dimension() + K = size(Ip) .÷ 2 + # up = interpolate_u_p(u, setup) + up = u + e = sum(up) do u + u = u[Ip] + uhat = fft(u)[ntuple(α -> 1:K[α], D)...] + # abs2.(uhat) + abs2.(uhat) ./ (2 * prod(size(u))^2) + # abs2.(uhat) ./ size(u, 1) + end + (; A, κ, K) = spectral_stuff(setup) + e = A * reshape(e, :) + # e = max.(e, eps(T)) # Avoid log(0) + ehat = Array(e) + (; κ, ehat) + end + kmax = maximum(specs[1].κ) + # Build inertial slope above energy + if D == 2 + # krange = [T(kmax)^(T(1) / 2), T(kmax)] + # krange = [T(50), T(400)] + krange = [T(16), T(128)] + elseif D == 3 + # krange = [T(kmax)^(T(1.5) / 3), T(kmax)] + # krange = [T(64), T(256)] + # krange = [T(8), T(64)] + krange = [T(16), T(100)] + end + slope, slopelabel = + D == 2 ? (-T(3), L"$\kappa^{-3}") : (-T(5 / 3), L"$\kappa^{-5/3}") + # slope, slopelabel = D == 2 ? (-T(3), "|k|⁻³") : (-T(5 / 3), "|k|⁻⁵³") + # slope, slopelabel = D == 2 ? (-T(3), "κ⁻³") : (-T(5 / 3), "κ⁻⁵³") + slopeconst = maximum(specs[1].ehat ./ specs[1].κ .^ slope) + offset = D == 2 ? 3 : 2 + inertia = offset .* slopeconst .* krange .^ slope + # Nice ticks + logmax = round(Int, log2(kmax + 1)) + xticks = T(2) .^ (0:logmax) + # Make plot + fig = Figure(; size = (500, 400)) + ax = Axis( + fig[1, 1]; + xticks, + # xlabel = "k", + xlabel = "κ", + # ylabel = "e(κ)", + xscale = log10, + yscale = log10, + limits = (1, kmax, T(1e-8), T(1)), + title = "Kinetic energy ($(D)D)", + ) + # plotparts(i) = 1:specs[i].kmax, specs[i].ehat + # plotparts(i) = 1:specs[i].kmax+1, [specs[i].ehat; eps(T)] + plotparts(i) = specs[i].κ, specs[i].ehat + # lines!(ax, 1:specs[1].kmax, specs[1].ehat; color = Cycled(1), label = "DNS") + # lines!(ax, 1:specs[2].kmax, specs[2].ehat; color = Cycled(4), label = "DNS, t = 0") + lines!(ax, plotparts(1)...; color = Cycled(1), label = "DNS") + lines!(ax, plotparts(2)...; color = Cycled(4), label = "DNS, t = 0") + lines!(ax, plotparts(3)...; color = Cycled(2), label = "Filtered DNS (FA)") + lines!(ax, plotparts(4)...; color = Cycled(2)) + lines!(ax, plotparts(5)...; color = Cycled(2)) + lines!(ax, plotparts(6)...; color = Cycled(3), label = "Filtered DNS (VA)") + lines!(ax, plotparts(7)...; color = Cycled(3)) + lines!(ax, plotparts(8)...; color = Cycled(3)) + lines!(ax, krange, inertia; color = Cycled(1), label = slopelabel, linestyle = :dash) + D == 2 && axislegend(ax; position = :lb) + # D == 3 && axislegend(ax; position = :rt) + D == 3 && axislegend(ax; position = :lb) + autolimits!(ax) + if D == 2 + # limits!(ax, (T(0.68), T(520)), (T(1e-3), T(3e1))) + # limits!(ax, (ax.xaxis.attributes.limits[][1], T(1000)), (T(1e-15), ax.yaxis.attributes.limits[][2])) + limits!(ax, (T(0.8), T(800)), (T(1e-10), T(1))) + elseif D == 3 + # limits!(ax, ax.xaxis.attributes.limits[], (T(1e-1), T(3e1))) + # limits!(ax, ax.xaxis.attributes.limits[], (T(1e-3), ax.yaxis.attributes.limits[][2])) + # limits!(ax, ax.xaxis.attributes.limits[], (T(3e-3), T(2e0))) + # limits!(ax, (T(8e-1), T(400)), (T(2e-3), T(1.5e0))) + limits!(ax, (T(8e-1), T(200)), (T(4e-5), T(1.5e0))) + end + fig +end +GC.gc() +CUDA.reclaim() + +# save("$output/spectra_$(D)D_linear_Re$(Int(Re)).pdf", fig) +save("$output/spectra_$(D)D_dyadic_Re$(Int(Re)).pdf", fig) From 091dc08575f59f156d58d59db74d6a4b0f0e17fb Mon Sep 17 00:00:00 2001 From: Syver Agdestein Date: Mon, 18 Mar 2024 12:30:23 +0100 Subject: [PATCH 254/379] Up --- jobs/prioranalysis.jl | 31 +++++++++++++++++++++---------- 1 file changed, 21 insertions(+), 10 deletions(-) diff --git a/jobs/prioranalysis.jl b/jobs/prioranalysis.jl index a64abf54d..ec5ddb28f 100644 --- a/jobs/prioranalysis.jl +++ b/jobs/prioranalysis.jl @@ -5,6 +5,8 @@ if isdefined(@__MODULE__, :LanguageServer) #src using .IncompressibleNavierStokes #src end #src +@info "Loading packages" + using CairoMakie using IncompressibleNavierStokes using IncompressibleNavierStokes: @@ -17,6 +19,7 @@ using IncompressibleNavierStokes: spectral_stuff, kinetic_energy, interpolate_u_p +using JLD2 using LinearAlgebra using Printf using FFTW @@ -117,10 +120,11 @@ ArrayType = CuArray; CUDA.allowscalar(false); # 3D -# T = Float32 -T = Float64 +T = Float32 +# T = Float64 Re = T(2_000) -ndns = 512 +# ndns = 512 +ndns = 1024 D = 3 kp = 5 Δt = T(1e-4) @@ -150,6 +154,7 @@ filterdefs = [ # ] # Setup +@info "Building setup" lims = T(0), T(1) dns = Setup(ntuple(α -> LinRange(lims..., ndns + 1), D)...; Re, ArrayType); filters = map(filterdefs) do (Φ, nles) @@ -161,14 +166,14 @@ end; psolver_dns = SpectralPressureSolver(dns); # Create random initial conditions +@info "Generating initial conditions" u₀ = random_field(dns, T(0); kp, psolver = psolver_dns); - -state = (; u = u₀, t = T(0)); - GC.gc() CUDA.reclaim() + # Solve unsteady problem +@info "Launching time stepping" @time state, outputs = solve_unsteady( dns, u₀, @@ -185,8 +190,12 @@ GC.gc() CUDA.reclaim() # Save final solution +@info "Saving final solution" jldsave("$output/finalsolution.jld", u = Array.(state.u)) +# u = load("$output/finalsolution.jld")["u"] +# state = (; u = ArrayType.(u), t = T(1e-1)) + # fil = filters[2]; # apply_bc_u!(state.u, T(0), dns); # v = fil.Φ(state.u, fil.setup, fil.compression); @@ -327,6 +336,7 @@ jldsave("$output/finalsolution.jld", u = Array.(state.u)) # save(fname, current_figure()) # run(`convert $fname -trim $fname`) # Requires imagemagick +@info "Computing statistics" begin println("Φ\t\tM\tDu\tPv\tPc\tc") for o in outputs.obs @@ -349,13 +359,14 @@ begin end end; -# # To free up memory -# psolver_dns = nothing +# To free up memory +psolver_dns = nothing # fig = lines([1, 2, 3]) -# GC.gc() -# CUDA.reclaim() +GC.gc() +CUDA.reclaim() # Plot predicted spectra +@info "Computing and plotting spectra" fig = with_theme(; palette = (; color = ["#3366cc", "#cc0000", "#669900", "#ffcc00"])) do fields = [state.u, u₀, (f.Φ(state.u, f.setup, f.compression) for f in filters)...] setups = [dns, dns, (f.setup for f in filters)...] From 39d104f0e2b0899f91f14d28075f9ef56bae2162 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Mon, 18 Mar 2024 13:30:24 +0100 Subject: [PATCH 255/379] Add energy --- jobs/prioranalysis.jl | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/jobs/prioranalysis.jl b/jobs/prioranalysis.jl index ec5ddb28f..111c3028b 100644 --- a/jobs/prioranalysis.jl +++ b/jobs/prioranalysis.jl @@ -45,6 +45,7 @@ function observe_v(dnsobs, Φ, les, compression, psolver) Pv = zeros(T, 0), Pc = zeros(T, 0), c = zeros(T, 0), + E = zeros(T, 0), ) on(dnsobs) do (; u, PF, t) push!(results.t, t) @@ -78,6 +79,11 @@ function observe_v(dnsobs, Φ, les, compression, psolver) norm_ΦPF = sqrt(sum(α -> sum(abs2, ΦPF[α][Iu[α]]), 1:D)) push!(results.c, norm_c / norm_ΦPF) + + Eu = sum(α -> sum(abs2, view(u[α], ntuple(b -> 2:size(u[α], b)-1, 1:D)), 1:D) + Ev = norm_v^2 + E = compression * Ev / Eu + push!(results.E, E) end results end @@ -105,7 +111,7 @@ observe_u(dns, psolver_dns, filters; nupdate = 1) = end # Output directory -output = "output/prioranalysis" +output = "output/prioranalysis/dimension$D" ispath(output) || mkpath(output) # Array type @@ -338,23 +344,25 @@ jldsave("$output/finalsolution.jld", u = Array.(state.u)) @info "Computing statistics" begin - println("Φ\t\tM\tDu\tPv\tPc\tc") + println("Φ\t\tM\tDu\tPv\tPc\tc\tE") for o in outputs.obs nt = length(o.t) Dv = sum(o.Dv) / nt Pc = sum(o.Pc) / nt Pv = sum(o.Pv) / nt c = sum(o.c) / nt + E = sum(o.E) / nt @printf( - "%s\t%d^%d\t%.2g\t%.2g\t%.2g\t%.2g\n", - # "%s &\t\$%d^%d\$ &\t\$%.2g\$ &\t\$%.2g\$ &\t\$%.2g\$ &\t\$%.2g\$\n", + "%s\t%d^%d\t%.2g\t%.2g\t%.2g\t%.2g\t%.2g\n", + # "%s &\t\$%d^%d\$ &\t\$%.2g\$ &\t\$%.2g\$ &\t\$%.2g\$ &\t\$%.2g\$ &\t\$%.2g\$\n", typeof(o.Φ), o.Mα, D, Dv, Pv, Pc, - c + c, + E, ) end end; From 21d43f9ca04daba0ceeb5ac4ce8e92a1ce8d124b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Mon, 18 Mar 2024 14:35:54 +0100 Subject: [PATCH 256/379] Fix energy --- jobs/prioranalysis.jl | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/jobs/prioranalysis.jl b/jobs/prioranalysis.jl index 111c3028b..a02a86d64 100644 --- a/jobs/prioranalysis.jl +++ b/jobs/prioranalysis.jl @@ -80,9 +80,9 @@ function observe_v(dnsobs, Φ, les, compression, psolver) norm_ΦPF = sqrt(sum(α -> sum(abs2, ΦPF[α][Iu[α]]), 1:D)) push!(results.c, norm_c / norm_ΦPF) - Eu = sum(α -> sum(abs2, view(u[α], ntuple(b -> 2:size(u[α], b)-1, 1:D)), 1:D) + Eu = sum(α -> sum(abs2, view(u[α], ntuple(b -> 2:size(u[α], b)-1, D)...)), 1:D) Ev = norm_v^2 - E = compression * Ev / Eu + E = compression^D * Ev / Eu push!(results.E, E) end results @@ -110,10 +110,6 @@ observe_u(dns, psolver_dns, filters; nupdate = 1) = results end -# Output directory -output = "output/prioranalysis/dimension$D" -ispath(output) || mkpath(output) - # Array type ArrayType = Array # using CUDA; ArrayType = CuArray; @@ -129,8 +125,8 @@ CUDA.allowscalar(false); T = Float32 # T = Float64 Re = T(2_000) -# ndns = 512 -ndns = 1024 +ndns = 512 +# ndns = 1024 D = 3 kp = 5 Δt = T(1e-4) @@ -143,7 +139,7 @@ filterdefs = [ (VolumeAverage(), 128), ] -# # 2D +# # 2D # T = Float64; # Re = T(10_000) # ndns = 4096 @@ -159,6 +155,10 @@ filterdefs = [ # (VolumeAverage(), 256), # ] +# Output directory +output = "output/prioranalysis/dimension$D" +ispath(output) || mkpath(output) + # Setup @info "Building setup" lims = T(0), T(1) @@ -177,7 +177,6 @@ u₀ = random_field(dns, T(0); kp, psolver = psolver_dns); GC.gc() CUDA.reclaim() - # Solve unsteady problem @info "Launching time stepping" @time state, outputs = solve_unsteady( From aecb1d2c17592d62ca8d8d6d36b4ee7c5c5afea2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Wed, 20 Mar 2024 11:40:57 +0100 Subject: [PATCH 257/379] Update paths --- scratch/postanalysis.jl | 40 +++++++++++++++++++++------------------- 1 file changed, 21 insertions(+), 19 deletions(-) diff --git a/scratch/postanalysis.jl b/scratch/postanalysis.jl index a4637f26a..69b260762 100644 --- a/scratch/postanalysis.jl +++ b/scratch/postanalysis.jl @@ -41,7 +41,9 @@ GLMakie.activate!() set_theme!(; GLMakie = (; scalefactor = 1.5)) -output = "../SupervisedClosure/figures/" +plotdir = "../SupervisedClosure/figures/" +outdir = "output/postanalysis" +ispath(outdir) || mkpath(outdir) # Random number generator rng = Random.default_rng() @@ -101,14 +103,14 @@ data_valid = [create_les_data(; params_valid...) for _ = 1:1]; data_test = create_les_data(; params_test...); # Save filtered DNS data -jldsave("output/divfree/data_train.jld2"; data_train) -jldsave("output/divfree/data_valid.jld2"; data_valid) -jldsave("output/divfree/data_test.jld2"; data_test) +jldsave("$outdir/data_train.jld2"; data_train) +jldsave("$outdir/data_valid.jld2"; data_valid) +jldsave("$outdir/data_test.jld2"; data_test) # Load filtered DNS data -data_train = load("output/divfree/data_train.jld2", "data_train"); -data_valid = load("output/divfree/data_valid.jld2", "data_valid"); -data_test = load("output/divfree/data_test.jld2", "data_test"); +data_train = load("$outdir/data_train.jld2", "data_train"); +data_valid = load("$outdir/data_valid.jld2", "data_valid"); +data_test = load("$outdir/data_test.jld2", "data_test"); data_train[5].comptime data_valid[1].comptime @@ -146,8 +148,8 @@ io_train = create_io_arrays(data_train, setups_train); io_valid = create_io_arrays(data_valid, setups_valid); io_test = create_io_arrays([data_test], setups_test); -# jldsave("output/divfree/io_train.jld2"; io_train) -# jldsave("output/divfree/io_train.jld2"; io_valid) +# jldsave("$outdir/io_train.jld2"; io_train) +# jldsave("$outdir/io_train.jld2"; io_valid) io_train[1].u |> extrema io_train[1].c |> extrema @@ -242,7 +244,7 @@ fig = with_theme(; palette = (; color = ["#3366cc", "#cc0000", "#669900", "#ffcc fig end -save("$output/training_data.pdf", fig) +save("$plotdir/training_data.pdf", fig) # Architecture 1 mname = "balzac" @@ -267,7 +269,7 @@ closure, θ₀ = cnn(; rng, ); closure.chain -savepath = "output/divfree/$mname" +savepath = "$outdir/$mname" ispath(savepath) || mkpath(savepath) closure(device(io_train[1, 1].u[:, :, :, 1:50]), device(θ₀)); @@ -433,10 +435,10 @@ clean() smag # Save trained parameters -jldsave("output/divfree/smag.jld2"; smag); +jldsave("$outdir/smag.jld2"; smag); # Load trained parameters -smag = load("output/divfree/smag.jld2")["smag"]; +smag = load("$outdir/smag.jld2")["smag"]; # Extract coefficients θ_smag = map(s -> s.θ, smag) @@ -636,7 +638,7 @@ fig = with_theme(; axislegend(; position = :lb) ylims!(ax, (T(-0.05), T(1.05))) # iorder == 2 && limits!(ax, (T(60), T(1050)), (T(2e-2), T(1e1))) - save("$output/convergence/$(mname)_prior_ifilter$ifil.pdf", fig) + save("$plotdir/convergence/$(mname)_prior_ifilter$ifil.pdf", fig) fig end @@ -736,7 +738,7 @@ with_theme(; fig end -name = "$output/convergence" +name = "$plotdir/convergence" ispath(name) || mkpath(name) save("$name/$(mname)_gen.pdf", current_figure()) save("$name/$(mname)_dcf.pdf", current_figure()) @@ -900,7 +902,7 @@ with_theme(; palette = (; color = ["#3366cc", "#cc0000", "#669900", "#ffcc00"])) iorder == 2 && axislegend(; position = :lb) # axislegend(; position = :lb) # axislegend() - name = "$output/energy_evolution/$mname/" + name = "$plotdir/energy_evolution/$mname/" ispath(name) || mkpath(name) save("$(name)/iorder$(iorder)_ifilter$(ifil)_igrid$(igrid).pdf", fig) end @@ -1105,7 +1107,7 @@ with_theme(; iorder == 2 && ifil == 1 && axislegend(; position = :rt) # axislegend() islog && ylims!(ax, (T(1e-6), T(1e3))) - name = "$output/divergence/$mname/$(islog ? "log" : "lin")" + name = "$plotdir/divergence/$mname/$(islog ? "log" : "lin")" ispath(name) || mkpath(name) # save("$(name)/iorder$(iorder)_igrid$(igrid).pdf", fig) save("$(name)/iorder$(iorder)_ifilter$(ifil)_igrid$(igrid).pdf", fig) @@ -1283,7 +1285,7 @@ fig = with_theme(; palette = (; color = ["#3366cc", "#cc0000", "#669900", "#ffcc axislegend(ax; position = :cb) autolimits!(ax) # limits!(ax, (T(0.8), T(800)), (T(1e-10), T(1))) - name = "$output/energy_spectra/$mname" + name = "$plotdir/energy_spectra/$mname" ispath(name) || mkpath(name) save("$(name)/iorder$(iorder)_ifilter$(ifil)_igrid$(igrid).pdf", fig) end @@ -1310,7 +1312,7 @@ with_theme(; Point2f(x1, y1), # Point2f(x2, y1), ] - path = "$output/les_fields/$mname" + path = "$plotdir/les_fields/$mname" ispath(path) || mkpath(path) for iorder = 1:2, ifil = 1:2, igrid = 1:3 setup = setups_test[igrid] From 5412a8e6e2fdcf7e17e8db08c9ab18d1ef866610 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Thu, 21 Mar 2024 13:30:55 +0100 Subject: [PATCH 258/379] Up --- scratch/postanalysis.jl | 24 +++++++++++++----------- src/closures/create_les_data.jl | 2 +- 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/scratch/postanalysis.jl b/scratch/postanalysis.jl index 69b260762..247f108a4 100644 --- a/scratch/postanalysis.jl +++ b/scratch/postanalysis.jl @@ -93,9 +93,9 @@ get_params(nlesscalar) = (; ), ) -params_train = (; get_params([64, 128, 256])..., savefreq = 10); -params_valid = (; get_params([64, 128, 256])..., tsim = T(0.1), savefreq = 40); -params_test = (; get_params([64, 128, 256, 512, 1024])..., tsim = T(0.1), savefreq = 10); +params_train = (; get_params([64, 128, 256, 512])..., tsim = T(0.5), savefreq = 10); +params_valid = (; get_params([64, 128, 256, 512])..., tsim = T(0.1), savefreq = 40); +params_test = (; get_params([64, 128, 256, 512])..., tsim = T(0.1), savefreq = 10); # Create LES data from DNS data_train = [create_les_data(; params_train...) for _ = 1:5]; @@ -149,12 +149,15 @@ io_valid = create_io_arrays(data_valid, setups_valid); io_test = create_io_arrays([data_test], setups_test); # jldsave("$outdir/io_train.jld2"; io_train) -# jldsave("$outdir/io_train.jld2"; io_valid) +# jldsave("$outdir/io_valid.jld2"; io_valid) +# jldsave("$outdir/io_test.jld2"; io_test) io_train[1].u |> extrema io_train[1].c |> extrema io_valid[1].u |> extrema io_valid[1].c |> extrema +io_test[1].u |> extrema +io_test[1].c |> extrema # Inspect data let @@ -274,11 +277,11 @@ ispath(savepath) || mkpath(savepath) closure(device(io_train[1, 1].u[:, :, :, 1:50]), device(θ₀)); -# A-priori training -prior = map(CartesianIndices(size(io_train))) do I - # Prepare training +# A-priori training ########################################################### + +ispath("$savepath/priortraining") || mkpath("$savepath/priortraining") +for ifil = 1:1, ig = 4:4 starttime = time() - ig, ifil = I.I println("ig = $ig, ifil = $ifil") d = create_dataloader_prior(io_train[ig, ifil]; batchsize = 50, device) θ = T(1.0e0) * device(θ₀) @@ -304,15 +307,14 @@ prior = map(CartesianIndices(size(io_train))) do I ) θ = callbackstate.θmin # Use best θ instead of last θ prior = (; θ = Array(θ), comptime = time() - starttime, callbackstate.hist) - jldsave("$savepath/prior_ifilter$(ifil)_igrid$(ig).jld2"; prior) - prior + jldsave("$savepath/priortraining/ifilter$(ifil)_igrid$(ig).jld2"; prior) end clean() # Load trained parameters prior = map(CartesianIndices(size(io_train))) do I ig, ifil = I.I - name = "$savepath/prior_ifilter$(ifil)_igrid$(ig).jld2" + name = "$savepath/priortraining/ifilter$(ifil)_igrid$(ig).jld2" load(name)["prior"] end; θ_cnn_prior = [copyto!(device(θ₀), p.θ) for p in prior]; diff --git a/src/closures/create_les_data.jl b/src/closures/create_les_data.jl index e1c5be373..93e429d2b 100644 --- a/src/closures/create_les_data.jl +++ b/src/closures/create_les_data.jl @@ -157,7 +157,7 @@ function create_les_data(; D * 2 * length(bitstring(zero(T))) / 8 / 1e6 - @info "Generating $datasize Mb of LES data" + @info "Generating $datasize Mb of filtered DNS data" # Initial conditions u₀ = icfunc(dns, psolver) From ee68ce7e2fbbd8462c4e6be5f45c2a40e3cef182 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Mon, 25 Mar 2024 09:24:20 +0100 Subject: [PATCH 259/379] Update script --- scratch/postanalysis.jl | 74 ++++++++++++++++++++++++----------------- 1 file changed, 43 insertions(+), 31 deletions(-) diff --git a/scratch/postanalysis.jl b/scratch/postanalysis.jl index 247f108a4..9ca69c30f 100644 --- a/scratch/postanalysis.jl +++ b/scratch/postanalysis.jl @@ -26,6 +26,9 @@ using SparseArrays using KernelAbstractions using FFTW +# palette = (; color = ["#3366cc", "#cc0000", "#669900", "#ffcc00"]) +palette = (; color = ["#3366cc", "#cc0000", "#669900", "#ff9900"]) + getorder(i) = if i == 1 :first @@ -93,9 +96,13 @@ get_params(nlesscalar) = (; ), ) -params_train = (; get_params([64, 128, 256, 512])..., tsim = T(0.5), savefreq = 10); -params_valid = (; get_params([64, 128, 256, 512])..., tsim = T(0.1), savefreq = 40); -params_test = (; get_params([64, 128, 256, 512])..., tsim = T(0.1), savefreq = 10); +params_train = (; get_params([64, 128, 256])..., tsim = T(0.5), savefreq = 10); +params_valid = (; get_params([64, 128, 256])..., tsim = T(0.1), savefreq = 40); +params_test = (; get_params([64, 128, 256, 512, 1024])..., tsim = T(0.1), savefreq = 10); + +# params_train = (; get_params([64, 128, 256, 512])..., tsim = T(0.5), savefreq = 10); +# params_valid = (; get_params([64, 128, 256, 512])..., tsim = T(0.1), savefreq = 40); +# params_test = (; get_params([64, 128, 256, 512])..., tsim = T(0.1), savefreq = 10); # Create LES data from DNS data_train = [create_les_data(; params_train...) for _ = 1:5]; @@ -545,10 +552,10 @@ fig = with_theme(; # markersize = 10, # markersize = 20, # fontsize = 20, - palette = (; color = ["#3366cc", "#cc0000", "#669900", "#ffcc00"]), + palette, ) do nles = [n[1] for n in params_test.nles][1:3] - ifil = 2 + ifil = 1 fig = Figure(; size = (500, 400)) ax = Axis( fig[1, 1]; @@ -604,7 +611,7 @@ fig = with_theme(; # end # for ifil = 1:2 # linestyle = ifil == 1 ? :solid : :dash - label = "CNN (Lpost, Gen)" + label = "CNN (Lpost, DIF)" # label = label * (ifil == 1 ? " (FA)" : " (VA)") # ifil == 2 && (label = nothing) scatterlines!( @@ -640,24 +647,28 @@ fig = with_theme(; axislegend(; position = :lb) ylims!(ax, (T(-0.05), T(1.05))) # iorder == 2 && limits!(ax, (T(60), T(1050)), (T(2e-2), T(1e1))) - save("$plotdir/convergence/$(mname)_prior_ifilter$ifil.pdf", fig) + name = "$plotdir/convergence" + ispath(name) || mkpath(name) + save("$name/$(mname)_prior_ifilter$ifil.pdf", fig) fig end -# Plot convergence ############################################################ +# Plot a-posteriori errors ################################################### with_theme(; # linewidth = 5, # markersize = 10, # markersize = 20, # fontsize = 20, - palette = (; color = ["#3366cc", "#cc0000", "#669900", "#ffcc00"]), + palette, ) do - iorder = 1 + iorder = 2 # lesmodel = iorder == 1 ? "project-then-closure" : "closure-then-project" # lesmodel = iorder == 1 ? "project-then-closure" : "closure-then-project" - lesmodel = iorder == 1 ? "Gen" : "DCF" - nles = [n[1] for n in params_test.nles] + lesmodel = iorder == 1 ? "DIF" : "DCF" + ntrain = size(data_train[1].data, 1) + # nles = [n[1] for n in params_test.nles] + nles = [n[1] for n in params_test.nles][1:ntrain] fig = Figure(; size = (500, 400)) ax = Axis( fig[1, 1]; @@ -677,7 +688,8 @@ with_theme(; ifil == 2 && (label = nothing) scatterlines!( nles, - e_nm[:, ifil]; + # e_nm[:, ifil]; + e_nm[1:ntrain, ifil]; color = Cycled(1), linestyle, marker = :circle, @@ -691,7 +703,8 @@ with_theme(; ifil == 2 && (label = nothing) scatterlines!( nles, - e_smag[:, ifil, iorder]; + # e_smag[:, ifil, iorder]; + e_smag[1:ntrain, ifil, iorder]; color = Cycled(2), linestyle, marker = :utriangle, @@ -699,7 +712,6 @@ with_theme(; ) end for ifil = 1:2 - ntrain = size(data_train[1].data, 1) linestyle = ifil == 1 ? :solid : :dash label = "CNN (prior)" # label = label * (ifil == 1 ? " (FA)" : " (VA)") @@ -714,7 +726,6 @@ with_theme(; ) end for ifil = 1:2 - ntrain = size(data_train[1].data, 1) linestyle = ifil == 1 ? :solid : :dash label = "CNN (post)" # label = label * (ifil == 1 ? " (FA)" : " (VA)") @@ -735,16 +746,16 @@ with_theme(; # label = "n⁻²", # color = Cycled(1), # ) - axislegend(; position = :rt) + # axislegend(; position = :rt) + axislegend(; position = :lb) + ylims!(ax, (T(0.025), T(1.00))) # iorder == 2 && limits!(ax, (T(60), T(1050)), (T(2e-2), T(1e1))) + name = "$plotdir/convergence" + ispath(name) || mkpath(name) + save("$name/$(mname)_iorder$iorder.pdf", fig) fig end -name = "$plotdir/convergence" -ispath(name) || mkpath(name) -save("$name/$(mname)_gen.pdf", current_figure()) -save("$name/$(mname)_dcf.pdf", current_figure()) - # Energy evolution ########################################################### kineticenergy = let @@ -844,7 +855,7 @@ CairoMakie.activate!() # Plot energy evolution ######################################################## -with_theme(; palette = (; color = ["#3366cc", "#cc0000", "#669900", "#ffcc00"])) do +with_theme(; palette) do # t = data_test.t[2:end] t = data_test.t for iorder = 1:2, ifil = 1:2, igrid = 1:3 @@ -852,7 +863,7 @@ with_theme(; palette = (; color = ["#3366cc", "#cc0000", "#669900", "#ffcc00"])) # reflevel = kineticenergy.ke_ref[igrid, ifil][2:end] reflevel = copy(kineticenergy.ke_ref[igrid, ifil]) reflevel = fill!(reflevel, 1) - lesmodel = iorder == 1 ? "Gen" : "DCF" + lesmodel = iorder == 1 ? "DIF" : "DCF" fil = ifil == 1 ? "FA" : "VA" nles = params_test.nles[igrid] fig = Figure(; size = (500, 400)) @@ -1038,8 +1049,8 @@ CairoMakie.activate!() # Plot Divergence ############################################################# with_theme(; - fontsize = 20, - palette = (; color = ["#3366cc", "#cc0000", "#669900", "#ffcc00"]), + # fontsize = 20, + palette, ) do t = data_test.t for islog in (true, false) @@ -1047,7 +1058,7 @@ with_theme(; # println("iorder = $iorder, igrid = $igrid") println("iorder = $iorder, ifil = $ifil, igrid = $igrid") lesmodel = if iorder == 1 - "Gen" + "DIF" elseif iorder == 2 "DCF" else @@ -1223,10 +1234,10 @@ markers_labels = [ # Plot spectra ############################################################### -fig = with_theme(; palette = (; color = ["#3366cc", "#cc0000", "#669900", "#ffcc00"])) do +fig = with_theme(; palette) do for iorder = 1:2, ifil = 1:2, igrid = 1:3 println("iorder = $iorder, ifil = $ifil, igrid = $igrid") - lesmodel = iorder == 1 ? "Gen" : "DCF" + lesmodel = iorder == 1 ? "DIF" : "DCF" fil = ifil == 1 ? "FA" : "VA" nles = params_test.nles[igrid] setup = setups_test[igrid] @@ -1287,6 +1298,7 @@ fig = with_theme(; palette = (; color = ["#3366cc", "#cc0000", "#669900", "#ffcc axislegend(ax; position = :cb) autolimits!(ax) # limits!(ax, (T(0.8), T(800)), (T(1e-10), T(1))) + ylims!(ax, (T(1e-3), T(0.35))) name = "$plotdir/energy_spectra/$mname" ispath(name) || mkpath(name) save("$(name)/iorder$(iorder)_ifilter$(ifil)_igrid$(igrid).pdf", fig) @@ -1300,7 +1312,7 @@ GLMakie.activate!() with_theme(; fontsize = 25, - palette = (; color = ["#3366cc", "#cc0000", "#669900", "#ffcc00"]), + palette, ) do x1 = 0.3 x2 = 0.5 @@ -1319,7 +1331,7 @@ with_theme(; for iorder = 1:2, ifil = 1:2, igrid = 1:3 setup = setups_test[igrid] name = "$path/iorder$(iorder)_ifilter$(ifil)_igrid$(igrid)" - lesmodel = iorder == 1 ? "Gen" : "DCF" + lesmodel = iorder == 1 ? "DIF" : "DCF" fil = ifil == 1 ? "FA" : "VA" nles = params_test.nles[igrid] function makeplot(u, title, suffix) From b279950ed9cae60ac40323fbea445dc81586e16b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Fri, 29 Mar 2024 16:33:40 +0100 Subject: [PATCH 260/379] Add script --- scratch/energy.jl | 143 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 143 insertions(+) create mode 100644 scratch/energy.jl diff --git a/scratch/energy.jl b/scratch/energy.jl new file mode 100644 index 000000000..ebfcb42b5 --- /dev/null +++ b/scratch/energy.jl @@ -0,0 +1,143 @@ +# Little LSP hack to get function signatures, go #src +# to definition etc. #src +if isdefined(@__MODULE__, :LanguageServer) #src + include("../src/IncompressibleNavierStokes.jl") #src + using .IncompressibleNavierStokes #src +end #src + +using GLMakie +using IncompressibleNavierStokes +using IncompressibleNavierStokes: apply_bc_u!, total_kinetic_energy, diffusion! + +# Output directory +output = "output/energy" + +# Array type +ArrayType = Array +# using CUDA; ArrayType = CuArray; +## using AMDGPU; ArrayType = ROCArray +## using oneAPI; ArrayType = oneArray +## using Metal; ArrayType = MtlArray + +clean() = GC.gc() + +using CUDA; +ArrayType = CuArray; +CUDA.allowscalar(false); +clean() = (GC.gc(); CUDA.reclaim()) + +set_theme!() +set_theme!(; GLMakie = (; scalefactor = 1.5)) + +# 2D +T = Float64; +Re = T(4_000) +# ndns = 1024 +# nles = 128 +ndns = 256 +nles = 32 +compression = ndns ÷ nles +D = 2 +kp = 20 +lims = T(0), T(1); + +dns, les = map((ndns, nles)) do n + setup = Setup(ntuple(α -> LinRange(lims..., n + 1), D)...; Re, ArrayType) + psolver = SpectralPressureSolver(setup) + (; setup, psolver) +end; + +# Create random initial conditions +u₀ = random_field(dns.setup, T(0); kp, dns.psolver); +clean() + +state = (; u = u₀, t = T(0)); + +observe_u(dns, les, compression; Δt, nupdate = 1) = + processor() do state + u = state[].u + v = zero.(FaceAverage()(u, les.setup, compression)) + # u = copy.(u₀) + diffu = zero.(u) + diffv = zero.(v) + results = (; + t = zeros(T, 0), + Ku = zeros(T, 0), + Kv = zeros(T, 0), + Kuref = zeros(T, 0), + Kvref = zeros(T, 0), + ) + on(state) do (; u, t, n) + n % nupdate == 0 || return + apply_bc_u!(u, t, dns.setup) + FaceAverage()(v, u, les.setup, compression) + apply_bc_u!(v, t, les.setup) + Ku = total_kinetic_energy(u, dns.setup) + Kv = total_kinetic_energy(v, les.setup) + if n == 0 + push!(results.Kuref, Ku) + push!(results.Kvref, Kv) + else + fill!.(diffu, 0) + fill!.(diffv, 0) + diffusion!(diffu, u, dns.setup) + diffusion!(diffv, v, les.setup) + apply_bc_u!(diffu, t, dns.setup) + apply_bc_u!(diffv, t, les.setup) + for α = 1:D + diffu[α] .*= u[α] + diffv[α] .*= v[α] + end + sum(sum.(diffu)) / ndns^D + push!( + results.Kuref, + results.Kuref[end] + nupdate * Δt * sum(sum.(diffu)) / ndns^D, + ) + push!( + results.Kvref, + results.Kvref[end] + nupdate * Δt * sum(sum.(diffv)) / nles^D, + ) + end + push!(results.t, t) + push!(results.Ku, Ku) + push!(results.Kv, Kv) + end + state[] = state[] # Save initial conditions + results + end + +Δt = 1e-4 + +# Solve unsteady problem +@time state, outputs = solve_unsteady( + dns.setup, + u₀, + # state.u, + (T(0), T(1e-1)); + Δt, + docopy = true, + dns.psolver, + processors = ( + # rtp = realtimeplotter(; dns.setup, nupdate = 5), + obs = observe_u(dns, les, compression; Δt, nupdate = 1), + log = timelogger(; nupdate = 1), + ), +); +clean() + +outputs.obs.t +outputs.obs.Ku +outputs.obs.Kuref +outputs.obs.Kv +outputs.obs.Kvref + +with_theme() do + fig = Figure() + ax = Axis(fig[1, 1]) + lines!(ax, outputs.obs.t, outputs.obs.Ku; label = "Ku") + lines!(ax, outputs.obs.t, outputs.obs.Kuref; label = "Kuref") + lines!(ax, outputs.obs.t, outputs.obs.Kv; label = "Kv") + lines!(ax, outputs.obs.t, outputs.obs.Kvref; label = "Kvref") + axislegend() + fig +end From 502d5669bf56f0169f250cc4cbfd03eba58d0c13 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Fri, 29 Mar 2024 16:37:44 +0100 Subject: [PATCH 261/379] Up --- scratch/energy.jl | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/scratch/energy.jl b/scratch/energy.jl index ebfcb42b5..57046bb82 100644 --- a/scratch/energy.jl +++ b/scratch/energy.jl @@ -11,6 +11,7 @@ using IncompressibleNavierStokes: apply_bc_u!, total_kinetic_energy, diffusion! # Output directory output = "output/energy" +mkdir(output) # Array type ArrayType = Array @@ -131,7 +132,9 @@ outputs.obs.Kuref outputs.obs.Kv outputs.obs.Kvref -with_theme() do +using CairoMakie + +fig = with_theme() do fig = Figure() ax = Axis(fig[1, 1]) lines!(ax, outputs.obs.t, outputs.obs.Ku; label = "Ku") @@ -141,3 +144,5 @@ with_theme() do axislegend() fig end + +save("$output/energy.pdf", fig) From c1de80b0c4905aab8f95a96d01dfdd9608f4254b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Wed, 3 Apr 2024 21:35:42 +0200 Subject: [PATCH 262/379] Add CUDA pressure solver --- Project.toml | 2 ++ src/IncompressibleNavierStokes.jl | 6 +++++- src/solvers/pressure/poisson.jl | 12 +++++++++++- src/solvers/pressure/solvers.jl | 32 +++++++++++++++++++++++++++++++ 4 files changed, 50 insertions(+), 2 deletions(-) diff --git a/Project.toml b/Project.toml index ac09f1035..a83465bb3 100644 --- a/Project.toml +++ b/Project.toml @@ -6,6 +6,7 @@ version = "0.4.1" [deps] Adapt = "79e6a3ab-5dfb-504d-930d-738a2a938a0e" CUDA = "052768ef-5323-5732-b1bb-66c8b64840ba" +CUDSS = "45b445bb-4962-46a0-9369-b4df9d0f772e" ChainRulesCore = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4" ComponentArrays = "b0b7db55-cfe3-40fc-9ded-d10e2dbeff66" FFTW = "7a1cc6ca-52ef-59f5-83cd-3a7055c09341" @@ -30,6 +31,7 @@ Zygote = "e88e6eb3-aa80-5325-afca-941959d7151f" Adapt = "3, 4" Aqua = "0.8" CUDA = "5" +CUDSS = "1" CairoMakie = "0.11" ComponentArrays = "0.15" FFTW = "1" diff --git a/src/IncompressibleNavierStokes.jl b/src/IncompressibleNavierStokes.jl index 17aa76f41..dc91b42cf 100644 --- a/src/IncompressibleNavierStokes.jl +++ b/src/IncompressibleNavierStokes.jl @@ -29,6 +29,7 @@ using Zygote using CUDA using CUDA.CUSPARSE # using CUSOLVERRF +using CUDSS # # Easily retrieve value from Val # (::Val{x})() where {x} = x @@ -114,7 +115,10 @@ export stretched_grid, cosine_grid # Pressure solvers export DirectPressureSolver, - CGPressureSolver, SpectralPressureSolver, LowMemorySpectralPressureSolver + CUDSSPressureSolver, + CGPressureSolver, + SpectralPressureSolver, + LowMemorySpectralPressureSolver # Solvers export solve_unsteady, solve_steady_state diff --git a/src/solvers/pressure/poisson.jl b/src/solvers/pressure/poisson.jl index ee06fc42a..5011456b5 100644 --- a/src/solvers/pressure/poisson.jl +++ b/src/solvers/pressure/poisson.jl @@ -59,6 +59,17 @@ function poisson!(solver::DirectPressureSolver, p, f) p end +function poisson!(solver::CUDSSPressureSolver, p, f) + (; setup) = solver + (; Ip) = setup.grid + T = eltype(p) + pp = view(view(p, Ip), :) + copyto!(view(solver.f, 1:length(solver.f)-1), view(view(f, Ip), :)) + cudss("solve", solver.solver, solver.p, solver.f) + copyto!(pp, view(solver.p, 1:length(solver.p)-1)) + p +end + function poisson!(solver::CGMatrixPressureSolver, p, f) (; L, qin, qout, abstol, reltol, maxiter) = solver copyto!(qin, view(view(f, Ip), :)) @@ -188,7 +199,6 @@ function poisson!(solver::LowMemorySpectralPressureSolver, p, f) ax = ahat ay = reshape(ahat, 1, :) @. phat = -phat / (ax + ay) - else ax = ahat ay = reshape(ahat, 1, :) diff --git a/src/solvers/pressure/solvers.jl b/src/solvers/pressure/solvers.jl index 45b94f2d9..756631167 100644 --- a/src/solvers/pressure/solvers.jl +++ b/src/solvers/pressure/solvers.jl @@ -76,6 +76,38 @@ Adapt.adapt_structure(to, s::DirectPressureSolver) = error( "`DirectPressureSolver` is not yet implemented for CUDA. Consider using `CGPressureSolver`.", ) +""" + CUDSSPressureSolver() + +Direct pressure solver using a CUDSS LDLt decomposition. +""" +struct CUDSSPressureSolver{T,S,F,A} <: AbstractPressureSolver{T} + setup::S + solver::F + f::A + p::A + function CUDSSPressureSolver(setup) + (; grid, ArrayType) = setup + (; x, Np) = grid + T = eltype(x[1]) + @assert x[1] isa CuArray "CUDSSPressureSolver only works for CuArray." + f = fill!(similar(x[1], prod(Np) + 1), 0) + p = fill!(similar(x[1], prod(Np) + 1), 0) + L = laplacian_mat(setup) + e = ones(T, size(L, 2)) + L = [L e; e' 0] + L = CuSparseMatrixCSR(L) + solver = CudssSolver( + L, + "S", # Symmetric (not positive definite) + 'L', # Lower triangular representation + ) + cudss("analysis", solver, p, f) + cudss("factorization", solver, p, f) # Compute factorization + new{T,typeof(setup),typeof(solver),typeof(f)}(setup, solver, f, p) + end +end + # """ # CGPressureSolver(setup; [abstol], [reltol], [maxiter]) # From d672d117e86100000fcb1a205345c546ae0bbbb5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Wed, 3 Apr 2024 21:36:14 +0200 Subject: [PATCH 263/379] Add low memory spectrum generator --- src/utils/spectral_stuff.jl | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/src/utils/spectral_stuff.jl b/src/utils/spectral_stuff.jl index ef4b1f81e..471c32d50 100644 --- a/src/utils/spectral_stuff.jl +++ b/src/utils/spectral_stuff.jl @@ -51,3 +51,38 @@ function spectral_stuff(setup; npoint = 100, a = typeof(setup.Re)(1 + sqrt(5)) / (; A, κ, K) end + +function get_spectrum(setup; npoint = 100, a = typeof(e.setup.Re)(1 + sqrt(5)) / 2) + (; dimension, xp, Ip) = setup.grid + T = eltype(xp[1]) + D = dimension() + + @assert all(==(size(Ip, 1)), size(Ip)) + + K = size(Ip, 1) .÷ 2 + kmax = K - 1 + k = ntuple( + i -> reshape(0:kmax, ntuple(Returns(1), i - 1)..., :, ntuple(Returns(1), D - i)...), + D, + ) + + # Output query points (evenly log-spaced, but only integer wavenumbers) + logκ = LinRange(T(0), log(T(kmax) / a), npoint) + κ = exp.(logκ) + κ = sort(unique(round.(Int, κ))) + npoint = length(κ) + + masks = map(κ) do κ + if D == 2 + @. (κ / a)^2 ≤ k[1]^2 + k[2]^2 < (κ * a)^2 + elseif D == 3 + @. (κ / a)^2 ≤ k[1]^2 + k[2]^2 + k[3]^2 < (κ * a)^2 + else + error("Not implemented") + end + end + + BoolArray = typeof(similar(xp[1], Bool, ntuple(Returns(0), D)...)) + masks = adapt.(BoolArray, masks) + (; κ, masks, K) +end From d1fc5facf3fa35638dee364c40344b4f2139d17d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Wed, 3 Apr 2024 21:37:17 +0200 Subject: [PATCH 264/379] Add test case --- examples/MultiActuator.jl | 208 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 208 insertions(+) create mode 100644 examples/MultiActuator.jl diff --git a/examples/MultiActuator.jl b/examples/MultiActuator.jl new file mode 100644 index 000000000..bf6bce877 --- /dev/null +++ b/examples/MultiActuator.jl @@ -0,0 +1,208 @@ +# Little LSP hack to get function signatures, go #src +# to definition etc. #src +if isdefined(@__MODULE__, :LanguageServer) #src + include("../src/IncompressibleNavierStokes.jl") #src + using .IncompressibleNavierStokes #src +end #src + +# # Unsteady actuator case - 2D +# +# In this example, an unsteady inlet velocity profile at encounters a wind +# turbine blade in a wall-less domain. The blade is modeled as a uniform body +# force on a thin rectangle. + +#md using CairoMakie +using GLMakie #!md +using IncompressibleNavierStokes +using Adapt +using Random + +# Output directory +output = "output/Actuator2D" + +# Floating point precision +T = Float64 + +# Array type +ArrayType = Array +## using CUDA; ArrayType = CuArray +## using AMDGPU; ArrayType = ROCArray +## using oneAPI; ArrayType = oneArray +## using Metal; ArrayType = MtlArray + +using CUDA; +T = Float32; +# T = Float64; +ArrayType = CuArray; +CUDA.allowscalar(false); +device = x -> adapt(ArrayType, x); + +set_theme!(; GLMakie = (; scalefactor = 1.5)) + +const ππ = T(π) + +# Boundary conditions +boundary_conditions = ( + ## x left, x right + ( + ## Unsteady BC requires time derivatives + DirichletBC( + (dim, x, y, t) -> sin(ππ / 6 * sin(ππ / 6 * t) + ππ / 2 * (dim() == 1)), + (dim, x, y, t) -> + (ππ / 6)^2 * + cos(ππ / 6 * t) * + cos(ππ / 6 * sin(ππ / 6 * t) + ππ / 2 * (dim() == 1)), + ), + PressureBC(), + ), + + ## y rear, y front + (PressureBC(), PressureBC()), +) + +# Actuator body force: A thrust coefficient `Cₜ` distributed over a thin rectangle +create_bodyforce(; xc, yc, D, δ, C) = + (dim, x, y, t) -> + dim() == 1 && abs(x - xc) ≤ δ / 2 && abs(y - yc) ≤ D / 2 ? -C / (D * δ) : zero(x) + +create_manyforce(forces...) = function (dim, x, y, t) + out = zero(x) + for f in forces + out += f(dim, x, y, t) + end + out +end + +bodyforce = create_bodyforce(; + xc = T(2), # Disk center + yc = T(0), # Disk center + D = T(1), # Disk diameter + δ = T(0.11), # Disk thickness + C = T(0.2), # Thrust coefficient +) + +disk = (; D = T(1), δ = T(0.11), C = T(0.2)) +bodyforce = create_manyforce( + create_bodyforce(; xc = T(2), yc = T(0), disk...), + create_bodyforce(; xc = T(4), yc = T(0.7), disk...), + create_bodyforce(; xc = T(6.4), yc = T(-1), disk...), +) + +# A 2D grid is a Cartesian product of two vectors +n = 160 +x = LinRange(T(0), T(10), 5n + 1) +y = LinRange(-T(2), T(2), 2n + 1) + +# Build setup and assemble operators +setup = Setup(x, y; Re = T(2000), boundary_conditions, bodyforce, ArrayType); + +using CUDA.CUSPARSE +using CUDSS +using LinearAlgebra + +psolver = CUDSSPressureSolver(setup); +psolver2 = DirectPressureSolver(setup); + +p = fill!(similar(setup.grid.x[1], setup.grid.N), 0) +f = similar(setup.grid.x[1], setup.grid.N) +f .= randn.(T) +IncompressibleNavierStokes.apply_bc_p!(f, T(0), setup) + +IncompressibleNavierStokes.poisson!(psolver, p, f) +IncompressibleNavierStokes.poisson!(psolver2, p, f) + +using KernelAbstractions +using BenchmarkTools + +@benchmark begin + # IncompressibleNavierStokes.poisson!(psolver, p, f) + IncompressibleNavierStokes.poisson!(psolver2, p, f) + KernelAbstractions.synchronize(get_backend(p)) +end + +# Initial conditions (extend inflow) +u₀ = create_initial_conditions(setup, (dim, x, y) -> dim() == 1 ? one(x) : zero(x); psolver); +u = u₀ +t = T(0) + +# Solve unsteady problem +state, outputs = solve_unsteady( + setup, + u₀, + (T(0), 4 * T(12)); + # (T(0), T(1)); + method = RK44P2(), + Δt = T(0.01), + psolver, + processors = ( + rtp = realtimeplotter(; + setup, + # plot = fieldplot, + # fieldname = :velocity, + # fieldname = :pressure, + psolver, + nupdate = 1, + ), + boxplotter = processor() do state + for box in boxes + lines!(current_axis(), box; color = :red); + end + end, + ## ehist = realtimeplotter(; setup, plot = energy_history_plot, nupdate = 1), + ## espec = realtimeplotter(; setup, plot = energy_spectrum_plot, nupdate = 1), + ## anim = animator(; setup, path = "$output/vorticity.mkv", nupdate = 20), + ## vtk = vtk_writer(; setup, nupdate = 10, dir = "$output", filename = "solution"), + ## field = fieldsaver(; setup, nupdate = 10), + log = timelogger(; nupdate = 1), + ), +); + +# ## Post-process +# +# We may visualize or export the computed fields `(u, p)`. + +# Export to VTK +save_vtk(setup, state.u, state.p, "$output/solution") + +# We create a box to visualize the actuator. +(; xc, yc, D, δ) = setup.bodyforce +box = [ + Point2f(xc - δ / 2, yc + D / 2), + Point2f(xc - δ / 2, yc - D / 2), + Point2f(xc + δ / 2, yc - D / 2), + Point2f(xc + δ / 2, yc + D / 2), + Point2f(xc - δ / 2, yc + D / 2), +] + +box = boxes[1] +boxes = map(bodyforce.forces) do (; xc, yc, D, δ) + [ + Point2f(xc - δ / 2, yc + D / 2), + Point2f(xc - δ / 2, yc - D / 2), + Point2f(xc + δ / 2, yc - D / 2), + Point2f(xc + δ / 2, yc + D / 2), + Point2f(xc - δ / 2, yc + D / 2), + ] +end + +state = (; u = F, t) + +# Plot pressure +fig = fieldplot(state; setup, fieldname = :pressure, psolver) +# lines!(box...; color = :red) +lines!.(boxes; color = :red); +fig + +sum(IncompressibleNavierStokes.pressure(u, t, setup; psolver)) + +# Plot velocity +fig = fieldplot(state; setup, fieldname = :velocity) +# lines!(box...; color = :red) +lines!.(boxes; color = :red); +fig + +# Plot vorticity +fig = fieldplot(state; setup, fieldname = :vorticity) +# lines!(box...; color = :red) +lines!.(boxes; color = :red); +fig From d069fe9c3dfc93b10363f5a5e236566137958be2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Thu, 4 Apr 2024 09:39:22 +0200 Subject: [PATCH 265/379] Fix bound --- Project.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Project.toml b/Project.toml index a83465bb3..8908a619d 100644 --- a/Project.toml +++ b/Project.toml @@ -31,7 +31,7 @@ Zygote = "e88e6eb3-aa80-5325-afca-941959d7151f" Adapt = "3, 4" Aqua = "0.8" CUDA = "5" -CUDSS = "1" +CUDSS = "0.1" CairoMakie = "0.11" ComponentArrays = "0.15" FFTW = "1" From 53a5da4829283b4ca3a606efe94eec59ff030d24 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Thu, 4 Apr 2024 17:05:02 +0200 Subject: [PATCH 266/379] Add reconstructor --- src/IncompressibleNavierStokes.jl | 2 +- src/filter.jl | 40 +++++++++++++++++++++++++++++++ 2 files changed, 41 insertions(+), 1 deletion(-) diff --git a/src/IncompressibleNavierStokes.jl b/src/IncompressibleNavierStokes.jl index dc91b42cf..7f33cd6cf 100644 --- a/src/IncompressibleNavierStokes.jl +++ b/src/IncompressibleNavierStokes.jl @@ -138,7 +138,7 @@ export create_dataloader_prior, create_dataloader_post export create_callback, create_les_data, create_io_arrays export wrappedclosure -export FaceAverage, VolumeAverage +export FaceAverage, VolumeAverage, reconstruct, reconstruct! # ODE methods diff --git a/src/filter.jl b/src/filter.jl index db01cc15c..a95fa1ce3 100644 --- a/src/filter.jl +++ b/src/filter.jl @@ -39,6 +39,46 @@ function (::FaceAverage)(v, u, setup_les, comp) v end +""" + reconstruct!(u, v, setup_dns, setup_les) + +Average fine grid `u` over coarse volume face. Put result in `v`. +""" +function reconstruct!(u, v, setup_dns, setup_les, comp) + (; grid, boundary_conditions, workgroupsize) = setup_les + (; N, Iu) = grid + D = length(u) + δ = Offset{D}() + @assert all(bc -> bc[1] isa PeriodicBC && bc[2] isa PeriodicBC, boundary_conditions) + @kernel function R!(u, v, ::Val{α}, volume) where {α} + J = @index(Global, Cartesian) + I = oneunit(J) + comp * J + J = oneunit(J) + J + Jleft = J - δ(α) + Jleft.I[α] == 1 && (Jleft += (N[α] - 2) * δ(α)) + for i in volume + s = zero(eltype(v[α])) + s += (comp - i.I[α]) * v[α][J] + s += i.I[α] * v[α][Jleft] + u[α][I-i] = s / comp + end + end + for α = 1:D + ndrange = N .- 2 + volume = CartesianIndices(ntuple(β -> 0:comp-1, D)) + R!(get_backend(v[1]), workgroupsize)(u, v, Val(α), volume; ndrange) + end + u +end + +reconstruct(v, setup_dns, setup_les, comp) = reconstruct!( + ntuple(α -> fill!(similar(v[1], setup_dns.grid.N), 0), length(v)), + v, + setup_dns, + setup_les, + comp, +) + """ (::VolumeAverage)(v, u, setup_les, comp) From ca5e3556b4d97121e0448b480aeed50658a0f306 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Fri, 5 Apr 2024 16:12:50 +0200 Subject: [PATCH 267/379] Up --- src/operators.jl | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/operators.jl b/src/operators.jl index 6fd4048cf..e0a0aaf40 100644 --- a/src/operators.jl +++ b/src/operators.jl @@ -97,7 +97,7 @@ end Compute divergence of velocity field. """ -divergence(u, setup) = divergence!(similar(u[1], setup.grid.N), u, setup) +divergence(u, setup) = divergence!(fill!(similar(u[1], setup.grid.N), 0), u, setup) ChainRulesCore.rrule(::typeof(divergence), u, setup) = ( divergence(u, setup), @@ -1192,14 +1192,14 @@ kinetic_energy(u, setup; kwargs...) = kinetic_energy!(similar(u[1], setup.grid.N), u, setup; kwargs...) """ - total_kinetic_energy(setup, u) + total_kinetic_energy(setup, u; kwargs...) Compute total kinetic energy. The velocity components are interpolated to the volume centers and squared. """ -function total_kinetic_energy(u, setup) +function total_kinetic_energy(u, setup; kwargs...) (; Ω, Ip) = setup.grid - e = kinetic_energy(u, setup) + e = kinetic_energy(u, setup; kwargs...) e .*= Ω sum(e[Ip]) end From f9acf4d6bcd298b96a735c7488cd9ba482c6024d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Mon, 15 Apr 2024 10:25:30 +0200 Subject: [PATCH 268/379] Add tensor basis --- docs/references.bib | 13 +++++++++++++ src/operators.jl | 45 ++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 57 insertions(+), 1 deletion(-) diff --git a/docs/references.bib b/docs/references.bib index b35e3baba..255f289b1 100644 --- a/docs/references.bib +++ b/docs/references.bib @@ -208,6 +208,19 @@ @article{Sanderse2020 volume = {421}, year = {2020} } +@article{Silvis2017, + title = {Physical consistency of subgrid-scale models for large-eddy simulation of incompressible turbulent flows}, + volume = {29}, + ISSN = {1089-7666}, + url = {http://dx.doi.org/10.1063/1.4974093}, + DOI = {10.1063/1.4974093}, + number = {1}, + journal = {Physics of Fluids}, + publisher = {AIP Publishing}, + author = {Silvis, Maurits H. and Remmerswaal, Ronald A. and Verstappen, Roel}, + year = {2017}, + month = jan, +} @article{Verstappen1997, doi = {10.1023/a:1004255329158}, url = {https://doi.org/10.1023%2Fa%3A1004255329158}, diff --git a/src/operators.jl b/src/operators.jl index e0a0aaf40..017d784d2 100644 --- a/src/operators.jl +++ b/src/operators.jl @@ -814,8 +814,12 @@ end ) / 4 @inline ∇(u, I::CartesianIndex{2}, Δ, Δu) = @SMatrix [∂x(u[α], I, α, β, Δ[β], Δu[β]) for α = 1:2, β = 1:2] -∇(u, I::CartesianIndex{3}, Δ, Δu) = +@inline ∇(u, I::CartesianIndex{3}, Δ, Δu) = @SMatrix [∂x(u[α], I, α, β, Δ[β], Δu[β]) for α = 1:3, β = 1:3] +@inline idtensor(u, I::CartesianIndex{2}) = + @SMatrix [(α == β) * oneunit(eltype(u[1])) for α = 1:2, β = 1:2] +@inline idtensor(u, I::CartesianIndex{3}) = + @SMatrix [(α == β) * oneunit(eltype(u[1])) for α = 1:3, β = 1:3] @inline function strain(u, I, Δ, Δu) ∇u = ∇(u, I, Δ, Δu) (∇u + ∇u') / 2 @@ -918,6 +922,45 @@ function smagorinsky_closure(setup) end end +""" + tensorbasis!(T, V, u, setup) + +Compute symmetry tensor basis `T[1]`-`T[11]` and invariants `V[1]`-`V[5]`, +as specified in [Silvis2017](@cite) in equations (9) and (11). +Note that `T[1]` corresponds to ``T_0`` in the paper. +""" +function tensorbasis!(T, V, u, setup) + (; grid, workgroupsize) = setup + (; Np, Ip, Δ, Δu) = grid + @kernel function basis!(T, V, u, I0) + I = @index(Global, Cartesian) + I = I + I0 + ∇u = ∇(u, I, Δ, Δu) + S = (∇u + ∇u') / 2 + R = (∇u - ∇u') / 2 + T[1][I] = idtensor(u, I) + T[2][I] = S + T[3][I] = S * S + T[4][I] = R * R + T[5][I] = S * R - R * S + T[6][I] = S * S * R - R * S * S + T[7][I] = S * R * R + R * R * S + T[8][I] = R * S * R * R - R * R * S * R + T[9][I] = S * R * S * S - S * S * R * S + T[10][I] = S * S * R * R + R * R * S * S + T[11][I] = R * S * S * R * R - R * R * S * S * R + V[1][I] = tr(S * S) + V[2][I] = tr(R * R) + V[3][I] = tr(S * S * S) + V[4][I] = tr(S * R * R) + V[5][I] = tr(S * S * R * R) + end + I0 = first(Ip) + I0 -= oneunit(I0) + basis!(get_backend(u[1]), workgroupsize)(T, V, u, I0; ndrange = Np) + T, V +end + """ interpolate_u_p(u, setup) From 5164efdf1fb96b305580ca9182cd4c8bb2e6ebb7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Wed, 17 Apr 2024 17:05:15 +0200 Subject: [PATCH 269/379] Add functions --- src/closures/closure.jl | 13 ++++++++- src/operators.jl | 62 ++++++++++++++++++++++++++--------------- 2 files changed, 51 insertions(+), 24 deletions(-) diff --git a/src/closures/closure.jl b/src/closures/closure.jl index e99b6b04a..9267c847a 100644 --- a/src/closures/closure.jl +++ b/src/closures/closure.jl @@ -32,7 +32,7 @@ function wrappedclosure(m, setup) end """ - create_closure(layers...) + create_closure(layers...; rng) Create neural closure model from layers. """ @@ -49,6 +49,17 @@ function create_closure(layers...; rng) closure, θ end +function create_tensorclosure(layers...; setup, rng) + D = setup.grid.dimension() + cnn, θ = create_closure(layers...; rng) + function closure(u, θ) + B, V = tensorbasis(u, setup) + V = stack(V) + α = cnn(V, θ) + τ = sum(k -> α[ntuple(Returns(:), D)..., k] .* B[k], 1:length(B)) + end +end + """ collocate(u) diff --git a/src/operators.jl b/src/operators.jl index 017d784d2..d1b9efe55 100644 --- a/src/operators.jl +++ b/src/operators.jl @@ -852,12 +852,12 @@ function smagtensor!(σ, u, θ, setup) end """ - smagorinsky!(s, σ, setup) + divoftensor!(s, σ, setup) -Compute the Smagorinsky closure term `s` (additional diffusive force). -The Smagorinsky stress tensors should be precomputed and stored in `σ`. +Compute divergence of a tensor with all components in the pressure points. +The stress tensors should be precomputed and stored in `σ`. """ -function smagorinsky!(s, σ, setup) +function divoftensor!(s, σ, setup) (; grid, workgroupsize) = setup (; dimension, Nu, Iu, Δ, Δu, A) = grid D = dimension() @@ -918,37 +918,37 @@ function smagorinsky_closure(setup) function closure(u, θ) smagtensor!(σ, u, θ, setup) apply_bc_p!(σ, zero(T), setup) - smagorinsky!(s, σ, setup) + divoftensor!(s, σ, setup) end end """ - tensorbasis!(T, V, u, setup) + tensorbasis!(B, V, u, setup) -Compute symmetry tensor basis `T[1]`-`T[11]` and invariants `V[1]`-`V[5]`, +Compute symmetry tensor basis `B[1]`-`B[11]` and invariants `V[1]`-`V[5]`, as specified in [Silvis2017](@cite) in equations (9) and (11). -Note that `T[1]` corresponds to ``T_0`` in the paper. +Note that `B[1]` corresponds to ``T_0`` in the paper, and `V` to ``I``. """ -function tensorbasis!(T, V, u, setup) +function tensorbasis!(B, V, u, setup) (; grid, workgroupsize) = setup (; Np, Ip, Δ, Δu) = grid - @kernel function basis!(T, V, u, I0) + @kernel function basis!(B, V, u, I0) I = @index(Global, Cartesian) I = I + I0 ∇u = ∇(u, I, Δ, Δu) S = (∇u + ∇u') / 2 R = (∇u - ∇u') / 2 - T[1][I] = idtensor(u, I) - T[2][I] = S - T[3][I] = S * S - T[4][I] = R * R - T[5][I] = S * R - R * S - T[6][I] = S * S * R - R * S * S - T[7][I] = S * R * R + R * R * S - T[8][I] = R * S * R * R - R * R * S * R - T[9][I] = S * R * S * S - S * S * R * S - T[10][I] = S * S * R * R + R * R * S * S - T[11][I] = R * S * S * R * R - R * R * S * S * R + B[1][I] = idtensor(u, I) + B[2][I] = S + B[3][I] = S * S + B[4][I] = R * R + B[5][I] = S * R - R * S + B[6][I] = S * S * R - R * S * S + B[7][I] = S * R * R + R * R * S + B[8][I] = R * S * R * R - R * R * S * R + B[9][I] = S * R * S * S - S * S * R * S + B[10][I] = S * S * R * R + R * R * S * S + B[11][I] = R * S * S * R * R - R * R * S * S * R V[1][I] = tr(S * S) V[2][I] = tr(R * R) V[3][I] = tr(S * S * S) @@ -957,8 +957,24 @@ function tensorbasis!(T, V, u, setup) end I0 = first(Ip) I0 -= oneunit(I0) - basis!(get_backend(u[1]), workgroupsize)(T, V, u, I0; ndrange = Np) - T, V + basis!(get_backend(u[1]), workgroupsize)(B, V, u, I0; ndrange = Np) + B, V +end + +""" + tensorbasis(u, setup) + +Compute symmetry tensor basis `T[1]`-`T[11]` and invariants `V[1]`-`V[5]`. +""" +function tensorbasis(u, setup) + T = eltype(u[1]) + D = setup.grid.dimension() + tensorbasis!( + ntuple(k -> similar(u[1], SMatrix{D,D,T,D * D}, setup.grid.N), 11), + ntuple(k -> similar(u[1], setup.grid.N), 5), + u, + setup, + ) end """ From df42e7eb4342cfd7fe4171501dc345854108805b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Fri, 19 Apr 2024 14:57:46 +0200 Subject: [PATCH 270/379] Old updates --- src/closures/fno.jl | 60 +++++++++++++++++++++++++++++++++++++++------ 1 file changed, 53 insertions(+), 7 deletions(-) diff --git a/src/closures/fno.jl b/src/closures/fno.jl index f5e103340..1246b3fe1 100644 --- a/src/closures/fno.jl +++ b/src/closures/fno.jl @@ -27,6 +27,8 @@ function fno(; setup, kmax, c, σ, ψ, rng = Random.default_rng(), kwargs...) # Put inputs in pressure points collocate, + # Conv(ntuple(Returns(1), D), D => c[1]; use_bias = false, init_weight), + # Some Fourier layers ( FourierLayer(dimension, kmax[i], c[i] => c[i+1]; σ = σ[i], init_weight) for @@ -34,9 +36,9 @@ function fno(; setup, kmax, c, σ, ψ, rng = Random.default_rng(), kwargs...) )..., # Compress with a final dense layer - Conv(ntuple(Returns(1), D), c[end] => D; use_bias = false), - # Conv(ntuple(Returns(1), D), c[end] => 2 * c[end], ψ; use_bias = false), - # Conv(ntuple(Returns(1), D), 2 * c[end] => D; use_bias = false), + # Conv(ntuple(Returns(1), D), c[end] => D; use_bias = false, init_weight), + Conv(ntuple(Returns(1), D), c[end] => 2 * c[end], ψ; init_weight), + Conv(ntuple(Returns(1), D), 2 * c[end] => D; use_bias = false, init_weight), # Differentiate output to velocity points decollocate, @@ -82,11 +84,14 @@ Lux.initialparameters( (; dimension, kmax, cin, cout, init_weight)::FourierLayer, ) = (; spatial_weight = init_weight(rng, cout, cin), - spectral_weights = init_weight(rng, fill(kmax + 1, dimension())..., cout, cin, 2), + # spectral_weights = init_weight(rng, fill(kmax + 1, dimension())..., cout, cin, 2), + spectral_weights = init_weight(rng, fill(2 * (kmax + 1), dimension())..., cout, cin, 2), ) Lux.initialstates(::AbstractRNG, ::FourierLayer) = (;) +# Lux.parameterlength((; dimension, kmax, cin, cout)::FourierLayer) = +# cout * cin + (kmax + 1)^dimension() * 2 * cout * cin Lux.parameterlength((; dimension, kmax, cin, cout)::FourierLayer) = - cout * cin + (kmax + 1)^dimension() * 2 * cout * cin + cout * cin + (2 * (kmax + 1))^dimension() * 2 * cout * cin Lux.statelength(::FourierLayer) = 0 ## Pretty printing @@ -110,6 +115,7 @@ function ((; dimension, kmax, cout, cin, σ)::FourierLayer)(x, params, state) D = dimension() nx..., _cin, nsample = size(x) + K = nx[1] @assert _cin == cin "Number of input channels must be compatible with weights" @assert all(==(first(nx)), nx) "Fourier layer requires same number of grid points in each dimension" @assert kmax ≤ first(nx) "Fourier layer input must be discretized on at least `kmax` points" @@ -136,16 +142,56 @@ function ((; dimension, kmax, cout, cin, σ)::FourierLayer)(x, params, state) # - multiply with weights mode-wise # - pad with zeros to restore original shape # - go back to real valued spatial representation - ikeep = ntuple(Returns(1:kmax+1), D) + ikeep = ntuple(Returns([1:kmax+1; K-kmax:K]), D) + # ikeep = ntuple(Returns(1:kmax+1), D) dims = ntuple(identity, D) xhat = fft(x, dims) xhat = xhat[ikeep..., :, :] if D == 2 @tullio z[k₁, k₂, b, s] := R[k₁, k₂, b, a] * xhat[k₁, k₂, a, s] + z = cat( + z[1:kmax+1, :, :, :], + zero(similar(z, K - 2 * (kmax + 1), 2 * (kmax + 1), cout, nsample)), + z[end-kmax:end, :, :, :]; + dims = 1, + ) + z = cat( + z[:, 1:kmax+1, :, :], + zero(similar(z, K, K - 2 * (kmax + 1), cout, nsample)), + z[:, end-kmax:end, :, :]; + dims = 2, + ) elseif D == 3 @tullio z[k₁, k₂, k₃, b, s] := R[k₁, k₂, k₃, b, a] * xhat[k₁, k₂, k₃, a, s] + z = cat( + z[1:kmax+1, :, :, :, :], + zero( + similar( + z, + K - 2 * (kmax + 1), + 2 * (kmax + 1), + 2 * (kmax + 1), + cout, + nsample, + ), + ), + z[end-kmax:end, :, :, :, :]; + dims = 1, + ) + z = cat( + z[:, 1:kmax+1, :, :, :], + zero(similar(z, K, K - 2 * (kmax + 1), 2 * (kmax + 1), cout, nsample)), + z[:, end-kmax:end, :, :, :]; + dims = 2, + ) + z = cat( + z[:, :, 1:kmax+1, :, :], + zero(similar(z, K, K, K - 2 * (kmax + 1), cout, nsample)), + z[:, :, end-kmax:end, :, :]; + dims = 3, + ) end - z = pad_zeros(z, ntuple(i -> isodd(i) ? 0 : first(nx) - kmax - 1, 2N); dims) + # z = pad_zeros(z, ntuple(i -> isodd(i) ? 0 : first(nx) - kmax - 1, 2 * D); dims) z = real.(ifft(z, dims)) # Outer layer: Activation over combined spatial and spectral parts From 3c3725e034be375739f0e56b9c0b5467f581455d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Fri, 19 Apr 2024 15:42:53 +0200 Subject: [PATCH 271/379] Split out neural stuff in a sub-package --- Project.toml | 6 ---- libs/NeuralClosure/LICENSE | 21 ++++++++++++++ libs/NeuralClosure/Project.toml | 14 ++++++++++ libs/NeuralClosure/src/NeuralClosure.jl | 28 +++++++++++++++++++ .../NeuralClosure/src}/closure.jl | 0 .../NeuralClosure/src}/cnn.jl | 0 .../NeuralClosure/src}/create_les_data.jl | 0 .../NeuralClosure/src}/fno.jl | 2 ++ .../NeuralClosure/src}/training.jl | 0 src/IncompressibleNavierStokes.jl | 22 --------------- 10 files changed, 65 insertions(+), 28 deletions(-) create mode 100644 libs/NeuralClosure/LICENSE create mode 100644 libs/NeuralClosure/Project.toml create mode 100644 libs/NeuralClosure/src/NeuralClosure.jl rename {src/closures => libs/NeuralClosure/src}/closure.jl (100%) rename {src/closures => libs/NeuralClosure/src}/cnn.jl (100%) rename {src/closures => libs/NeuralClosure/src}/create_les_data.jl (100%) rename {src/closures => libs/NeuralClosure/src}/fno.jl (99%) rename {src/closures => libs/NeuralClosure/src}/training.jl (100%) diff --git a/Project.toml b/Project.toml index 8908a619d..084b868ba 100644 --- a/Project.toml +++ b/Project.toml @@ -8,24 +8,18 @@ Adapt = "79e6a3ab-5dfb-504d-930d-738a2a938a0e" CUDA = "052768ef-5323-5732-b1bb-66c8b64840ba" CUDSS = "45b445bb-4962-46a0-9369-b4df9d0f772e" ChainRulesCore = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4" -ComponentArrays = "b0b7db55-cfe3-40fc-9ded-d10e2dbeff66" FFTW = "7a1cc6ca-52ef-59f5-83cd-3a7055c09341" IterativeSolvers = "42fd0dbc-a981-5370-80f2-aaf504508153" KernelAbstractions = "63c18a36-062a-441e-b654-da1e3ab1ce7c" LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" -Lux = "b2108857-7c20-44ae-9111-449ecde12c47" Makie = "ee78f7c6-11fb-53f2-987a-cfe4a2b5a57a" -NNlib = "872c559c-99b0-510c-b3b7-b6c96a88d5cd" Observables = "510215fc-4207-5dde-b226-833fc4488ee2" -Optimisers = "3bd65402-5787-11e9-1adc-39752487f4e2" Printf = "de0858da-6303-5e67-8744-51eddeeeb8d7" Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" SparseArrays = "2f01184e-e22b-5df5-ae63-d93ebab69eaf" StaticArrays = "90137ffa-7385-5640-81b9-e52037218182" Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2" -Tullio = "bc48ee85-29a4-5162-ae0b-a64e1601d4bc" WriteVTK = "64499a7a-5c06-52f2-abe2-ccb03c286192" -Zygote = "e88e6eb3-aa80-5325-afca-941959d7151f" [compat] Adapt = "3, 4" diff --git a/libs/NeuralClosure/LICENSE b/libs/NeuralClosure/LICENSE new file mode 100644 index 000000000..d76a52488 --- /dev/null +++ b/libs/NeuralClosure/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2024 Syver Døving Agdestein, Benjamin Sanderse, and contributors + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/libs/NeuralClosure/Project.toml b/libs/NeuralClosure/Project.toml new file mode 100644 index 000000000..a96650128 --- /dev/null +++ b/libs/NeuralClosure/Project.toml @@ -0,0 +1,14 @@ +name = "NeuralClosure" +uuid = "099dac27-d7f2-4047-93d5-0baee36b9c25" +authors = ["Syver Døving Agdestein "] +version = "0.1.0" + +[deps] +IncompressibleNavierStokes = "5e318141-6589-402b-868d-77d7df8c442e" +ComponentArrays = "b0b7db55-cfe3-40fc-9ded-d10e2dbeff66" +Lux = "b2108857-7c20-44ae-9111-449ecde12c47" +NNlib = "872c559c-99b0-510c-b3b7-b6c96a88d5cd" +Optimisers = "3bd65402-5787-11e9-1adc-39752487f4e2" +Tullio = "bc48ee85-29a4-5162-ae0b-a64e1601d4bc" +Zygote = "e88e6eb3-aa80-5325-afca-941959d7151f" +Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" diff --git a/libs/NeuralClosure/src/NeuralClosure.jl b/libs/NeuralClosure/src/NeuralClosure.jl new file mode 100644 index 000000000..1ef3f0565 --- /dev/null +++ b/libs/NeuralClosure/src/NeuralClosure.jl @@ -0,0 +1,28 @@ +""" +Neural closure modelling tools. +""" +module NeuralClosure + +using CUDA +using ComponentArrays: ComponentArray +using Lux +using NNlib +using Tullio +using Zygote + +include("closure.jl") +include("cnn.jl") +include("fno.jl") +include("training.jl") +include("create_les_data.jl") + +export smagorinsky_closure +export cnn, fno, FourierLayer +export train +export mean_squared_error, create_relerr_prior, create_relerr_post +export create_loss_prior, create_loss_post +export create_dataloader_prior, create_dataloader_post +export create_callback, create_les_data, create_io_arrays +export wrappedclosure + +end diff --git a/src/closures/closure.jl b/libs/NeuralClosure/src/closure.jl similarity index 100% rename from src/closures/closure.jl rename to libs/NeuralClosure/src/closure.jl diff --git a/src/closures/cnn.jl b/libs/NeuralClosure/src/cnn.jl similarity index 100% rename from src/closures/cnn.jl rename to libs/NeuralClosure/src/cnn.jl diff --git a/src/closures/create_les_data.jl b/libs/NeuralClosure/src/create_les_data.jl similarity index 100% rename from src/closures/create_les_data.jl rename to libs/NeuralClosure/src/create_les_data.jl diff --git a/src/closures/fno.jl b/libs/NeuralClosure/src/fno.jl similarity index 99% rename from src/closures/fno.jl rename to libs/NeuralClosure/src/fno.jl index 1246b3fe1..38edb78da 100644 --- a/src/closures/fno.jl +++ b/libs/NeuralClosure/src/fno.jl @@ -203,6 +203,8 @@ function ((; dimension, kmax, cout, cin, σ)::FourierLayer)(x, params, state) # Fourier layer. v = σ.(y .+ z) + # @infiltrate + # Fourier layer does not modify state v, state end diff --git a/src/closures/training.jl b/libs/NeuralClosure/src/training.jl similarity index 100% rename from src/closures/training.jl rename to libs/NeuralClosure/src/training.jl diff --git a/src/IncompressibleNavierStokes.jl b/src/IncompressibleNavierStokes.jl index 7f33cd6cf..0d67f930a 100644 --- a/src/IncompressibleNavierStokes.jl +++ b/src/IncompressibleNavierStokes.jl @@ -7,23 +7,18 @@ module IncompressibleNavierStokes using Adapt using ChainRulesCore -using ComponentArrays: ComponentArray using FFTW using IterativeSolvers using KernelAbstractions using LinearAlgebra -using Lux using Makie -using NNlib using Optimisers using Printf using Random using SparseArrays using StaticArrays using Statistics -using Tullio using WriteVTK: CollectionFile, paraview_collection, vtk_grid, vtk_save -using Zygote # Must be loaded inside for Tullio to work correctly using CUDA @@ -89,13 +84,6 @@ include("utils/get_lims.jl") include("utils/plotmat.jl") include("utils/spectral_stuff.jl") -# Closure models -include("closures/closure.jl") -include("closures/cnn.jl") -include("closures/fno.jl") -include("closures/training.jl") -include("closures/create_les_data.jl") - # Boundary conditions export PeriodicBC, DirichletBC, SymmetricBC, PressureBC @@ -128,16 +116,6 @@ export create_initial_conditions, random_field export plotgrid, save_vtk export plotmat -# Closure models -export smagorinsky_closure -export cnn, fno, FourierLayer -export train -export mean_squared_error, create_relerr_prior, create_relerr_post -export create_loss_prior, create_loss_post -export create_dataloader_prior, create_dataloader_post -export create_callback, create_les_data, create_io_arrays -export wrappedclosure - export FaceAverage, VolumeAverage, reconstruct, reconstruct! # ODE methods From dbe84588f6c29b404b46e1bd849cb41cefdc3692 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Fri, 19 Apr 2024 15:44:53 +0200 Subject: [PATCH 272/379] Add README --- libs/NeuralClosure/README.md | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 libs/NeuralClosure/README.md diff --git a/libs/NeuralClosure/README.md b/libs/NeuralClosure/README.md new file mode 100644 index 000000000..792d724d6 --- /dev/null +++ b/libs/NeuralClosure/README.md @@ -0,0 +1,4 @@ +## NeuralClosure + +Neural closure modeling tools for +[IncompressibleNavierStokes.jl](https://github.com/agdestein/IncompressibleNavierStokes.jl). From eb670c6ab327af5dcd1769ebf4b7250158e0227f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Fri, 19 Apr 2024 15:58:10 +0200 Subject: [PATCH 273/379] Update project files --- Project.toml | 15 --------------- libs/NeuralClosure/Project.toml | 16 ++++++++++++++-- test/Project.toml | 4 ++++ 3 files changed, 18 insertions(+), 17 deletions(-) create mode 100644 test/Project.toml diff --git a/Project.toml b/Project.toml index 084b868ba..b53adce4d 100644 --- a/Project.toml +++ b/Project.toml @@ -23,37 +23,22 @@ WriteVTK = "64499a7a-5c06-52f2-abe2-ccb03c286192" [compat] Adapt = "3, 4" -Aqua = "0.8" CUDA = "5" CUDSS = "0.1" -CairoMakie = "0.11" -ComponentArrays = "0.15" FFTW = "1" -GLMakie = "0.9" IterativeSolvers = "0.9" KernelAbstractions = "0.9" LinearAlgebra = "1" -Lux = "0.5" Makie = "0.20" -NNlib = "0.9" Observables = "0.5" -Optimisers = "0.3" Printf = "1" Random = "1" SparseArrays = "1" StaticArrays = "1" Statistics = "1" -Test = "1" -Tullio = "0.3" WriteVTK = "1" -Zygote = "0.6" julia = "1.7" [extras] -Aqua = "4c88cf16-eb10-579e-8560-4a9242c79595" CairoMakie = "13f3f980-e62b-5c42-98c6-ff1f3baf88f0" GLMakie = "e9467ef8-e4e7-5192-8a1a-b1aee30e663a" -Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" - -[targets] -test = ["Aqua", "CairoMakie", "Test"] diff --git a/libs/NeuralClosure/Project.toml b/libs/NeuralClosure/Project.toml index a96650128..c2becd591 100644 --- a/libs/NeuralClosure/Project.toml +++ b/libs/NeuralClosure/Project.toml @@ -4,11 +4,23 @@ authors = ["Syver Døving Agdestein "] version = "0.1.0" [deps] -IncompressibleNavierStokes = "5e318141-6589-402b-868d-77d7df8c442e" +CUDA = "052768ef-5323-5732-b1bb-66c8b64840ba" ComponentArrays = "b0b7db55-cfe3-40fc-9ded-d10e2dbeff66" +IncompressibleNavierStokes = "5e318141-6589-402b-868d-77d7df8c442e" Lux = "b2108857-7c20-44ae-9111-449ecde12c47" NNlib = "872c559c-99b0-510c-b3b7-b6c96a88d5cd" Optimisers = "3bd65402-5787-11e9-1adc-39752487f4e2" +Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" Tullio = "bc48ee85-29a4-5162-ae0b-a64e1601d4bc" Zygote = "e88e6eb3-aa80-5325-afca-941959d7151f" -Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" + +[compat] +ComponentArrays = "0.15" +CUDA = "5" +Lux = "0.5" +NNlib = "0.9" +Optimisers = "0.3" +Random = "1" +Tullio = "0.3" +Zygote = "0.6" +julia = "1.7" diff --git a/test/Project.toml b/test/Project.toml new file mode 100644 index 000000000..b62ce12ba --- /dev/null +++ b/test/Project.toml @@ -0,0 +1,4 @@ +[deps] +Aqua = "4c88cf16-eb10-579e-8560-4a9242c79595" +CairoMakie = "13f3f980-e62b-5c42-98c6-ff1f3baf88f0" +IncompressibleNavierStokes = "5e318141-6589-402b-868d-77d7df8c442e" From 56188f4361cf96d0ce34c57bc0b5ddab0b24400b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Fri, 19 Apr 2024 15:58:33 +0200 Subject: [PATCH 274/379] Remove trailing package --- src/IncompressibleNavierStokes.jl | 1 - 1 file changed, 1 deletion(-) diff --git a/src/IncompressibleNavierStokes.jl b/src/IncompressibleNavierStokes.jl index 0d67f930a..c5b84b3f2 100644 --- a/src/IncompressibleNavierStokes.jl +++ b/src/IncompressibleNavierStokes.jl @@ -12,7 +12,6 @@ using IterativeSolvers using KernelAbstractions using LinearAlgebra using Makie -using Optimisers using Printf using Random using SparseArrays From f8fc8240bd9a10592ef74fcf4fff814eb613ed29 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Fri, 19 Apr 2024 16:55:44 +0200 Subject: [PATCH 275/379] Put paper scripts in subfolder --- PaperDC/LICENSE | 21 +++++ PaperDC/Project.toml | 24 ++++++ PaperDC/README.md | 33 ++++++++ {scratch => PaperDC}/postanalysis.jl | 22 +++--- {scratch => PaperDC}/prioranalysis.jl | 100 +++--------------------- PaperDC/src/PaperDC.jl | 99 +++++++++++++++++++++++ libs/NeuralClosure/src/NeuralClosure.jl | 1 + 7 files changed, 200 insertions(+), 100 deletions(-) create mode 100644 PaperDC/LICENSE create mode 100644 PaperDC/Project.toml create mode 100644 PaperDC/README.md rename {scratch => PaperDC}/postanalysis.jl (98%) rename {scratch => PaperDC}/prioranalysis.jl (82%) create mode 100644 PaperDC/src/PaperDC.jl diff --git a/PaperDC/LICENSE b/PaperDC/LICENSE new file mode 100644 index 000000000..d76a52488 --- /dev/null +++ b/PaperDC/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2024 Syver Døving Agdestein, Benjamin Sanderse, and contributors + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/PaperDC/Project.toml b/PaperDC/Project.toml new file mode 100644 index 000000000..16bc477da --- /dev/null +++ b/PaperDC/Project.toml @@ -0,0 +1,24 @@ +name = "PaperDC" +uuid = "0d06c178-ba13-4c21-b726-6f4489412f0d" +authors = ["Syver Døving Agdestein "] +version = "0.1.0" + +[deps] +Adapt = "79e6a3ab-5dfb-504d-930d-738a2a938a0e" +CUDA = "052768ef-5323-5732-b1bb-66c8b64840ba" +CairoMakie = "13f3f980-e62b-5c42-98c6-ff1f3baf88f0" +FFTW = "7a1cc6ca-52ef-59f5-83cd-3a7055c09341" +GLMakie = "e9467ef8-e4e7-5192-8a1a-b1aee30e663a" +IncompressibleNavierStokes = "5e318141-6589-402b-868d-77d7df8c442e" +JLD2 = "033835bb-8acc-5ee8-8aae-3f567f8a3819" +LaTeXStrings = "b964fa9f-0449-5b57-a5c2-d3ea65f4040f" +LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" +Lux = "b2108857-7c20-44ae-9111-449ecde12c47" +LuxCUDA = "d0bbae9a-e099-4d5b-a835-1c6931763bda" +NNlib = "872c559c-99b0-510c-b3b7-b6c96a88d5cd" +NeuralClosure = "099dac27-d7f2-4047-93d5-0baee36b9c25" +Observables = "510215fc-4207-5dde-b226-833fc4488ee2" +Optimisers = "3bd65402-5787-11e9-1adc-39752487f4e2" +Printf = "de0858da-6303-5e67-8744-51eddeeeb8d7" +Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" +SparseArrays = "2f01184e-e22b-5df5-ae63-d93ebab69eaf" diff --git a/PaperDC/README.md b/PaperDC/README.md new file mode 100644 index 000000000..0c14fbc75 --- /dev/null +++ b/PaperDC/README.md @@ -0,0 +1,33 @@ +# PaperDC + +Scripts for generating results of the paper +[Discretize first, filter next: learning divergence-consistent closure models for large-eddy simulation](https://arxiv.org/abs/2403.18088). + +## Set up environment + +From this directory, run: + +```sh +julia --project -e ' +using Pkg +Pkg.develop([ + PackageSpec(; path = ".."), + PackageSpec(; path = "../libs/NeuralClosure"), +]) +Pkg.instantiate()' +' +``` + +or interactively from a Julia REPL: + +```julia-repl +julia> ] +(v1.10) pkg> activate . +(PaperDC) pkg> dev .. ../libs/NeuralClosure +(PaperDC) pkg> instantiate +``` + +Now you can run the scripts in this directory: + +- `prioranalysis.jl`: Generate results for section 5.1 "Filtered DNS (2D and 3D)" +- `postanalysis.jl`: Generate results for section 5.2 "LES (2D)" diff --git a/scratch/postanalysis.jl b/PaperDC/postanalysis.jl similarity index 98% rename from scratch/postanalysis.jl rename to PaperDC/postanalysis.jl index 9ca69c30f..c80a46aff 100644 --- a/scratch/postanalysis.jl +++ b/PaperDC/postanalysis.jl @@ -1,9 +1,9 @@ -# Little LSP hack to get function signatures, go #src -# to definition etc. #src -if isdefined(@__MODULE__, :LanguageServer) #src - include("../src/IncompressibleNavierStokes.jl") #src - using .IncompressibleNavierStokes #src -end #src +# # Little LSP hack to get function signatures, go #src +# # to definition etc. #src +# if isdefined(@__MODULE__, :LanguageServer) #src +# include("../src/IncompressibleNavierStokes.jl") #src +# using .IncompressibleNavierStokes #src +# end #src # # Train closure model # @@ -11,8 +11,8 @@ end #src # uniform Cartesian grid with square cells. using Adapt -using CairoMakie using GLMakie +using CairoMakie using IncompressibleNavierStokes using JLD2 using LaTeXStrings @@ -21,9 +21,7 @@ using Lux using NNlib using Optimisers using Random -using Zygote using SparseArrays -using KernelAbstractions using FFTW # palette = (; color = ["#3366cc", "#cc0000", "#669900", "#ffcc00"]) @@ -44,8 +42,9 @@ GLMakie.activate!() set_theme!(; GLMakie = (; scalefactor = 1.5)) -plotdir = "../SupervisedClosure/figures/" -outdir = "output/postanalysis" +plotdir = "../SupervisedClosure/figures" +# outdir = "output/postanalysis" +outdir = "output/divfree" ispath(outdir) || mkpath(outdir) # Random number generator @@ -308,6 +307,7 @@ for ifil = 1:1, ig = 4:4 opt, θ; niter = 10_000, + # niter = 100, ncallback = 20, callbackstate, callback, diff --git a/scratch/prioranalysis.jl b/PaperDC/prioranalysis.jl similarity index 82% rename from scratch/prioranalysis.jl rename to PaperDC/prioranalysis.jl index 8dc6a39b0..9c594b9d9 100644 --- a/scratch/prioranalysis.jl +++ b/PaperDC/prioranalysis.jl @@ -1,11 +1,12 @@ -# Little LSP hack to get function signatures, go #src -# to definition etc. #src -if isdefined(@__MODULE__, :LanguageServer) #src - include("../src/IncompressibleNavierStokes.jl") #src - using .IncompressibleNavierStokes #src -end #src +# # Little LSP hack to get function signatures, go #src +# # to definition etc. #src +# if isdefined(@__MODULE__, :LanguageServer) #src +# include("../src/IncompressibleNavierStokes.jl") #src +# using .IncompressibleNavierStokes #src +# end #src using GLMakie +using CairoMakie using IncompressibleNavierStokes using IncompressibleNavierStokes: momentum, @@ -17,89 +18,10 @@ using IncompressibleNavierStokes: spectral_stuff, kinetic_energy, interpolate_u_p -using LinearAlgebra +using NeuralClosure using Printf using FFTW - -function observe_v(dnsobs, Φ, les, compression, psolver) - (; grid) = les - (; dimension, N, Iu, Ip) = grid - D = dimension() - Mα = N[1] - 2 - v = zero.(Φ(dnsobs[].u, les, compression)) - Pv = zero.(v) - p = zero(v[1]) - div = zero(p) - ΦPF = zero.(v) - PFΦ = zero.(v) - c = zero.(v) - T = eltype(v[1]) - results = (; - Φ, - Mα, - t = zeros(T, 0), - Dv = zeros(T, 0), - Pv = zeros(T, 0), - Pc = zeros(T, 0), - c = zeros(T, 0), - ) - on(dnsobs) do (; u, PF, t) - push!(results.t, t) - - Φ(v, u, les, compression) - apply_bc_u!(v, t, les) - Φ(ΦPF, PF, les, compression) - momentum!(PFΦ, v, t, les) - apply_bc_u!(PFΦ, t, les; dudt = true) - project!(PFΦ, les; psolver, div, p) - foreach(α -> c[α] .= ΦPF[α] .- PFΦ[α], 1:D) - apply_bc_u!(c, t, les) - divergence!(div, v, les) - norm_Du = norm(div[Ip]) - norm_v = sqrt(sum(α -> sum(abs2, v[α][Iu[α]]), 1:D)) - push!(results.Dv, norm_Du / norm_v) - - copyto!.(Pv, v) - project!(Pv, les; psolver, div, p) - foreach(α -> Pv[α] .= Pv[α] .- v[α], 1:D) - norm_vmPv = sqrt(sum(α -> sum(abs2, Pv[α][Iu[α]]), 1:D)) - push!(results.Pv, norm_vmPv / norm_v) - - Pc = Pv - copyto!.(Pc, c) - project!(Pc, les; psolver, div, p) - foreach(α -> Pc[α] .= Pc[α] .- c[α], 1:D) - norm_cmPc = sqrt(sum(α -> sum(abs2, Pc[α][Iu[α]]), 1:D)) - norm_c = sqrt(sum(α -> sum(abs2, c[α][Iu[α]]), 1:D)) - push!(results.Pc, norm_cmPc / norm_c) - - norm_ΦPF = sqrt(sum(α -> sum(abs2, ΦPF[α][Iu[α]]), 1:D)) - push!(results.c, norm_c / norm_ΦPF) - end - results -end - -observe_u(dns, psolver_dns, filters; nupdate = 1) = - processor() do state - PF = zero.(state[].u) - div = zero(state[].u[1]) - p = zero(state[].u[1]) - dnsobs = Observable((; state[].u, PF, state[].t)) - results = [ - observe_v(dnsobs, Φ, setup, compression, psolver) for - (; setup, Φ, compression, psolver) in filters - ] - on(state) do (; u, t, n) - n % nupdate == 0 || return - apply_bc_u!(u, t, dns) - momentum!(PF, u, t, dns) - apply_bc_u!(PF, t, dns; dudt = true) - project!(PF, dns; psolver = psolver_dns, div, p) - dnsobs[] = (; u, PF, t) - end - # state[] = state[] # Save initial conditions - results - end +using PaperDC # Output directory output = "output/divergence" @@ -138,7 +60,7 @@ filterdefs = [ # 2D T = Float64; Re = T(10_000) -ndns = 4096 +ndns = 1024 D = 2 kp = 20 Δt = T(5e-5) @@ -200,7 +122,7 @@ fieldplot( # # levels = 5, # docolorbar = false, # ), - # obs = observe_u(dns, psolver_dns, filters; nupdate = 20), + obs = observe_u(dns, psolver_dns, filters; nupdate = 20), # ehist = realtimeplotter(; # setup, # plot = energy_history_plot, diff --git a/PaperDC/src/PaperDC.jl b/PaperDC/src/PaperDC.jl new file mode 100644 index 000000000..ea09e1772 --- /dev/null +++ b/PaperDC/src/PaperDC.jl @@ -0,0 +1,99 @@ +module PaperDC + +using IncompressibleNavierStokes +using IncompressibleNavierStokes: + momentum, + momentum!, + divergence!, + project, + project!, + apply_bc_u!, + spectral_stuff, + kinetic_energy, + interpolate_u_p +using Observables +using LinearAlgebra + +function observe_v(dnsobs, Φ, les, compression, psolver) + (; grid) = les + (; dimension, N, Iu, Ip) = grid + D = dimension() + Mα = N[1] - 2 + v = zero.(Φ(dnsobs[].u, les, compression)) + Pv = zero.(v) + p = zero(v[1]) + div = zero(p) + ΦPF = zero.(v) + PFΦ = zero.(v) + c = zero.(v) + T = eltype(v[1]) + results = (; + Φ, + Mα, + t = zeros(T, 0), + Dv = zeros(T, 0), + Pv = zeros(T, 0), + Pc = zeros(T, 0), + c = zeros(T, 0), + ) + on(dnsobs) do (; u, PF, t) + push!(results.t, t) + + Φ(v, u, les, compression) + apply_bc_u!(v, t, les) + Φ(ΦPF, PF, les, compression) + momentum!(PFΦ, v, t, les) + apply_bc_u!(PFΦ, t, les; dudt = true) + project!(PFΦ, les; psolver, div, p) + foreach(α -> c[α] .= ΦPF[α] .- PFΦ[α], 1:D) + apply_bc_u!(c, t, les) + divergence!(div, v, les) + norm_Du = norm(div[Ip]) + norm_v = sqrt(sum(α -> sum(abs2, v[α][Iu[α]]), 1:D)) + push!(results.Dv, norm_Du / norm_v) + + copyto!.(Pv, v) + project!(Pv, les; psolver, div, p) + foreach(α -> Pv[α] .= Pv[α] .- v[α], 1:D) + norm_vmPv = sqrt(sum(α -> sum(abs2, Pv[α][Iu[α]]), 1:D)) + push!(results.Pv, norm_vmPv / norm_v) + + Pc = Pv + copyto!.(Pc, c) + project!(Pc, les; psolver, div, p) + foreach(α -> Pc[α] .= Pc[α] .- c[α], 1:D) + norm_cmPc = sqrt(sum(α -> sum(abs2, Pc[α][Iu[α]]), 1:D)) + norm_c = sqrt(sum(α -> sum(abs2, c[α][Iu[α]]), 1:D)) + push!(results.Pc, norm_cmPc / norm_c) + + norm_ΦPF = sqrt(sum(α -> sum(abs2, ΦPF[α][Iu[α]]), 1:D)) + push!(results.c, norm_c / norm_ΦPF) + end + results +end + +observe_u(dns, psolver_dns, filters; nupdate = 1) = + processor() do state + PF = zero.(state[].u) + div = zero(state[].u[1]) + p = zero(state[].u[1]) + dnsobs = Observable((; state[].u, PF, state[].t)) + results = [ + observe_v(dnsobs, Φ, setup, compression, psolver) for + (; setup, Φ, compression, psolver) in filters + ] + on(state) do (; u, t, n) + n % nupdate == 0 || return + apply_bc_u!(u, t, dns) + momentum!(PF, u, t, dns) + apply_bc_u!(PF, t, dns; dudt = true) + project!(PF, dns; psolver = psolver_dns, div, p) + dnsobs[] = (; u, PF, t) + end + # state[] = state[] # Save initial conditions + results + end + +export observe_u, observe_v + +end # module PaperDC diff --git a/libs/NeuralClosure/src/NeuralClosure.jl b/libs/NeuralClosure/src/NeuralClosure.jl index 1ef3f0565..083c173db 100644 --- a/libs/NeuralClosure/src/NeuralClosure.jl +++ b/libs/NeuralClosure/src/NeuralClosure.jl @@ -7,6 +7,7 @@ using CUDA using ComponentArrays: ComponentArray using Lux using NNlib +using Random using Tullio using Zygote From ce962c32b501a179a7915fcfdad5fedb9cacd730 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Fri, 19 Apr 2024 17:00:09 +0200 Subject: [PATCH 276/379] Add sub-tests --- libs/NeuralClosure/test/Project.toml | 4 ++++ libs/NeuralClosure/test/runtests.jl | 10 ++++++++++ 2 files changed, 14 insertions(+) create mode 100644 libs/NeuralClosure/test/Project.toml create mode 100644 libs/NeuralClosure/test/runtests.jl diff --git a/libs/NeuralClosure/test/Project.toml b/libs/NeuralClosure/test/Project.toml new file mode 100644 index 000000000..53c677c2a --- /dev/null +++ b/libs/NeuralClosure/test/Project.toml @@ -0,0 +1,4 @@ +[deps] +Aqua = "4c88cf16-eb10-579e-8560-4a9242c79595" +IncompressibleNavierStokes = "5e318141-6589-402b-868d-77d7df8c442e" +NeuralClosure = "099dac27-d7f2-4047-93d5-0baee36b9c25" diff --git a/libs/NeuralClosure/test/runtests.jl b/libs/NeuralClosure/test/runtests.jl new file mode 100644 index 000000000..c4a7c4c87 --- /dev/null +++ b/libs/NeuralClosure/test/runtests.jl @@ -0,0 +1,10 @@ +using Aqua +using IncompressibleNavierStokes +using NeuralClosure + +@testset "NeuralClosure" begin + @testset "Aqua" begin + @info "Testing code with Aqua" + Aqua.test_all(NeuralClosure; ambiguities = false) + end +end From 2f7e81e3d205d94677b3f3f1b86c8a44c111307c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Fri, 19 Apr 2024 17:03:16 +0200 Subject: [PATCH 277/379] Add project file --- examples/Project.toml | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 examples/Project.toml diff --git a/examples/Project.toml b/examples/Project.toml new file mode 100644 index 000000000..fd652a236 --- /dev/null +++ b/examples/Project.toml @@ -0,0 +1,11 @@ +[deps] +CUDA = "052768ef-5323-5732-b1bb-66c8b64840ba" +CairoMakie = "13f3f980-e62b-5c42-98c6-ff1f3baf88f0" +GLMakie = "e9467ef8-e4e7-5192-8a1a-b1aee30e663a" +IncompressibleNavierStokes = "5e318141-6589-402b-868d-77d7df8c442e" +JLD2 = "033835bb-8acc-5ee8-8aae-3f567f8a3819" +LaTeXStrings = "b964fa9f-0449-5b57-a5c2-d3ea65f4040f" +LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" +Literate = "98b081ad-f1c9-55d3-8b20-4c87d4299306" +Printf = "de0858da-6303-5e67-8744-51eddeeeb8d7" +SparseArrays = "2f01184e-e22b-5df5-ae63-d93ebab69eaf" From a0a19804a8e28757c0217abb729ac691939affdc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Fri, 19 Apr 2024 17:06:26 +0200 Subject: [PATCH 278/379] Remove LSP trick --- PaperDC/postanalysis.jl | 7 ------- PaperDC/prioranalysis.jl | 7 ------- docs/make.jl | 7 ------- examples/Actuator2D.jl | 7 ------- examples/Actuator3D.jl | 7 ------- examples/BackwardFacingStep2D.jl | 7 ------- examples/BackwardFacingStep3D.jl | 7 ------- examples/DecayingTurbulence2D.jl | 7 ------- examples/DecayingTurbulence3D.jl | 7 ------- examples/LidDrivenCavity2D.jl | 7 ------- examples/LidDrivenCavity3D.jl | 7 ------- examples/MultiActuator.jl | 7 ------- examples/PlanarMixing2D.jl | 7 ------- examples/PlaneJets2D.jl | 7 ------- examples/ShearLayer2D.jl | 7 ------- examples/TaylorGreenVortex2D.jl | 7 ------- examples/TaylorGreenVortex3D.jl | 7 ------- test/runtests.jl | 7 ------- 18 files changed, 126 deletions(-) diff --git a/PaperDC/postanalysis.jl b/PaperDC/postanalysis.jl index c80a46aff..cc520e8aa 100644 --- a/PaperDC/postanalysis.jl +++ b/PaperDC/postanalysis.jl @@ -1,10 +1,3 @@ -# # Little LSP hack to get function signatures, go #src -# # to definition etc. #src -# if isdefined(@__MODULE__, :LanguageServer) #src -# include("../src/IncompressibleNavierStokes.jl") #src -# using .IncompressibleNavierStokes #src -# end #src - # # Train closure model # # Here, we consider a periodic box ``[0, 1]^2``. It is discretized with a diff --git a/PaperDC/prioranalysis.jl b/PaperDC/prioranalysis.jl index 9c594b9d9..e2cb35b70 100644 --- a/PaperDC/prioranalysis.jl +++ b/PaperDC/prioranalysis.jl @@ -1,10 +1,3 @@ -# # Little LSP hack to get function signatures, go #src -# # to definition etc. #src -# if isdefined(@__MODULE__, :LanguageServer) #src -# include("../src/IncompressibleNavierStokes.jl") #src -# using .IncompressibleNavierStokes #src -# end #src - using GLMakie using CairoMakie using IncompressibleNavierStokes diff --git a/docs/make.jl b/docs/make.jl index ff5b6fef6..d3ab79201 100644 --- a/docs/make.jl +++ b/docs/make.jl @@ -1,10 +1,3 @@ -# LSP indexing solution -# https://github.com/julia-vscode/julia-vscode/issues/800#issuecomment-650085983 -if isdefined(@__MODULE__, :LanguageServer) - include("../src/IncompressibleNavierStokes.jl") - using .IncompressibleNavierStokes -end - @info "" Threads.nthreads() using IncompressibleNavierStokes diff --git a/examples/Actuator2D.jl b/examples/Actuator2D.jl index b0a722efa..4a5182383 100644 --- a/examples/Actuator2D.jl +++ b/examples/Actuator2D.jl @@ -1,10 +1,3 @@ -# Little LSP hack to get function signatures, go #src -# to definition etc. #src -if isdefined(@__MODULE__, :LanguageServer) #src - include("../src/IncompressibleNavierStokes.jl") #src - using .IncompressibleNavierStokes #src -end #src - # # Unsteady actuator case - 2D # # In this example, an unsteady inlet velocity profile at encounters a wind diff --git a/examples/Actuator3D.jl b/examples/Actuator3D.jl index 32f3a13bd..afb65984d 100644 --- a/examples/Actuator3D.jl +++ b/examples/Actuator3D.jl @@ -1,10 +1,3 @@ -# Little LSP hack to get function signatures, go #src -# to definition etc. #src -if isdefined(@__MODULE__, :LanguageServer) #src - include("../src/IncompressibleNavierStokes.jl") #src - using .IncompressibleNavierStokes #src -end #src - # # Unsteady actuator case - 3D # # In this example, an unsteady inlet velocity profile at encounters a wind diff --git a/examples/BackwardFacingStep2D.jl b/examples/BackwardFacingStep2D.jl index 6c0f545fc..528c7b12d 100644 --- a/examples/BackwardFacingStep2D.jl +++ b/examples/BackwardFacingStep2D.jl @@ -1,10 +1,3 @@ -# Little LSP hack to get function signatures, go #src -# to definition etc. #src -if isdefined(@__MODULE__, :LanguageServer) #src - include("../src/IncompressibleNavierStokes.jl") #src - using .IncompressibleNavierStokes #src -end #src - # # Backward Facing Step - 2D # # In this example we consider a channel with walls at the top and bottom, and a diff --git a/examples/BackwardFacingStep3D.jl b/examples/BackwardFacingStep3D.jl index 3cf2376d2..f9deb774f 100644 --- a/examples/BackwardFacingStep3D.jl +++ b/examples/BackwardFacingStep3D.jl @@ -1,10 +1,3 @@ -# Little LSP hack to get function signatures, go #src -# to definition etc. #src -if isdefined(@__MODULE__, :LanguageServer) #src - include("../src/IncompressibleNavierStokes.jl") #src - using .IncompressibleNavierStokes #src -end #src - # # Backward Facing Step - 3D # # In this example we consider a channel with periodic side boundaries, walls at diff --git a/examples/DecayingTurbulence2D.jl b/examples/DecayingTurbulence2D.jl index d242819f3..acb247d73 100644 --- a/examples/DecayingTurbulence2D.jl +++ b/examples/DecayingTurbulence2D.jl @@ -1,10 +1,3 @@ -# Little LSP hack to get function signatures, go #src -# to definition etc. #src -if isdefined(@__MODULE__, :LanguageServer) #src - include("../src/IncompressibleNavierStokes.jl") #src - using .IncompressibleNavierStokes #src -end #src - # # Decaying Homogeneous Isotropic Turbulence - 2D # # In this example we consider decaying homogeneous isotropic turbulence, diff --git a/examples/DecayingTurbulence3D.jl b/examples/DecayingTurbulence3D.jl index e24155f42..1f7e0deb5 100644 --- a/examples/DecayingTurbulence3D.jl +++ b/examples/DecayingTurbulence3D.jl @@ -1,10 +1,3 @@ -# Little LSP hack to get function signatures, go #src -# to definition etc. #src -if isdefined(@__MODULE__, :LanguageServer) #src - include("../src/IncompressibleNavierStokes.jl") #src - using .IncompressibleNavierStokes #src -end #src - # # Decaying Homogeneous Isotropic Turbulence - 3D # # In this example we consider decaying homogeneous isotropic turbulence, diff --git a/examples/LidDrivenCavity2D.jl b/examples/LidDrivenCavity2D.jl index 54bd31f73..aec3a2898 100644 --- a/examples/LidDrivenCavity2D.jl +++ b/examples/LidDrivenCavity2D.jl @@ -1,10 +1,3 @@ -# Little LSP hack to get function signatures, go #src -# to definition etc. #src -if isdefined(@__MODULE__, :LanguageServer) #src - include("../src/IncompressibleNavierStokes.jl") #src - using .IncompressibleNavierStokes #src -end #src - # # Tutorial: Lid-Driven Cavity - 2D # # In this example we consider a box with a moving lid. The velocity is diff --git a/examples/LidDrivenCavity3D.jl b/examples/LidDrivenCavity3D.jl index af444f929..8605fc9bd 100644 --- a/examples/LidDrivenCavity3D.jl +++ b/examples/LidDrivenCavity3D.jl @@ -1,10 +1,3 @@ -# Little LSP hack to get function signatures, go #src -# to definition etc. #src -if isdefined(@__MODULE__, :LanguageServer) #src - include("../src/IncompressibleNavierStokes.jl") #src - using .IncompressibleNavierStokes #src -end #src - # # Lid-Driven Cavity - 3D # # In this example we consider a box with a moving lid. The velocity is initially at rest. The diff --git a/examples/MultiActuator.jl b/examples/MultiActuator.jl index bf6bce877..003c33d2d 100644 --- a/examples/MultiActuator.jl +++ b/examples/MultiActuator.jl @@ -1,10 +1,3 @@ -# Little LSP hack to get function signatures, go #src -# to definition etc. #src -if isdefined(@__MODULE__, :LanguageServer) #src - include("../src/IncompressibleNavierStokes.jl") #src - using .IncompressibleNavierStokes #src -end #src - # # Unsteady actuator case - 2D # # In this example, an unsteady inlet velocity profile at encounters a wind diff --git a/examples/PlanarMixing2D.jl b/examples/PlanarMixing2D.jl index 26b2ea446..4b3558300 100644 --- a/examples/PlanarMixing2D.jl +++ b/examples/PlanarMixing2D.jl @@ -1,10 +1,3 @@ -# Little LSP hack to get function signatures, go #src -# to definition etc. #src -if isdefined(@__MODULE__, :LanguageServer) #src - include("../src/IncompressibleNavierStokes.jl") #src - using .IncompressibleNavierStokes #src -end #src - # # Planar mixing - 2D # # Planar mixing example, as presented in [List2022](@cite). diff --git a/examples/PlaneJets2D.jl b/examples/PlaneJets2D.jl index d547b98fd..c4d83cb98 100644 --- a/examples/PlaneJets2D.jl +++ b/examples/PlaneJets2D.jl @@ -1,10 +1,3 @@ -# Little LSP hack to get function signatures, go #src -# to definition etc. #src -if isdefined(@__MODULE__, :LanguageServer) #src - include("../src/IncompressibleNavierStokes.jl") #src - using .IncompressibleNavierStokes #src -end #src - # # Plane jets - 2D # # Plane jets example, as presented in [MacArt2021](@cite). Note that the diff --git a/examples/ShearLayer2D.jl b/examples/ShearLayer2D.jl index cde432ee2..746a037b3 100644 --- a/examples/ShearLayer2D.jl +++ b/examples/ShearLayer2D.jl @@ -1,10 +1,3 @@ -# Little LSP hack to get function signatures, go #src -# to definition etc. #src -if isdefined(@__MODULE__, :LanguageServer) #src - include("../src/IncompressibleNavierStokes.jl") #src - using .IncompressibleNavierStokes #src -end #src - # # Shear layer - 2D # # Shear layer example. diff --git a/examples/TaylorGreenVortex2D.jl b/examples/TaylorGreenVortex2D.jl index 41153fd3c..e8ef0e863 100644 --- a/examples/TaylorGreenVortex2D.jl +++ b/examples/TaylorGreenVortex2D.jl @@ -1,10 +1,3 @@ -# Little LSP hack to get function signatures, go #src -# to definition etc. #src -if isdefined(@__MODULE__, :LanguageServer) #src - include("../src/IncompressibleNavierStokes.jl") #src - using .IncompressibleNavierStokes #src -end #src - # # Convergence study: Taylor-Green vortex (2D) # # In this example we consider the Taylor-Green vortex. diff --git a/examples/TaylorGreenVortex3D.jl b/examples/TaylorGreenVortex3D.jl index 2989530cc..44cf7cdd5 100644 --- a/examples/TaylorGreenVortex3D.jl +++ b/examples/TaylorGreenVortex3D.jl @@ -1,10 +1,3 @@ -# Little LSP hack to get function signatures, go #src -# to definition etc. #src -if isdefined(@__MODULE__, :LanguageServer) #src - include("../src/IncompressibleNavierStokes.jl") #src - using .IncompressibleNavierStokes #src -end #src - # # Taylor-Green vortex - 3D # # In this example we consider the Taylor-Green vortex. diff --git a/test/runtests.jl b/test/runtests.jl index ff2da57f8..f878eb785 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -1,10 +1,3 @@ -# LSP indexing solution -# https://github.com/julia-vscode/julia-vscode/issues/800#issuecomment-650085983 -if isdefined(@__MODULE__, :LanguageServer) - include("../src/IncompressibleNavierStokes.jl") - using .IncompressibleNavierStokes -end - using Aqua using CairoMakie using IncompressibleNavierStokes From d6514e22a85824e09e14debfa57654a22157aea3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Fri, 19 Apr 2024 17:10:19 +0200 Subject: [PATCH 279/379] Update docs environment --- docs/Project.toml | 4 ---- docs/make.jl | 4 ++++ 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/Project.toml b/docs/Project.toml index d3322b939..fe265ad8d 100644 --- a/docs/Project.toml +++ b/docs/Project.toml @@ -1,9 +1,5 @@ [deps] -CairoMakie = "13f3f980-e62b-5c42-98c6-ff1f3baf88f0" Documenter = "e30172f5-a6a5-5a46-863b-614d45cd2de4" DocumenterCitations = "daee34ce-89f3-4625-b898-19384cb65244" -FFTW = "7a1cc6ca-52ef-59f5-83cd-3a7055c09341" -GLMakie = "e9467ef8-e4e7-5192-8a1a-b1aee30e663a" IncompressibleNavierStokes = "5e318141-6589-402b-868d-77d7df8c442e" -LaTeXStrings = "b964fa9f-0449-5b57-a5c2-d3ea65f4040f" Literate = "98b081ad-f1c9-55d3-8b20-4c87d4299306" diff --git a/docs/make.jl b/docs/make.jl index d3ab79201..3d17fd607 100644 --- a/docs/make.jl +++ b/docs/make.jl @@ -1,5 +1,9 @@ +# Check number of threads aviailable on GitHub Actions @info "" Threads.nthreads() +# Load examples environment +push!(LOAD_PATH, joinpath(@__DIR__, "..", "examples")) + using IncompressibleNavierStokes using Literate using Documenter From 062ec9a65b107b84af91087315a9a009a4b09a2b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Fri, 19 Apr 2024 17:19:31 +0200 Subject: [PATCH 280/379] chore: Format files --- PaperDC/postanalysis.jl | 111 +++++++++++++++-------------- examples/MultiActuator.jl | 2 +- libs/NeuralClosure/src/training.jl | 8 +-- scratch/testgrad.jl | 8 +-- src/boundary_conditions.jl | 18 +---- src/create_initial_conditions.jl | 2 +- src/solvers/pressure/poisson.jl | 2 +- src/solvers/pressure/solvers.jl | 8 ++- 8 files changed, 71 insertions(+), 88 deletions(-) diff --git a/PaperDC/postanalysis.jl b/PaperDC/postanalysis.jl index cc520e8aa..0dc8ef23a 100644 --- a/PaperDC/postanalysis.jl +++ b/PaperDC/postanalysis.jl @@ -109,7 +109,7 @@ jldsave("$outdir/data_test.jld2"; data_test) # Load filtered DNS data data_train = load("$outdir/data_train.jld2", "data_train"); data_valid = load("$outdir/data_valid.jld2", "data_valid"); -data_test = load("$outdir/data_test.jld2", "data_test"); +data_test = load("$outdir/data_test.jld2", "data_test"); data_train[5].comptime data_valid[1].comptime @@ -530,7 +530,13 @@ e_smag e_cnn e_cnn_post -round.([e_nm[:] reshape(e_smag, :, 2) reshape(e_cnn, :, 2) reshape(e_cnn_post, :, 2)][[1:3; 6:8], :]; sigdigits = 2) +round.( + [e_nm[:] reshape(e_smag, :, 2) reshape(e_cnn, :, 2) reshape(e_cnn_post, :, 2)][ + [1:3; 6:8], + :, + ]; + sigdigits = 2, +) data_train[1].t[2] - data_train[1].t[1] data_test.t[2] - data_test.t[1] @@ -561,18 +567,18 @@ fig = with_theme(; ) linestyle = :solid # for ifil = 1:2 - # linestyle = ifil == 1 ? :solid : :dash - label = "No closure" - # label = label * (ifil == 1 ? " (FA)" : " (VA)") - # ifil == 2 && (label = nothing) - scatterlines!( - nles, - ones(T, length(nles)); - color = Cycled(1), - linestyle, - marker = :circle, - label, - ) + # linestyle = ifil == 1 ? :solid : :dash + label = "No closure" + # label = label * (ifil == 1 ? " (FA)" : " (VA)") + # ifil == 2 && (label = nothing) + scatterlines!( + nles, + ones(T, length(nles)); + color = Cycled(1), + linestyle, + marker = :circle, + label, + ) # end # for ifil = 1:2 # linestyle = ifil == 1 ? :solid : :dash @@ -589,46 +595,46 @@ fig = with_theme(; # ) # end # for ifil = 1:2 - # linestyle = ifil == 1 ? :solid : :dash - label = "CNN (Lprior)" - # label = label * (ifil == 1 ? " (FA)" : " (VA)") - # ifil == 2 && (label = nothing) - scatterlines!( - nles, - eprior.prior[:, ifil]; - color = Cycled(2), - linestyle, - marker = :utriangle, - label, - ) + # linestyle = ifil == 1 ? :solid : :dash + label = "CNN (Lprior)" + # label = label * (ifil == 1 ? " (FA)" : " (VA)") + # ifil == 2 && (label = nothing) + scatterlines!( + nles, + eprior.prior[:, ifil]; + color = Cycled(2), + linestyle, + marker = :utriangle, + label, + ) # end # for ifil = 1:2 - # linestyle = ifil == 1 ? :solid : :dash - label = "CNN (Lpost, DIF)" - # label = label * (ifil == 1 ? " (FA)" : " (VA)") - # ifil == 2 && (label = nothing) - scatterlines!( - nles, - eprior.post[:, ifil, 1]; - color = Cycled(3), - linestyle, - marker = :rect, - label, - ) + # linestyle = ifil == 1 ? :solid : :dash + label = "CNN (Lpost, DIF)" + # label = label * (ifil == 1 ? " (FA)" : " (VA)") + # ifil == 2 && (label = nothing) + scatterlines!( + nles, + eprior.post[:, ifil, 1]; + color = Cycled(3), + linestyle, + marker = :rect, + label, + ) # end # for ifil = 1:2 - # linestyle = ifil == 1 ? :solid : :dash - label = "CNN (Lpost, DCF)" - # label = label * (ifil == 1 ? " (FA)" : " (VA)") - # ifil == 2 && (label = nothing) - scatterlines!( - nles, - eprior.post[:, ifil, 2]; - color = Cycled(4), - linestyle, - marker = :diamond, - label, - ) + # linestyle = ifil == 1 ? :solid : :dash + label = "CNN (Lpost, DCF)" + # label = label * (ifil == 1 ? " (FA)" : " (VA)") + # ifil == 2 && (label = nothing) + scatterlines!( + nles, + eprior.post[:, ifil, 2]; + color = Cycled(4), + linestyle, + marker = :diamond, + label, + ) # end # lines!( # collect(extrema(nles[4:end])), @@ -1303,10 +1309,7 @@ clean(); GLMakie.activate!() -with_theme(; - fontsize = 25, - palette, -) do +with_theme(; fontsize = 25, palette) do x1 = 0.3 x2 = 0.5 y1 = 0.5 diff --git a/examples/MultiActuator.jl b/examples/MultiActuator.jl index 003c33d2d..94da3befb 100644 --- a/examples/MultiActuator.jl +++ b/examples/MultiActuator.jl @@ -138,7 +138,7 @@ state, outputs = solve_unsteady( ), boxplotter = processor() do state for box in boxes - lines!(current_axis(), box; color = :red); + lines!(current_axis(), box; color = :red) end end, ## ehist = realtimeplotter(; setup, plot = energy_history_plot, nupdate = 1), diff --git a/libs/NeuralClosure/src/training.jl b/libs/NeuralClosure/src/training.jl index b1b91ebe0..56532f599 100644 --- a/libs/NeuralClosure/src/training.jl +++ b/libs/NeuralClosure/src/training.jl @@ -149,7 +149,7 @@ function create_relerr_post(; for α = 1:D # a += sum(abs2, (stepper.u[α]-u[it][α])[Iu[α]]) # b += sum(abs2, u[it][α][Iu[α]]) - a += sum(abs2, view(stepper.u[α]-u[it][α], Iu[α])) + a += sum(abs2, view(stepper.u[α] - u[it][α], Iu[α])) b += sum(abs2, view(u[it][α], Iu[α])) end e += sqrt(a) / sqrt(b) @@ -178,11 +178,7 @@ If not using interactive GLMakie window, set `display_each_iteration` to function create_callback( err; θ, - callbackstate = (; - θmin = θ, - emin = eltype(θ)(Inf), - hist = Point2f[], - ), + callbackstate = (; θmin = θ, emin = eltype(θ)(Inf), hist = Point2f[]), displayref = true, display_each_iteration = false, ) diff --git a/scratch/testgrad.jl b/scratch/testgrad.jl index 1d9826e4e..bf31744d8 100644 --- a/scratch/testgrad.jl +++ b/scratch/testgrad.jl @@ -146,13 +146,7 @@ finitediff(u -> f(u, setup), u, α, I) function f(u, setup) (; Ω, Iu) = setup.grid method = RK44(; T) - stepper = IncompressibleNavierStokes.create_stepper( - method; - setup, - psolver, - u, - t = T(0), - ) + stepper = IncompressibleNavierStokes.create_stepper(method; setup, psolver, u, t = T(0)) stepper = IncompressibleNavierStokes.timestep(method, stepper, T(1e-4)) φ = stepper.u # dot(φ, φ) diff --git a/src/boundary_conditions.jl b/src/boundary_conditions.jl index 75d4cf383..ff1efb632 100644 --- a/src/boundary_conditions.jl +++ b/src/boundary_conditions.jl @@ -183,22 +183,8 @@ function apply_bc_p_pullback!(φbar, t, setup; kwargs...) (; dimension) = grid D = dimension() for β = 1:D - apply_bc_p_pullback!( - boundary_conditions[β][1], - φbar, - β, - t, - setup; - atend = false, - ) - apply_bc_p_pullback!( - boundary_conditions[β][2], - φbar, - β, - t, - setup; - atend = true, - ) + apply_bc_p_pullback!(boundary_conditions[β][1], φbar, β, t, setup; atend = false) + apply_bc_p_pullback!(boundary_conditions[β][2], φbar, β, t, setup; atend = true) end φbar end diff --git a/src/create_initial_conditions.jl b/src/create_initial_conditions.jl index e0446c8de..e3d42ef86 100644 --- a/src/create_initial_conditions.jl +++ b/src/create_initial_conditions.jl @@ -148,7 +148,7 @@ function create_spectrum(; setup, kp) ke = sum(α -> e[α] .* kkkk[α], 1:D) CUDA.@allowscalar e0 = getindex.(e, 1) for α = 1:D - @. e[α] -= kkkk[α] * ke / knorm ^ 2 + @. e[α] -= kkkk[α] * ke / knorm^2 end # Restore k=0 component, which is divergence free anyways diff --git a/src/solvers/pressure/poisson.jl b/src/solvers/pressure/poisson.jl index 5011456b5..40447347e 100644 --- a/src/solvers/pressure/poisson.jl +++ b/src/solvers/pressure/poisson.jl @@ -196,7 +196,7 @@ function poisson!(solver::LowMemorySpectralPressureSolver, p, f) # Solve for coefficients in Fourier space if D == 2 - ax = ahat + ax = ahat ay = reshape(ahat, 1, :) @. phat = -phat / (ax + ay) else diff --git a/src/solvers/pressure/solvers.jl b/src/solvers/pressure/solvers.jl index 756631167..d4d738863 100644 --- a/src/solvers/pressure/solvers.jl +++ b/src/solvers/pressure/solvers.jl @@ -101,7 +101,7 @@ struct CUDSSPressureSolver{T,S,F,A} <: AbstractPressureSolver{T} L, "S", # Symmetric (not positive definite) 'L', # Lower triangular representation - ) + ) cudss("analysis", solver, p, f) cudss("factorization", solver, p, f) # Compute factorization new{T,typeof(setup),typeof(solver),typeof(f)}(setup, solver, f, p) @@ -344,5 +344,9 @@ function LowMemorySpectralPressureSolver(setup) # Placeholders for intermediate results phat = fill!(similar(x[1], Complex{T}, Np), 0) - LowMemorySpectralPressureSolver{T,typeof(ahat),typeof(phat),typeof(setup)}(setup, ahat, phat) + LowMemorySpectralPressureSolver{T,typeof(ahat),typeof(phat),typeof(setup)}( + setup, + ahat, + phat, + ) end From 0f3578f3337f6c798cae4a25e8d2ccb950b9af46 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Fri, 19 Apr 2024 17:23:57 +0200 Subject: [PATCH 281/379] Add README --- examples/README.md | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 examples/README.md diff --git a/examples/README.md b/examples/README.md new file mode 100644 index 000000000..c736e914d --- /dev/null +++ b/examples/README.md @@ -0,0 +1,25 @@ +# Examples + +Example scripts for +[IncompressibleNavierStokes.jl](https://github.com/agdestein/IncompressibleNavierStokes.jl). + +## Set up environment + +From this directory, run: + +```sh +julia --project -e ' +using Pkg +Pkg.develop(PackageSpec(; path = "..")) +Pkg.instantiate()' +' +``` + +or interactively from a Julia REPL: + +```julia-repl +julia> ] +(v1.10) pkg> activate . +(examples) pkg> dev .. +(examples) pkg> instantiate +``` From 127eabe959f5200a19fedb532fee8f361c46b4d1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Fri, 19 Apr 2024 23:42:04 +0200 Subject: [PATCH 282/379] Add missing package --- Project.toml | 2 ++ libs/NeuralClosure/src/NeuralClosure.jl | 2 ++ src/IncompressibleNavierStokes.jl | 1 + 3 files changed, 5 insertions(+) diff --git a/Project.toml b/Project.toml index b53adce4d..655ff39d5 100644 --- a/Project.toml +++ b/Project.toml @@ -13,6 +13,7 @@ IterativeSolvers = "42fd0dbc-a981-5370-80f2-aaf504508153" KernelAbstractions = "63c18a36-062a-441e-b654-da1e3ab1ce7c" LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" Makie = "ee78f7c6-11fb-53f2-987a-cfe4a2b5a57a" +NNlib = "872c559c-99b0-510c-b3b7-b6c96a88d5cd" Observables = "510215fc-4207-5dde-b226-833fc4488ee2" Printf = "de0858da-6303-5e67-8744-51eddeeeb8d7" Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" @@ -30,6 +31,7 @@ IterativeSolvers = "0.9" KernelAbstractions = "0.9" LinearAlgebra = "1" Makie = "0.20" +NNlib = "0.9" Observables = "0.5" Printf = "1" Random = "1" diff --git a/libs/NeuralClosure/src/NeuralClosure.jl b/libs/NeuralClosure/src/NeuralClosure.jl index 083c173db..ae4c44fdb 100644 --- a/libs/NeuralClosure/src/NeuralClosure.jl +++ b/libs/NeuralClosure/src/NeuralClosure.jl @@ -5,6 +5,8 @@ module NeuralClosure using CUDA using ComponentArrays: ComponentArray +using IncompressibleNavierStokes +using IncompressibleNavierStokes: Dimension using Lux using NNlib using Random diff --git a/src/IncompressibleNavierStokes.jl b/src/IncompressibleNavierStokes.jl index c5b84b3f2..cdb176749 100644 --- a/src/IncompressibleNavierStokes.jl +++ b/src/IncompressibleNavierStokes.jl @@ -12,6 +12,7 @@ using IterativeSolvers using KernelAbstractions using LinearAlgebra using Makie +using NNlib using Printf using Random using SparseArrays From ce6965058d348b686436e917660e2cd773d79e39 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Fri, 19 Apr 2024 23:44:19 +0200 Subject: [PATCH 283/379] Fix script --- examples/MultiActuator.jl | 84 +++++++++++---------------------------- 1 file changed, 23 insertions(+), 61 deletions(-) diff --git a/examples/MultiActuator.jl b/examples/MultiActuator.jl index 94da3befb..d56eebf04 100644 --- a/examples/MultiActuator.jl +++ b/examples/MultiActuator.jl @@ -7,11 +7,10 @@ #md using CairoMakie using GLMakie #!md using IncompressibleNavierStokes -using Adapt using Random # Output directory -output = "output/Actuator2D" +output = "output/MultiActuator" # Floating point precision T = Float64 @@ -28,7 +27,6 @@ T = Float32; # T = Float64; ArrayType = CuArray; CUDA.allowscalar(false); -device = x -> adapt(ArrayType, x); set_theme!(; GLMakie = (; scalefactor = 1.5)) @@ -66,14 +64,6 @@ create_manyforce(forces...) = function (dim, x, y, t) out end -bodyforce = create_bodyforce(; - xc = T(2), # Disk center - yc = T(0), # Disk center - D = T(1), # Disk diameter - δ = T(0.11), # Disk thickness - C = T(0.2), # Thrust coefficient -) - disk = (; D = T(1), δ = T(0.11), C = T(0.2)) bodyforce = create_manyforce( create_bodyforce(; xc = T(2), yc = T(0), disk...), @@ -82,42 +72,41 @@ bodyforce = create_manyforce( ) # A 2D grid is a Cartesian product of two vectors -n = 160 +n = 80 x = LinRange(T(0), T(10), 5n + 1) y = LinRange(-T(2), T(2), 2n + 1) # Build setup and assemble operators setup = Setup(x, y; Re = T(2000), boundary_conditions, bodyforce, ArrayType); -using CUDA.CUSPARSE -using CUDSS -using LinearAlgebra - psolver = CUDSSPressureSolver(setup); -psolver2 = DirectPressureSolver(setup); - -p = fill!(similar(setup.grid.x[1], setup.grid.N), 0) -f = similar(setup.grid.x[1], setup.grid.N) -f .= randn.(T) -IncompressibleNavierStokes.apply_bc_p!(f, T(0), setup) - -IncompressibleNavierStokes.poisson!(psolver, p, f) -IncompressibleNavierStokes.poisson!(psolver2, p, f) - -using KernelAbstractions -using BenchmarkTools - -@benchmark begin - # IncompressibleNavierStokes.poisson!(psolver, p, f) - IncompressibleNavierStokes.poisson!(psolver2, p, f) - KernelAbstractions.synchronize(get_backend(p)) -end # Initial conditions (extend inflow) u₀ = create_initial_conditions(setup, (dim, x, y) -> dim() == 1 ? one(x) : zero(x); psolver); u = u₀ t = T(0) +# # We create a box to visualize the actuator. +# (; xc, yc, D, δ) = setup.bodyforce +# box = [ +# Point2f(xc - δ / 2, yc + D / 2), +# Point2f(xc - δ / 2, yc - D / 2), +# Point2f(xc + δ / 2, yc - D / 2), +# Point2f(xc + δ / 2, yc + D / 2), +# Point2f(xc - δ / 2, yc + D / 2), +# ] + +boxes = map(bodyforce.forces) do (; xc, yc, D, δ) + [ + Point2f(xc - δ / 2, yc + D / 2), + Point2f(xc - δ / 2, yc - D / 2), + Point2f(xc + δ / 2, yc - D / 2), + Point2f(xc + δ / 2, yc + D / 2), + Point2f(xc - δ / 2, yc + D / 2), + ] +end +box = boxes[1] + # Solve unsteady problem state, outputs = solve_unsteady( setup, @@ -157,45 +146,18 @@ state, outputs = solve_unsteady( # Export to VTK save_vtk(setup, state.u, state.p, "$output/solution") -# We create a box to visualize the actuator. -(; xc, yc, D, δ) = setup.bodyforce -box = [ - Point2f(xc - δ / 2, yc + D / 2), - Point2f(xc - δ / 2, yc - D / 2), - Point2f(xc + δ / 2, yc - D / 2), - Point2f(xc + δ / 2, yc + D / 2), - Point2f(xc - δ / 2, yc + D / 2), -] - -box = boxes[1] -boxes = map(bodyforce.forces) do (; xc, yc, D, δ) - [ - Point2f(xc - δ / 2, yc + D / 2), - Point2f(xc - δ / 2, yc - D / 2), - Point2f(xc + δ / 2, yc - D / 2), - Point2f(xc + δ / 2, yc + D / 2), - Point2f(xc - δ / 2, yc + D / 2), - ] -end - -state = (; u = F, t) - # Plot pressure fig = fieldplot(state; setup, fieldname = :pressure, psolver) # lines!(box...; color = :red) lines!.(boxes; color = :red); fig -sum(IncompressibleNavierStokes.pressure(u, t, setup; psolver)) - # Plot velocity fig = fieldplot(state; setup, fieldname = :velocity) -# lines!(box...; color = :red) lines!.(boxes; color = :red); fig # Plot vorticity fig = fieldplot(state; setup, fieldname = :vorticity) -# lines!(box...; color = :red) lines!.(boxes; color = :red); fig From fbdd8ad548eaf22435abd8345ce6f423bc6a563e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Fri, 19 Apr 2024 23:44:32 +0200 Subject: [PATCH 284/379] Add missing rng --- src/create_initial_conditions.jl | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/src/create_initial_conditions.jl b/src/create_initial_conditions.jl index e3d42ef86..894bc38a6 100644 --- a/src/create_initial_conditions.jl +++ b/src/create_initial_conditions.jl @@ -47,7 +47,7 @@ function create_initial_conditions( u end -# function create_spectrum(; setup, A, σ, s) +# function create_spectrum(; setup, A, σ, s, rng = Random.default_rng()) # (; dimension, x, N) = setup.grid # T = eltype(x[1]) # D = dimension() @@ -63,14 +63,14 @@ end # kα = k[α] # @. a *= exp(-max(abs(kα) - s, 0)^2 / 2σ^2) # end -# @. a *= randn(T) * exp(im * τ * rand(T)) +# @. a *= randn(rng, T) * exp(im * τ * rand(rng, T)) # for α = 1:D # a = cat(a, reverse(a; dims = α); dims = α) # end # a # end -function create_spectrum(; setup, kp) +function create_spectrum(; setup, kp, rng = Random.default_rng()) (; dimension, x, N) = setup.grid T = eltype(x[1]) D = dimension() @@ -106,7 +106,7 @@ function create_spectrum(; setup, kp) a .*= prod(N) # Apply random phase shift - ξ = ntuple(α -> rand!(similar(x[1], K)), D) + ξ = ntuple(α -> rand!(rng, similar(x[1], K)), D) for α = 1:D a = cat(a, reverse(a; dims = α); dims = α) ξ = ntuple(D) do β @@ -136,11 +136,11 @@ function create_spectrum(; setup, kp) # Create random unit vector for each wavenumber if D == 2 - θ = rand!(similar(x[1], KK)) + θ = rand!(rng, similar(x[1], KK)) e = (cospi.(2 .* θ), sinpi.(2 .* θ)) elseif D == 3 - θ = rand!(similar(x[1], KK)) - ϕ = rand!(similar(x[1], KK)) + θ = rand!(rng, similar(x[1], KK)) + ϕ = rand!(rng, similar(x[1], KK)) e = (sinpi.(θ) .* cospi.(2 .* ϕ), sinpi.(θ) .* sinpi.(2 .* ϕ), cospi.(θ)) end @@ -176,6 +176,7 @@ end A = 1, kp = 10, psolver = SpectralPressureSolver(setup), + rng = Random.default_rng(), ) Create random field, as in [Orlandi2000](@cite). @@ -190,14 +191,14 @@ function random_field( A = 1, kp = 10, psolver = SpectralPressureSolver(setup), + rng = Random.default_rng(), ) (; dimension, x, Ip, Ω) = setup.grid D = dimension() T = eltype(x[1]) - backend = get_backend(x[1]) # Create random velocity field - uhat = create_spectrum(; setup, kp) + uhat = create_spectrum(; setup, kp, rng) u = ifft.(uhat) u = map(u -> A .* real.(u), u) From e971cfd03e3348cadb5efe3269070cfc759c7f2b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Fri, 19 Apr 2024 23:45:48 +0200 Subject: [PATCH 285/379] Clean up script --- PaperDC/prioranalysis.jl | 558 ++++++++++++--------------------------- PaperDC/src/PaperDC.jl | 10 +- 2 files changed, 176 insertions(+), 392 deletions(-) diff --git a/PaperDC/prioranalysis.jl b/PaperDC/prioranalysis.jl index e2cb35b70..71f99b5ce 100644 --- a/PaperDC/prioranalysis.jl +++ b/PaperDC/prioranalysis.jl @@ -1,447 +1,248 @@ -using GLMakie using CairoMakie +using FFTW +using GLMakie using IncompressibleNavierStokes -using IncompressibleNavierStokes: - momentum, - momentum!, - divergence!, - project, - project!, - apply_bc_u!, - spectral_stuff, - kinetic_energy, - interpolate_u_p +using IncompressibleNavierStokes: momentum, project, apply_bc_u!, spectral_stuff using NeuralClosure -using Printf -using FFTW using PaperDC +using Printf +using Random -# Output directory -output = "output/divergence" -output = "../SupervisedClosure/figures" +# Put output in PaperDC/output +cd(@__DIR__) +output = "output" -# Array type +# For running on CPU ArrayType = Array -# using CUDA; ArrayType = CuArray; -## using AMDGPU; ArrayType = ROCArray -## using oneAPI; ArrayType = oneArray -## using Metal; ArrayType = MtlArray +clean() = nothing +# For running on GPU using CUDA; -ArrayType = CuArray; CUDA.allowscalar(false); +ArrayType = CuArray; +clean() = (GC.gc(); CUDA.reclaim()) # This seems to be needed to free up memory -set_theme!() -set_theme!(; GLMakie = (; scalefactor = 1.5)) - -# 3D -T = Float32 -Re = T(2_000) -ndns = 512 -D = 3 -kp = 5 -Δt = T(1e-4) +# 2D setup +D = 2 +T = Float64; +ndns = 4096 +Re = T(10_000) +kp = 20 +Δt = T(5e-5) filterdefs = [ - (FaceAverage(), 32), (FaceAverage(), 64), (FaceAverage(), 128), - (VolumeAverage(), 32), + (FaceAverage(), 256), (VolumeAverage(), 64), (VolumeAverage(), 128), + (VolumeAverage(), 256), ] -# 2D -T = Float64; -Re = T(10_000) -ndns = 1024 -D = 2 -kp = 20 -Δt = T(5e-5) +# 3D setup +D = 3 +T, ndns = Float64, 512 # Works on a 40GB A100 GPU. Use Float32 and smaller n for less memory. +Re = T(2_000) +kp = 5 +Δt = T(1e-4) filterdefs = [ + (FaceAverage(), 32), (FaceAverage(), 64), (FaceAverage(), 128), - (FaceAverage(), 256), + (VolumeAverage(), 32), (VolumeAverage(), 64), (VolumeAverage(), 128), - (VolumeAverage(), 256), ] # Setup lims = T(0), T(1) -dns = Setup(ntuple(α -> LinRange(lims..., ndns + 1), D)...; Re, ArrayType); +dns = let + setup = Setup(ntuple(α -> LinRange(lims..., ndns + 1), D)...; Re, ArrayType) + psolver = SpectralPressureSolver(setup) + (; setup, psolver) +end; filters = map(filterdefs) do (Φ, nles) compression = ndns ÷ nles setup = Setup(ntuple(α -> LinRange(lims..., nles + 1), D)...; Re, ArrayType) psolver = SpectralPressureSolver(setup) (; setup, Φ, compression, psolver) end; -psolver_dns = SpectralPressureSolver(dns); # Create random initial conditions -u₀ = random_field(dns, T(0); kp, psolver = psolver_dns); - -state = (; u = u₀, t = T(0)); - -GC.gc() -CUDA.reclaim() - -energy_spectrum_plot((; u = u₀, t = T(0)); setup = dns) - -fieldplot( - (; u = u₀, t = T(0)); - setup = dns, - # type = image, - # colormap = :viridis, -) +rng = Random.seed!(Random.default_rng(), 12345) +u₀ = random_field(dns.setup, T(0); kp, dns.psolver, rng); +clean() # Solve unsteady problem @time state, outputs = solve_unsteady( - dns, + dns.setup, u₀, - # state.u, (T(0), T(1e-1)); Δt, - # Δt = T(1e-4), - # Δt = T(1e-5), - docopy = true, - psolver = psolver_dns, + docopy = true, # leave initial conditions unchanged, false to free up memory + dns.psolver, processors = ( - # anim = animator(; path = "$output/solution.mkv", - # rtp = realtimeplotter(; - # setup = dns, - # nupdate = 50, - # fieldname = :eig2field, - # levels = LinRange(T(2), T(10), 10), - # # levels = 5, - # docolorbar = false, - # ), - obs = observe_u(dns, psolver_dns, filters; nupdate = 20), - # ehist = realtimeplotter(; - # setup, - # plot = energy_history_plot, - # nupdate = 10, - # displayfig = false, - # ), - # espec = realtimeplotter(; setup, plot = energy_spectrum_plot, nupdate = 10), - # anim = animator(; path = "$output/solution_Re$(Int(Re)).mkv", nupdate = 10, - # setup = dns, - # fieldname = :eig2field, - # levels = LinRange(T(2), T(10), 10), - # # levels = LinRange(T(4), T(12), 10), - # # levels = LinRange(-1.0f0, 3.0f0, 5), - # # levels = LinRange(-2.0f0, 2.0f0, 5), - # # levels = 5, - # docolorbar = false, - # # size = (800, 800), - # size = (600, 600), - # ), - # vtk = vtk_writer(; setup, nupdate = 10, dir = output, filename = "solution"), - # field = fieldsaver(; setup, nupdate = 10), + obs = observe_u(dns.setup, dns.psolver, filters; nupdate = 20), log = timelogger(; nupdate = 5), ), ); -GC.gc() -CUDA.reclaim() - -# 103.5320324 - -state.u - -state.u[2] - -fil = filters[2]; -apply_bc_u!(state.u, T(0), dns); -v = fil.Φ(state.u, fil.setup, fil.compression); -apply_bc_u!(v, T(0), fil.setup); -Fv = momentum(v, T(0), fil.setup); -apply_bc_u!(Fv, T(0), fil.setup); -PFv = project(Fv, fil.setup; psolver = fil.psolver); -apply_bc_u!(PFv, T(0), fil.setup); -F = momentum(state.u, T(0), dns); -apply_bc_u!(F, T(0), dns); -PF = project(F, dns; psolver = psolver_dns); -apply_bc_u!(PF, T(0), dns); -ΦPF = fil.Φ(PF, fil.setup, fil.compression); -apply_bc_u!(ΦPF, T(0), fil.setup); -c = ΦPF .- PFv -apply_bc_u!(c, T(0), fil.setup) - -with_theme(; fontsize = 25) do - fig = fieldplot( - (; u = u₀, t = T(0)); - setup = dns, - # type = image, - # colormap = :viridis, - docolorbar = false, - size = (500, 500), - title = "u₀", +clean() + +# Plot 2D fields ##################################################### + +D == 2 && with_theme(; fontsize = 25) do + # Compute quantities + fil = filters[2] + apply_bc_u!(state.u, T(0), dns.setup) + v = fil.Φ(state.u, fil.setup, fil.compression) + apply_bc_u!(v, T(0), fil.setup) + Fv = momentum(v, T(0), fil.setup) + apply_bc_u!(Fv, T(0), fil.setup) + PFv = project(Fv, fil.setup; psolver = fil.psolver) + apply_bc_u!(PFv, T(0), fil.setup) + F = momentum(state.u, T(0), dns.setup) + apply_bc_u!(F, T(0), dns.setup) + PF = project(F, dns.setup; dns.psolver) + apply_bc_u!(PF, T(0), dns.setup) + ΦPF = fil.Φ(PF, fil.setup, fil.compression) + apply_bc_u!(ΦPF, T(0), fil.setup) + c = ΦPF .- PFv + apply_bc_u!(c, T(0), fil.setup) + + # Make plots + path = "$output/priorfields" + ispath(path) || mkpath(path) + makeplot(field, setup, title, name) = save( + "$path/$name.png", + fieldplot( + (; u = field, t = T(0)); + setup, + title, + docolorbar = false, + size = (500, 500), + ), ) - save("$output/priorfields/ustart.png", fig) - fig = fieldplot( - state, - setup = dns, - # type = image, - # colormap = :viridis, - docolorbar = false, - size = (500, 500), - title = "u", - ) - save("$output/priorfields/u.png", fig) - fig = fieldplot( - (; u = v, t = T(0)); - fil.setup, - # type = image, - # colormap = :viridis, - # fieldname = 1, - docolorbar = false, - size = (500, 500), - # title = "ubar" - title = "ū", - ) - save("$output/priorfields/v.png", fig) - fig = fieldplot( - (; u = PF, t = T(0)); - setup = dns, - # type = image, - # colormap = :viridis, - # fieldname = 1, - docolorbar = false, - size = (500, 500), - title = "PF(u)", - ) - save("$output/priorfields/PFu.png", fig) - fig = fieldplot( - (; u = PFv, t = T(0)); - fil.setup, - # type = image, - # colormap = :viridis, - # fieldname = 1, - docolorbar = false, - size = (500, 500), - # title = "PF(ubar)" - title = "P̄F̄(ū)", - ) - save("$output/priorfields/PFv.png", fig) - fig = fieldplot( - (; u = ΦPF, t = T(0)); - fil.setup, - # type = image, - # colormap = :viridis, - # fieldname = 1, - docolorbar = false, - size = (500, 500), - title = "ΦPF(u)", - ) - save("$output/priorfields/PhiPFu.png", fig) - fig = fieldplot( - (; u = c, t = T(0)); - fil.setup, - # type = image, - # colormap = :viridis, - # fieldname = 1, - # fieldname = :velocity, - docolorbar = false, - size = (500, 500), - # title = "c(u, ubar)" - title = "c(u, ū)", - ) - save("$output/priorfields/c.png", fig) + makeplot(u₀, dns.setup, "u₀", "ustart") + makeplot(state.u, dns.setup, "u", "u") + makeplot(v, fil.setup, "ū", "v") + makeplot(PF, dns.setup, "PF(u)", "PFu") + makeplot(PFv, fil.setup, "P̄F̄(ū)", "PFv") + makeplot(ΦPF, fil.setup, "ΦPF(u)", "PhiPFu") + makeplot(c, fil.setup, "c(u, ū)", "c") end -#################################################################### - -fieldplot( - # (; u = u₀, t = T(0)); - state; - setup = dns, - fieldname = :eig2field, - # levels = LinRange(T(2), T(10), 10), - levels = LinRange(T(4), T(12), 10), - # levels = LinRange(-1.0f0, 3.0f0, 5), - # levels = LinRange(-2.0f0, 2.0f0, 5), - # levels = 5, - docolorbar = false, - # size = (800, 800), - size = (600, 600), -) - -fname = "$output/prioranalysis/lambda2/Re$(Int(Re))_start.png" -fname = "$output/prioranalysis/lambda2/Re$(Int(Re))_end.png" -save(fname, current_figure()) -run(`convert $fname -trim $fname`) # Requires imagemagick - -i = 3 -fieldplot( - (; u = filters[i].Φ(state.u, filters[i].setup, filters[i].compression), t = T(0)); - setup = filters[i].setup, - fieldname = :eig2field, - # levels = LinRange(T(2), T(10), 10), - levels = LinRange(T(4), T(12), 10), - # levels = LinRange(-1.0f0, 3.0f0, 5), - # levels = LinRange(-2.0f0, 2.0f0, 5), - # levels = 5, - docolorbar = false, - # size = (800, 800), - size = (600, 600), -) - -fname = "$output/prioranalysis/lambda2/Re$(Int(Re))_end_filtered.png" -save(fname, current_figure()) -run(`convert $fname -trim $fname`) # Requires imagemagick - -field = IncompressibleNavierStokes.eig2field(state.u, dns)[dns.grid.Ip] -hist(vec(Array(log.(max.(eps(T), .-field))))) -field = nothing - -i = 2 -field = IncompressibleNavierStokes.eig2field( - filters[i].Φ(state.u, filters[i].setup, filters[i].compression), - filters[i].setup, -)[filters[i].setup.grid.Ip] -hist(vec(Array(log.(max.(eps(T), .-field))))) -field = nothing - -energy_spectrum_plot(state; setup = dns) - -save("spectrum.png", current_figure()) - -i = 6 -ubar = filters[i].Φ(state.u, filters[i].setup, filters[i].compression) -energy_spectrum_plot((; u = ubar, t = T(0)); filters[i].setup) - -state.u - -# Float32, 1024^2: -# -# 5.711019 seconds (46.76 M allocations: 2.594 GiB, 4.59% gc time, 2.47% compilation time) -# 5.584943 seconds (46.60 M allocations: 2.583 GiB, 4.43% gc time) - -# Float64, 1024^2: -# -# 9.584393 seconds (46.67 M allocations: 2.601 GiB, 2.93% gc time) -# 9.672491 seconds (46.67 M allocations: 2.601 GiB, 2.93% gc time) - -# Float64, 4096^2: -# -# 114.006495 seconds (47.90 M allocations: 15.499 GiB, 0.28% gc time) -# 100.907239 seconds (46.45 M allocations: 2.588 GiB, 0.26% gc time) - -# Float32, 512^3: -# -# 788.762194 seconds (162.34 M allocations: 11.175 GiB, 0.12% gc time) - -9.584393 / 5.711019 -9.672491 / 5.584943 - -# 1.0 * nbyte(Float32) * N * α * (u0 + ui + k1,k2,k3,k4 + p + maybe(complexFFT(Lap)) + maybe(boundaryfree p)) -1.0 * 4 * 1024^3 * 3 * (1 + 1 + 4 + 1 / 3 + 0 * 1 * 2 + 0 * 1 / 3) # RK4: 81.6GB (111GB) -1.0 * 4 * 1024^3 * 3 * (1 + 0 + 1 + 1 / 3 + 0 * 1 * 2 + 0 * 1 / 3) # RK1: 30.0GB (60.1 GB) - -1.0 * 4 * 512^3 * 3 * (1 + 1 + 4 + 1 / 3 + 0 * 1 * 2 + 0 * 1 / 3) # RK4: 10.2GB (13.9GB) -1.0 * 4 * 512^3 * 3 * (1 + 0 + 1 + 1 / 3 + 1 * 1 * 2 + 1 * 1 / 3) # RK1: 3.76GB (7.52GB) +# Plot 3D fields ##################################################### -1.0 * 8 * 512^3 * 3 * (1 + 1 + 3 + 1 / 3 + 0 * 1 * 2 + 0 * 1 / 3) # RK4: 10.2GB (13.9GB) +# Contour plots in 3D only work with GLMakie. +# For using GLMakie on headless servers, see +# +GLMakie.activate!() -begin - println("Φ\t\tM\tDu\tPv\tPc\tc") - for o in outputs.obs - nt = length(o.t) - Dv = sum(o.Dv) / nt - Pc = sum(o.Pc) / nt - Pv = sum(o.Pv) / nt - c = sum(o.c) / nt - @printf( - "%s\t%d^%d\t%.2g\t%.2g\t%.2g\t%.2g\n", - # "%s &\t\$%d^%d\$ &\t\$%.2g\$ &\t\$%.2g\$ &\t\$%.2g\$ &\t\$%.2g\$\n", - typeof(o.Φ), - o.Mα, - D, - Dv, - Pv, - Pc, - c +# Make plots +D == 3 && with_theme() do + path = "$output/priorfields/lambda2" + ispath(path) || mkpath(path) + function makeplot(field, setup, name) + name = "$path/$name.png" + save( + name, + fieldplot( + (; u = field, t = T(0)); + setup, + fieldname = :eig2field, + levels = LinRange(T(4), T(12), 10), + docolorbar = false, + size = (600, 600), + ), ) + try + # Trim whitespace with ImageMagick + run(`convert $name -trim $name`) + catch e + @warn """ + ImageMagick not found. + Skipping image trimming. + Install from . + """ + end end -end; - -(; u, t) = state; - -o = outputs.obs[1] -o.Dv -o.Pc -o.Pv -o.c - -# apply_bc_u!(u, t, dns) -ubar = FaceAverage()(u, les, comp); -ubar = VolumeAverage()(u, les, comp); -# apply_bc_u!(ubar, t, les) -fieldplot((; u = ubar, t); setup = les) -fieldplot((; u, t); setup = dns) - -IncompressibleNavierStokes.apply_bc_u!(ubar, t, les) -div = IncompressibleNavierStokes.divergence(ubar, les)[les.grid.Ip] - -norm(div) -norm(ubar[1][les.grid.Iu[1]]) - -GLMakie.activate!() + makeplot(u₀, dns.setup, "Re$(Int(Re))_start") # Requires docopy = true in solve + makeplot(state.u, dns.setup, "Re$(Int(Re))_end") + i = 3 + makeplot( + filters[i].Φ(state.u, filters[i].setup, filters[i].compression), + filters[i].setup, + "Re$(Int(Re))_end_filtered", + ) +end -# To free up memory -psolver_dns = nothing -fig = lines([1, 2, 3]) -GC.gc() -CUDA.reclaim() +# Compute average quantities ######################################### + +let + path = "$output/prioranalysis" + ispath(path) || mkpath(path) + open("$path/averages_$(D)D.txt", "w") do io + println(io, "Φ\t\tM\tDu\tPv\tPc\tc") + for o in outputs.obs + nt = length(o.t) + Dv = sum(o.Dv) / nt + Pc = sum(o.Pc) / nt + Pv = sum(o.Pv) / nt + c = sum(o.c) / nt + @printf( + io, + "%s\t%d^%d\t%.2g\t%.2g\t%.2g\t%.2g\n", + # "%s &\t\$%d^%d\$ &\t\$%.2g\$ &\t\$%.2g\$ &\t\$%.2g\$ &\t\$%.2g\$\n", + typeof(o.Φ), + o.Mα, + D, + Dv, + Pv, + Pc, + c + ) + end + end +end -using CairoMakie -CairoMakie.activate!() +# Plot spectra ####################################################### +# To free up memory in 3D (remove SpectralPressureSolver FFT arrays) +dns = (; dns.setup) filters = map(filters) do f (; f.Φ, f.setup, f.compression) end +fig = lines([1, 2, 3]) +clean() # Plot predicted spectra -fig = with_theme(; palette = (; color = ["#3366cc", "#cc0000", "#669900", "#ffcc00"])) do +CairoMakie.activate!() +with_theme(; palette = (; color = ["#3366cc", "#cc0000", "#669900", "#ffcc00"])) do fields = [state.u, u₀, (f.Φ(state.u, f.setup, f.compression) for f in filters)...] - setups = [dns, dns, (f.setup for f in filters)...] + setups = [dns.setup, dns.setup, (f.setup for f in filters)...] specs = map(fields, setups) do u, setup - GC.gc() - CUDA.reclaim() + clean() # Free up memory (; dimension, xp, Ip) = setup.grid T = eltype(xp[1]) D = dimension() K = size(Ip) .÷ 2 - # up = interpolate_u_p(u, setup) up = u e = sum(up) do u u = u[Ip] uhat = fft(u)[ntuple(α -> 1:K[α], D)...] - # abs2.(uhat) abs2.(uhat) ./ (2 * prod(size(u))^2) - # abs2.(uhat) ./ size(u, 1) end - (; A, κ, K) = spectral_stuff(setup) - e = A * reshape(e, :) - # e = max.(e, eps(T)) # Avoid log(0) - ehat = Array(e) + (; A, κ, K) = spectral_stuff(setup) # Requires some memory + e = A * reshape(e, :) # Dyadic binning + ehat = Array(e) # Store spectrum on CPU (; κ, ehat) end kmax = maximum(specs[1].κ) # Build inertial slope above energy - if D == 2 - # krange = [T(kmax)^(T(1) / 2), T(kmax)] - # krange = [T(50), T(400)] - krange = [T(16), T(128)] + krange, slope, slopelabel = if D == 2 + [T(16), T(128)], -T(3), L"$\kappa^{-3}" elseif D == 3 - # krange = [T(kmax)^(T(1.5) / 3), T(kmax)] - # krange = [T(64), T(256)] - # krange = [T(8), T(64)] - krange = [T(16), T(100)] + [T(16), T(100)], -T(5 / 3), L"$\kappa^{-5/3}" end - slope, slopelabel = - D == 2 ? (-T(3), L"$\kappa^{-3}") : (-T(5 / 3), L"$\kappa^{-5/3}") - # slope, slopelabel = D == 2 ? (-T(3), "|k|⁻³") : (-T(5 / 3), "|k|⁻⁵³") - # slope, slopelabel = D == 2 ? (-T(3), "κ⁻³") : (-T(5 / 3), "κ⁻⁵³") slopeconst = maximum(specs[1].ehat ./ specs[1].κ .^ slope) offset = D == 2 ? 3 : 2 inertia = offset .* slopeconst .* krange .^ slope @@ -453,19 +254,13 @@ fig = with_theme(; palette = (; color = ["#3366cc", "#cc0000", "#669900", "#ffcc ax = Axis( fig[1, 1]; xticks, - # xlabel = "k", xlabel = "κ", - # ylabel = "e(κ)", xscale = log10, yscale = log10, limits = (1, kmax, T(1e-8), T(1)), title = "Kinetic energy ($(D)D)", ) - # plotparts(i) = 1:specs[i].kmax, specs[i].ehat - # plotparts(i) = 1:specs[i].kmax+1, [specs[i].ehat; eps(T)] plotparts(i) = specs[i].κ, specs[i].ehat - # lines!(ax, 1:specs[1].kmax, specs[1].ehat; color = Cycled(1), label = "DNS") - # lines!(ax, 1:specs[2].kmax, specs[2].ehat; color = Cycled(4), label = "DNS, t = 0") lines!(ax, plotparts(1)...; color = Cycled(1), label = "DNS") lines!(ax, plotparts(2)...; color = Cycled(4), label = "DNS, t = 0") lines!(ax, plotparts(3)...; color = Cycled(2), label = "Filtered DNS (FA)") @@ -475,25 +270,16 @@ fig = with_theme(; palette = (; color = ["#3366cc", "#cc0000", "#669900", "#ffcc lines!(ax, plotparts(7)...; color = Cycled(3)) lines!(ax, plotparts(8)...; color = Cycled(3)) lines!(ax, krange, inertia; color = Cycled(1), label = slopelabel, linestyle = :dash) - D == 2 && axislegend(ax; position = :lb) - # D == 3 && axislegend(ax; position = :rt) - D == 3 && axislegend(ax; position = :lb) + axislegend(ax; position = :lb) autolimits!(ax) if D == 2 - # limits!(ax, (T(0.68), T(520)), (T(1e-3), T(3e1))) - # limits!(ax, (ax.xaxis.attributes.limits[][1], T(1000)), (T(1e-15), ax.yaxis.attributes.limits[][2])) limits!(ax, (T(0.8), T(800)), (T(1e-10), T(1))) elseif D == 3 - # limits!(ax, ax.xaxis.attributes.limits[], (T(1e-1), T(3e1))) - # limits!(ax, ax.xaxis.attributes.limits[], (T(1e-3), ax.yaxis.attributes.limits[][2])) - # limits!(ax, ax.xaxis.attributes.limits[], (T(3e-3), T(2e0))) - # limits!(ax, (T(8e-1), T(400)), (T(2e-3), T(1.5e0))) limits!(ax, (T(8e-1), T(200)), (T(4e-5), T(1.5e0))) end + path = "$output/prioranalysis" + ispath(path) || mkpath(path) + save("$path/spectra_$(D)D_dyadic_Re$(Int(Re)).pdf", fig) fig end -GC.gc() -CUDA.reclaim() - -# save("$output/prioranalysis/spectra_$(D)D_linear_Re$(Int(Re)).pdf", fig) -save("$output/prioranalysis/spectra_$(D)D_dyadic_Re$(Int(Re)).pdf", fig) +clean() diff --git a/PaperDC/src/PaperDC.jl b/PaperDC/src/PaperDC.jl index ea09e1772..d04b91782 100644 --- a/PaperDC/src/PaperDC.jl +++ b/PaperDC/src/PaperDC.jl @@ -1,16 +1,14 @@ +""" +Utility functions for scripts. +""" module PaperDC using IncompressibleNavierStokes using IncompressibleNavierStokes: - momentum, momentum!, divergence!, - project, project!, - apply_bc_u!, - spectral_stuff, - kinetic_energy, - interpolate_u_p + apply_bc_u! using Observables using LinearAlgebra From 3d11e246b8d6f0328f77e24a960b4acbcd78522f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Sat, 20 Apr 2024 15:06:46 +0200 Subject: [PATCH 286/379] Fix docs --- docs/Project.toml | 1 + docs/make.jl | 3 +- docs/references.bib | 4 +- docs/src/features/closure.md | 46 ++- docs/src/features/operators.md | 8 + docs/src/features/pressure.md | 6 + examples/Actuator2D.jl | 6 +- examples/BackwardFacingStep2D.jl | 3 +- examples/DecayingTurbulence2D.jl | 2 +- examples/LidDrivenCavity2D.jl | 2 +- examples/PlanarMixing2D.jl | 6 +- examples/TaylorGreenVortex2D.jl | 2 +- jobs/prioranalysis.jl | 472 ---------------------- libs/NeuralClosure/src/closure.jl | 5 + libs/NeuralClosure/src/create_les_data.jl | 5 + libs/NeuralClosure/src/training.jl | 35 ++ src/operators.jl | 1 - src/solvers/pressure/project.jl | 10 + src/utils/save_vtk.jl | 13 +- 19 files changed, 123 insertions(+), 507 deletions(-) delete mode 100644 jobs/prioranalysis.jl diff --git a/docs/Project.toml b/docs/Project.toml index fe265ad8d..190e805f4 100644 --- a/docs/Project.toml +++ b/docs/Project.toml @@ -3,3 +3,4 @@ Documenter = "e30172f5-a6a5-5a46-863b-614d45cd2de4" DocumenterCitations = "daee34ce-89f3-4625-b898-19384cb65244" IncompressibleNavierStokes = "5e318141-6589-402b-868d-77d7df8c442e" Literate = "98b081ad-f1c9-55d3-8b20-4c87d4299306" +NeuralClosure = "099dac27-d7f2-4047-93d5-0baee36b9c25" diff --git a/docs/make.jl b/docs/make.jl index 3d17fd607..35fb9b49e 100644 --- a/docs/make.jl +++ b/docs/make.jl @@ -5,6 +5,7 @@ push!(LOAD_PATH, joinpath(@__DIR__, "..", "examples")) using IncompressibleNavierStokes +using NeuralClosure using Literate using Documenter using DocumenterCitations @@ -44,7 +45,7 @@ for e ∈ examples end makedocs(; - modules = [IncompressibleNavierStokes], + modules = [IncompressibleNavierStokes, NeuralClosure], plugins = [bib], authors = "Syver Døving Agdestein, Benjamin Sanderse, and contributors", repo = "https://github.com/agdestein/IncompressibleNavierStokes.jl/blob/{commit}{path}#{line}", diff --git a/docs/references.bib b/docs/references.bib index 255f289b1..fe18b85a4 100644 --- a/docs/references.bib +++ b/docs/references.bib @@ -4,14 +4,14 @@ @article{San2012 ISSN={0045-7930}, url={http://dx.doi.org/10.1016/j.compfluid.2012.04.006}, DOI={10.1016/j.compfluid.2012.04.006}, - journal={Computers & Fluids}, + journal={Computers \& Fluids}, publisher={Elsevier BV}, author={San, Omer and Staples, Anne E.}, year={2012}, month={jun}, pages={105–127} } -@article{Sanderse2023, +@misc{Sanderse2023, Author = {Benjamin Sanderse and Francesc Xavier Trias}, Title = {Energy-consistent discretization of viscous dissipation with application to natural convection flow}, diff --git a/docs/src/features/closure.md b/docs/src/features/closure.md index a901fc6c3..55033c0eb 100644 --- a/docs/src/features/closure.md +++ b/docs/src/features/closure.md @@ -1,7 +1,3 @@ -```@meta -CurrentModule = IncompressibleNavierStokes -``` - # Neural closure models For [large eddy simulation (LES)](../features/les.md), a closure model is @@ -39,10 +35,15 @@ M \bar{v} & = 0, \\ ``` ```@docs -face_average! -volume_average! -create_les_data -create_io_arrays +IncompressibleNavierStokes.FaceAverage +IncompressibleNavierStokes.VolumeAverage +``` + +```@docs +NeuralClosure.NeuralClosure +NeuralClosure.filtersaver +NeuralClosure.create_les_data +NeuralClosure.create_io_arrays ``` ## Training @@ -69,11 +70,15 @@ error on the LES solution error. The posterior loss does, but has a longer computational chain involving solving the LES ODE. ```@docs -train -create_randloss -mean_squared_error -relative_error -create_callback +NeuralClosure.create_dataloader_prior +NeuralClosure.create_dataloader_post +NeuralClosure.train +NeuralClosure.create_loss_prior +NeuralClosure.create_relerr_prior +NeuralClosure.mean_squared_error +NeuralClosure.create_loss_post +NeuralClosure.create_relerr_post +NeuralClosure.create_callback ``` ## Neural architectures @@ -81,11 +86,12 @@ create_callback We provide two neural architectures: A convolutional neural network (CNN) and a Fourier neural operator (FNO). ```@docs -wrappedclosure -create_closure -collocate -decollocate -cnn -fno -FourierLayer +NeuralClosure.wrappedclosure +NeuralClosure.create_closure +NeuralClosure.create_tensorclosure +NeuralClosure.collocate +NeuralClosure.decollocate +NeuralClosure.cnn +NeuralClosure.fno +NeuralClosure.FourierLayer ``` diff --git a/docs/src/features/operators.md b/docs/src/features/operators.md index e79e8b890..de4f7fc9b 100644 --- a/docs/src/features/operators.md +++ b/docs/src/features/operators.md @@ -41,4 +41,12 @@ Qfield eig2field! eig2field kinetic_energy +kinetic_energy! +total_kinetic_energy +tensorbasis +smagtensor! +smagorinsky_closure +divoftensor! +tensorbasis! +reconstruct! ``` diff --git a/docs/src/features/pressure.md b/docs/src/features/pressure.md index ec3c628dc..0f178e9d8 100644 --- a/docs/src/features/pressure.md +++ b/docs/src/features/pressure.md @@ -14,10 +14,16 @@ system. ```@docs AbstractPressureSolver DirectPressureSolver +CUDSSPressureSolver CGPressureSolver +CGMatrixPressureSolver SpectralPressureSolver +LowMemorySpectralPressureSolver pressure pressure! poisson poisson! +applypressure! +project +project! ``` diff --git a/examples/Actuator2D.jl b/examples/Actuator2D.jl index 4a5182383..71435382e 100644 --- a/examples/Actuator2D.jl +++ b/examples/Actuator2D.jl @@ -53,9 +53,10 @@ bodyforce(dim, x, y, t) = dim() == 1 ? -cₜ * inside(x, y) : 0.0 # Build setup and assemble operators setup = Setup(x, y; Re = 100.0, boundary_conditions, bodyforce); +psolver = DirectPressureSolver(setup); # Initial conditions (extend inflow) -u₀ = create_initial_conditions(setup, (dim, x, y) -> dim() == 1 ? 1.0 : 0.0); +u₀ = create_initial_conditions(setup, (dim, x, y) -> dim() == 1 ? 1.0 : 0.0; psolver); u = u₀ # Solve unsteady problem @@ -63,6 +64,7 @@ state, outputs = solve_unsteady( setup, u₀, (0.0, 12.0); + psolver, method = RK44P2(), Δt = 0.05, processors = ( @@ -81,7 +83,7 @@ state, outputs = solve_unsteady( # We may visualize or export the computed fields `(u, p)`. # Export to VTK -save_vtk(setup, state.u, state.p, "$output/solution") +save_vtk(setup, state.u, state.t, "$output/solution"; psolver) # We create a box to visualize the actuator. box = ( diff --git a/examples/BackwardFacingStep2D.jl b/examples/BackwardFacingStep2D.jl index 528c7b12d..5a9c2ad2f 100644 --- a/examples/BackwardFacingStep2D.jl +++ b/examples/BackwardFacingStep2D.jl @@ -59,6 +59,7 @@ u₀ = create_initial_conditions(setup, (dim, x, y) -> U(dim, x, y, zero(x)); ps # Solve steady state problem ## u, p = solve_steady_state(setup, u₀, p₀); +nothing # Solve unsteady problem state, outputs = solve_unsteady( @@ -87,7 +88,7 @@ state, outputs = solve_unsteady( # We may visualize or export the computed fields # Export to VTK -save_vtk(setup, state.u, state.p, "$output/solution") +save_vtk(setup, state.u, state.t, "$output/solution"; psolver) # Plot pressure fieldplot(state; setup, fieldname = :pressure) diff --git a/examples/DecayingTurbulence2D.jl b/examples/DecayingTurbulence2D.jl index acb247d73..97f745514 100644 --- a/examples/DecayingTurbulence2D.jl +++ b/examples/DecayingTurbulence2D.jl @@ -75,7 +75,7 @@ outputs.ehist outputs.espec # Export to VTK -save_vtk(setup, state.u, state.p, "$output/solution") +save_vtk(setup, state.u, state.t, "$output/solution"; psolver) # Plot field fieldplot(state; setup) diff --git a/examples/LidDrivenCavity2D.jl b/examples/LidDrivenCavity2D.jl index aec3a2898..755098dff 100644 --- a/examples/LidDrivenCavity2D.jl +++ b/examples/LidDrivenCavity2D.jl @@ -130,7 +130,7 @@ state, outputs = solve_unsteady(setup, u₀, tlims; Δt = T(1e-3), psolver, proc # Export fields to VTK. The file `output/solution.vti` may be opened for # visualization in [ParaView](https://www.paraview.org/). This is particularly # useful for inspecting results from 3D simulations. -save_vtk(setup, state.u, state.p, "$output/solution") +save_vtk(setup, state.u, state.t, "$output/solution"; psolver) # Plot pressure fieldplot(state; setup, fieldname = :pressure) diff --git a/examples/PlanarMixing2D.jl b/examples/PlanarMixing2D.jl index 4b3558300..ffd0ca80b 100644 --- a/examples/PlanarMixing2D.jl +++ b/examples/PlanarMixing2D.jl @@ -47,15 +47,17 @@ plotgrid(x, y) # Build setup and assemble operators setup = Setup(x, y; Re, boundary_conditions); +psolver = DirectPressureSolver(setup); # Initial conditions (extend inflow) -u₀ = create_initial_conditions(setup, (dim, x, y) -> U(dim, x, y, 0.0)); +u₀ = create_initial_conditions(setup, (dim, x, y) -> U(dim, x, y, 0.0); psolver); # Solve unsteady problem state, outputs = solve_unsteady( setup, u₀, (0.0, 100.0); + psolver, method = RK44P2(), Δt = 0.1, processors = ( @@ -80,4 +82,4 @@ state, outputs = solve_unsteady( outputs.rtp # Export to VTK -save_vtk(setup, state.u, state.p, "$output/solution") +save_vtk(setup, state.u, state.t, "$output/solution"; psolver) diff --git a/examples/TaylorGreenVortex2D.jl b/examples/TaylorGreenVortex2D.jl index e8ef0e863..3cf8c1bcf 100644 --- a/examples/TaylorGreenVortex2D.jl +++ b/examples/TaylorGreenVortex2D.jl @@ -28,7 +28,7 @@ function compute_convergence(; D, nlist, lims, Re, tlims, Δt, uref, ArrayType = x = ntuple(α -> LinRange(lims..., n + 1), D) setup = Setup(x...; Re, ArrayType) psolver = SpectralPressureSolver(setup) - u = create_initial_conditions( + u₀ = create_initial_conditions( setup, (dim, x...) -> uref(dim, x..., tlims[1]), tlims[1]; diff --git a/jobs/prioranalysis.jl b/jobs/prioranalysis.jl deleted file mode 100644 index a02a86d64..000000000 --- a/jobs/prioranalysis.jl +++ /dev/null @@ -1,472 +0,0 @@ -# Little LSP hack to get function signatures, go #src -# to definition etc. #src -if isdefined(@__MODULE__, :LanguageServer) #src - include("../src/IncompressibleNavierStokes.jl") #src - using .IncompressibleNavierStokes #src -end #src - -@info "Loading packages" - -using CairoMakie -using IncompressibleNavierStokes -using IncompressibleNavierStokes: - momentum, - momentum!, - divergence!, - project, - project!, - apply_bc_u!, - spectral_stuff, - kinetic_energy, - interpolate_u_p -using JLD2 -using LinearAlgebra -using Printf -using FFTW - -function observe_v(dnsobs, Φ, les, compression, psolver) - (; grid) = les - (; dimension, N, Iu, Ip) = grid - D = dimension() - Mα = N[1] - 2 - v = zero.(Φ(dnsobs[].u, les, compression)) - Pv = zero.(v) - p = zero(v[1]) - div = zero(p) - ΦPF = zero.(v) - PFΦ = zero.(v) - c = zero.(v) - T = eltype(v[1]) - results = (; - Φ, - Mα, - t = zeros(T, 0), - Dv = zeros(T, 0), - Pv = zeros(T, 0), - Pc = zeros(T, 0), - c = zeros(T, 0), - E = zeros(T, 0), - ) - on(dnsobs) do (; u, PF, t) - push!(results.t, t) - - Φ(v, u, les, compression) - apply_bc_u!(v, t, les) - Φ(ΦPF, PF, les, compression) - momentum!(PFΦ, v, t, les) - apply_bc_u!(PFΦ, t, les; dudt = true) - project!(PFΦ, les; psolver, div, p) - foreach(α -> c[α] .= ΦPF[α] .- PFΦ[α], 1:D) - apply_bc_u!(c, t, les) - divergence!(div, v, les) - norm_Du = norm(div[Ip]) - norm_v = sqrt(sum(α -> sum(abs2, v[α][Iu[α]]), 1:D)) - push!(results.Dv, norm_Du / norm_v) - - copyto!.(Pv, v) - project!(Pv, les; psolver, div, p) - foreach(α -> Pv[α] .= Pv[α] .- v[α], 1:D) - norm_vmPv = sqrt(sum(α -> sum(abs2, Pv[α][Iu[α]]), 1:D)) - push!(results.Pv, norm_vmPv / norm_v) - - Pc = Pv - copyto!.(Pc, c) - project!(Pc, les; psolver, div, p) - foreach(α -> Pc[α] .= Pc[α] .- c[α], 1:D) - norm_cmPc = sqrt(sum(α -> sum(abs2, Pc[α][Iu[α]]), 1:D)) - norm_c = sqrt(sum(α -> sum(abs2, c[α][Iu[α]]), 1:D)) - push!(results.Pc, norm_cmPc / norm_c) - - norm_ΦPF = sqrt(sum(α -> sum(abs2, ΦPF[α][Iu[α]]), 1:D)) - push!(results.c, norm_c / norm_ΦPF) - - Eu = sum(α -> sum(abs2, view(u[α], ntuple(b -> 2:size(u[α], b)-1, D)...)), 1:D) - Ev = norm_v^2 - E = compression^D * Ev / Eu - push!(results.E, E) - end - results -end - -observe_u(dns, psolver_dns, filters; nupdate = 1) = - processor() do state - PF = zero.(state[].u) - div = zero(state[].u[1]) - p = zero(state[].u[1]) - dnsobs = Observable((; state[].u, PF, state[].t)) - results = [ - observe_v(dnsobs, Φ, setup, compression, psolver) for - (; setup, Φ, compression, psolver) in filters - ] - on(state) do (; u, t, n) - n % nupdate == 0 || return - apply_bc_u!(u, t, dns) - momentum!(PF, u, t, dns) - apply_bc_u!(PF, t, dns; dudt = true) - project!(PF, dns; psolver = psolver_dns, div, p) - dnsobs[] = (; u, PF, t) - end - # state[] = state[] # Save initial conditions - results - end - -# Array type -ArrayType = Array -# using CUDA; ArrayType = CuArray; -## using AMDGPU; ArrayType = ROCArray -## using oneAPI; ArrayType = oneArray -## using Metal; ArrayType = MtlArray - -using CUDA; -ArrayType = CuArray; -CUDA.allowscalar(false); - -# 3D -T = Float32 -# T = Float64 -Re = T(2_000) -ndns = 512 -# ndns = 1024 -D = 3 -kp = 5 -Δt = T(1e-4) -filterdefs = [ - (FaceAverage(), 32), - (FaceAverage(), 64), - (FaceAverage(), 128), - (VolumeAverage(), 32), - (VolumeAverage(), 64), - (VolumeAverage(), 128), -] - -# # 2D -# T = Float64; -# Re = T(10_000) -# ndns = 4096 -# D = 2 -# kp = 20 -# Δt = T(5e-5) -# filterdefs = [ -# (FaceAverage(), 64), -# (FaceAverage(), 128), -# (FaceAverage(), 256), -# (VolumeAverage(), 64), -# (VolumeAverage(), 128), -# (VolumeAverage(), 256), -# ] - -# Output directory -output = "output/prioranalysis/dimension$D" -ispath(output) || mkpath(output) - -# Setup -@info "Building setup" -lims = T(0), T(1) -dns = Setup(ntuple(α -> LinRange(lims..., ndns + 1), D)...; Re, ArrayType); -filters = map(filterdefs) do (Φ, nles) - compression = ndns ÷ nles - setup = Setup(ntuple(α -> LinRange(lims..., nles + 1), D)...; Re, ArrayType) - psolver = SpectralPressureSolver(setup) - (; setup, Φ, compression, psolver) -end; -psolver_dns = SpectralPressureSolver(dns); - -# Create random initial conditions -@info "Generating initial conditions" -u₀ = random_field(dns, T(0); kp, psolver = psolver_dns); -GC.gc() -CUDA.reclaim() - -# Solve unsteady problem -@info "Launching time stepping" -@time state, outputs = solve_unsteady( - dns, - u₀, - (T(0), T(1e-1)); - Δt, - docopy = true, - psolver = psolver_dns, - processors = ( - obs = observe_u(dns, psolver_dns, filters; nupdate = 20), - log = timelogger(; nupdate = 20), - ), -); -GC.gc() -CUDA.reclaim() - -# Save final solution -@info "Saving final solution" -jldsave("$output/finalsolution.jld", u = Array.(state.u)) - -# u = load("$output/finalsolution.jld")["u"] -# state = (; u = ArrayType.(u), t = T(1e-1)) - -# fil = filters[2]; -# apply_bc_u!(state.u, T(0), dns); -# v = fil.Φ(state.u, fil.setup, fil.compression); -# apply_bc_u!(v, T(0), fil.setup); -# Fv = momentum(v, T(0), fil.setup); -# apply_bc_u!(Fv, T(0), fil.setup); -# PFv = project(Fv, fil.setup; psolver = fil.psolver); -# apply_bc_u!(PFv, T(0), fil.setup); -# F = momentum(state.u, T(0), dns); -# apply_bc_u!(F, T(0), dns); -# PF = project(F, dns; psolver = psolver_dns); -# apply_bc_u!(PF, T(0), dns); -# ΦPF = fil.Φ(PF, fil.setup, fil.compression); -# apply_bc_u!(ΦPF, T(0), fil.setup); -# c = ΦPF .- PFv -# apply_bc_u!(c, T(0), fil.setup) - -# with_theme(; fontsize = 25) do -# fig = fieldplot( -# (; u = u₀, t = T(0)); -# setup = dns, -# # type = image, -# # colormap = :viridis, -# docolorbar = false, -# size = (500, 500), -# title = "u₀", -# ) -# save("$output/ustart.png", fig) -# fig = fieldplot( -# state, -# setup = dns, -# # type = image, -# # colormap = :viridis, -# docolorbar = false, -# size = (500, 500), -# title = "u", -# ) -# save("$output/u.png", fig) -# fig = fieldplot( -# (; u = v, t = T(0)); -# fil.setup, -# # type = image, -# # colormap = :viridis, -# # fieldname = 1, -# docolorbar = false, -# size = (500, 500), -# # title = "ubar" -# title = "ū", -# ) -# save("$output/v.png", fig) -# fig = fieldplot( -# (; u = PF, t = T(0)); -# setup = dns, -# # type = image, -# # colormap = :viridis, -# # fieldname = 1, -# docolorbar = false, -# size = (500, 500), -# title = "PF(u)", -# ) -# save("$output/PFu.png", fig) -# fig = fieldplot( -# (; u = PFv, t = T(0)); -# fil.setup, -# # type = image, -# # colormap = :viridis, -# # fieldname = 1, -# docolorbar = false, -# size = (500, 500), -# # title = "PF(ubar)" -# title = "P̄F̄(ū)", -# ) -# save("$output/PFv.png", fig) -# fig = fieldplot( -# (; u = ΦPF, t = T(0)); -# fil.setup, -# # type = image, -# # colormap = :viridis, -# # fieldname = 1, -# docolorbar = false, -# size = (500, 500), -# title = "ΦPF(u)", -# ) -# save("$output/PhiPFu.png", fig) -# fig = fieldplot( -# (; u = c, t = T(0)); -# fil.setup, -# # type = image, -# # colormap = :viridis, -# # fieldname = 1, -# # fieldname = :velocity, -# docolorbar = false, -# size = (500, 500), -# # title = "c(u, ubar)" -# title = "c(u, ū)", -# ) -# save("$output/c.png", fig) -# end - -# 3D fieldplot ################################################################# - -# fieldplot( -# # (; u = u₀, t = T(0)); -# state; -# setup = dns, -# fieldname = :eig2field, -# # levels = LinRange(T(2), T(10), 10), -# levels = LinRange(T(4), T(12), 10), -# # levels = LinRange(-1.0f0, 3.0f0, 5), -# # levels = LinRange(-2.0f0, 2.0f0, 5), -# # levels = 5, -# docolorbar = false, -# # size = (800, 800), -# size = (600, 600), -# ) -# -# fname = "$output/lambda2/Re$(Int(Re))_start.png" -# fname = "$output/lambda2/Re$(Int(Re))_end.png" -# save(fname, current_figure()) -# run(`convert $fname -trim $fname`) # Requires imagemagick - -# i = 3 -# fieldplot( -# (; u = filters[i].Φ(state.u, filters[i].setup, filters[i].compression), t = T(0)); -# setup = filters[i].setup, -# fieldname = :eig2field, -# # levels = LinRange(T(2), T(10), 10), -# levels = LinRange(T(4), T(12), 10), -# # levels = LinRange(-1.0f0, 3.0f0, 5), -# # levels = LinRange(-2.0f0, 2.0f0, 5), -# # levels = 5, -# docolorbar = false, -# # size = (800, 800), -# size = (600, 600), -# ) -# -# fname = "$output/lambda2/Re$(Int(Re))_end_filtered.png" -# save(fname, current_figure()) -# run(`convert $fname -trim $fname`) # Requires imagemagick - -@info "Computing statistics" -begin - println("Φ\t\tM\tDu\tPv\tPc\tc\tE") - for o in outputs.obs - nt = length(o.t) - Dv = sum(o.Dv) / nt - Pc = sum(o.Pc) / nt - Pv = sum(o.Pv) / nt - c = sum(o.c) / nt - E = sum(o.E) / nt - @printf( - "%s\t%d^%d\t%.2g\t%.2g\t%.2g\t%.2g\t%.2g\n", - # "%s &\t\$%d^%d\$ &\t\$%.2g\$ &\t\$%.2g\$ &\t\$%.2g\$ &\t\$%.2g\$ &\t\$%.2g\$\n", - typeof(o.Φ), - o.Mα, - D, - Dv, - Pv, - Pc, - c, - E, - ) - end -end; - -# To free up memory -psolver_dns = nothing -# fig = lines([1, 2, 3]) -GC.gc() -CUDA.reclaim() - -# Plot predicted spectra -@info "Computing and plotting spectra" -fig = with_theme(; palette = (; color = ["#3366cc", "#cc0000", "#669900", "#ffcc00"])) do - fields = [state.u, u₀, (f.Φ(state.u, f.setup, f.compression) for f in filters)...] - setups = [dns, dns, (f.setup for f in filters)...] - specs = map(fields, setups) do u, setup - GC.gc() - CUDA.reclaim() - (; dimension, xp, Ip) = setup.grid - T = eltype(xp[1]) - D = dimension() - K = size(Ip) .÷ 2 - # up = interpolate_u_p(u, setup) - up = u - e = sum(up) do u - u = u[Ip] - uhat = fft(u)[ntuple(α -> 1:K[α], D)...] - # abs2.(uhat) - abs2.(uhat) ./ (2 * prod(size(u))^2) - # abs2.(uhat) ./ size(u, 1) - end - (; A, κ, K) = spectral_stuff(setup) - e = A * reshape(e, :) - # e = max.(e, eps(T)) # Avoid log(0) - ehat = Array(e) - (; κ, ehat) - end - kmax = maximum(specs[1].κ) - # Build inertial slope above energy - if D == 2 - # krange = [T(kmax)^(T(1) / 2), T(kmax)] - # krange = [T(50), T(400)] - krange = [T(16), T(128)] - elseif D == 3 - # krange = [T(kmax)^(T(1.5) / 3), T(kmax)] - # krange = [T(64), T(256)] - # krange = [T(8), T(64)] - krange = [T(16), T(100)] - end - slope, slopelabel = - D == 2 ? (-T(3), L"$\kappa^{-3}") : (-T(5 / 3), L"$\kappa^{-5/3}") - # slope, slopelabel = D == 2 ? (-T(3), "|k|⁻³") : (-T(5 / 3), "|k|⁻⁵³") - # slope, slopelabel = D == 2 ? (-T(3), "κ⁻³") : (-T(5 / 3), "κ⁻⁵³") - slopeconst = maximum(specs[1].ehat ./ specs[1].κ .^ slope) - offset = D == 2 ? 3 : 2 - inertia = offset .* slopeconst .* krange .^ slope - # Nice ticks - logmax = round(Int, log2(kmax + 1)) - xticks = T(2) .^ (0:logmax) - # Make plot - fig = Figure(; size = (500, 400)) - ax = Axis( - fig[1, 1]; - xticks, - # xlabel = "k", - xlabel = "κ", - # ylabel = "e(κ)", - xscale = log10, - yscale = log10, - limits = (1, kmax, T(1e-8), T(1)), - title = "Kinetic energy ($(D)D)", - ) - # plotparts(i) = 1:specs[i].kmax, specs[i].ehat - # plotparts(i) = 1:specs[i].kmax+1, [specs[i].ehat; eps(T)] - plotparts(i) = specs[i].κ, specs[i].ehat - # lines!(ax, 1:specs[1].kmax, specs[1].ehat; color = Cycled(1), label = "DNS") - # lines!(ax, 1:specs[2].kmax, specs[2].ehat; color = Cycled(4), label = "DNS, t = 0") - lines!(ax, plotparts(1)...; color = Cycled(1), label = "DNS") - lines!(ax, plotparts(2)...; color = Cycled(4), label = "DNS, t = 0") - lines!(ax, plotparts(3)...; color = Cycled(2), label = "Filtered DNS (FA)") - lines!(ax, plotparts(4)...; color = Cycled(2)) - lines!(ax, plotparts(5)...; color = Cycled(2)) - lines!(ax, plotparts(6)...; color = Cycled(3), label = "Filtered DNS (VA)") - lines!(ax, plotparts(7)...; color = Cycled(3)) - lines!(ax, plotparts(8)...; color = Cycled(3)) - lines!(ax, krange, inertia; color = Cycled(1), label = slopelabel, linestyle = :dash) - D == 2 && axislegend(ax; position = :lb) - # D == 3 && axislegend(ax; position = :rt) - D == 3 && axislegend(ax; position = :lb) - autolimits!(ax) - if D == 2 - # limits!(ax, (T(0.68), T(520)), (T(1e-3), T(3e1))) - # limits!(ax, (ax.xaxis.attributes.limits[][1], T(1000)), (T(1e-15), ax.yaxis.attributes.limits[][2])) - limits!(ax, (T(0.8), T(800)), (T(1e-10), T(1))) - elseif D == 3 - # limits!(ax, ax.xaxis.attributes.limits[], (T(1e-1), T(3e1))) - # limits!(ax, ax.xaxis.attributes.limits[], (T(1e-3), ax.yaxis.attributes.limits[][2])) - # limits!(ax, ax.xaxis.attributes.limits[], (T(3e-3), T(2e0))) - # limits!(ax, (T(8e-1), T(400)), (T(2e-3), T(1.5e0))) - limits!(ax, (T(8e-1), T(200)), (T(4e-5), T(1.5e0))) - end - fig -end -GC.gc() -CUDA.reclaim() - -# save("$output/spectra_$(D)D_linear_Re$(Int(Re)).pdf", fig) -save("$output/spectra_$(D)D_dyadic_Re$(Int(Re)).pdf", fig) diff --git a/libs/NeuralClosure/src/closure.jl b/libs/NeuralClosure/src/closure.jl index 9267c847a..b2faa3499 100644 --- a/libs/NeuralClosure/src/closure.jl +++ b/libs/NeuralClosure/src/closure.jl @@ -49,6 +49,11 @@ function create_closure(layers...; rng) closure, θ end +""" + create_tensorclosure(layers...; setup, rng) + +Create tensor basis closure. +""" function create_tensorclosure(layers...; setup, rng) D = setup.grid.dimension() cnn, θ = create_closure(layers...; rng) diff --git a/libs/NeuralClosure/src/create_les_data.jl b/libs/NeuralClosure/src/create_les_data.jl index 93e429d2b..3ea1d072c 100644 --- a/libs/NeuralClosure/src/create_les_data.jl +++ b/libs/NeuralClosure/src/create_les_data.jl @@ -56,6 +56,11 @@ function lesdatagen(dnsobs, Φ, les, compression, psolver) results end +""" + filtersaver(dns, les, filters, compression, psolver_dns, psolver_les; nupdate = 1) + +Save filtered DNS data. +""" filtersaver(dns, les, filters, compression, psolver_dns, psolver_les; nupdate = 1) = processor( (results, state) -> (; results..., comptime = time() - results.comptime), diff --git a/libs/NeuralClosure/src/training.jl b/libs/NeuralClosure/src/training.jl index 56532f599..35e559616 100644 --- a/libs/NeuralClosure/src/training.jl +++ b/libs/NeuralClosure/src/training.jl @@ -15,6 +15,11 @@ create_dataloader_prior(data; batchsize = 50, device = identity) = function data xuse, yuse end +""" + create_dataloader_post(trajectories; nunroll = 10, device = identity) + +Create trajectory dataloader. +""" create_dataloader_post(trajectories; nunroll = 10, device = identity) = function dataloader() (; u, t) = rand(trajectories) @@ -74,6 +79,11 @@ The function `loss` should take inputs like `loss(f, x, y, θ)`. """ create_loss_prior(loss, f) = ((x, y), θ) -> loss(f, x, y, θ) +""" + create_relerr_prior(f, x, y) + +Create a-priori error. +""" create_relerr_prior(f, x, y) = θ -> norm(f(x, θ) - y) / norm(y) """ @@ -86,6 +96,18 @@ The MSE is further divided by `normalize(y)`. mean_squared_error(f, x, y, θ; normalize = y -> sum(abs2, y), λ = sqrt(eltype(x)(1e-8))) = sum(abs2, f(x, θ) - y) / normalize(y) + λ * sum(abs2, θ) +""" + create_loss_post(; + setup, + method = RK44(; T = eltype(setup.grid.x[1])), + psolver, + closure, + nupdate = 1, + projectorder = :last, + ) + +Create a-posteriori loss function. +""" function create_loss_post(; setup, method = RK44(; T = eltype(setup.grid.x[1])), @@ -120,6 +142,19 @@ function create_loss_post(; end end +""" + create_relerr_post(; + data, + setup, + method = RK44(; T = eltype(setup.grid.x[1])), + psolver, + closure_model, + nupdate = 1, + projectorder = :last, + ) + +Create a-posteriori relative error. +""" function create_relerr_post(; data, setup, diff --git a/src/operators.jl b/src/operators.jl index d1b9efe55..5a85ff66c 100644 --- a/src/operators.jl +++ b/src/operators.jl @@ -493,7 +493,6 @@ end Right hand side of momentum equations, excluding pressure gradient. """ -# momentum(u, t, setup) = momentum!(zero.(u), u, t, setup) function momentum(u, t, setup) (; grid, closure_model) = setup (; dimension) = grid diff --git a/src/solvers/pressure/project.jl b/src/solvers/pressure/project.jl index 98b2f8d70..6c24190fd 100644 --- a/src/solvers/pressure/project.jl +++ b/src/solvers/pressure/project.jl @@ -1,3 +1,8 @@ +""" + project(u, setup; psolver) + +Project velocity field onto divergence-free space. +""" function project(u, setup; psolver) (; Ω) = setup.grid T = eltype(u[1]) @@ -15,6 +20,11 @@ function project(u, setup; psolver) u .- G end +""" + project!(u, setup; psolver, div, p) + +Project velocity field onto divergence-free space. +""" function project!(u, setup; psolver, div, p) (; Ω) = setup.grid T = eltype(u[1]) diff --git a/src/utils/save_vtk.jl b/src/utils/save_vtk.jl index 351153fdc..7535ded6d 100644 --- a/src/utils/save_vtk.jl +++ b/src/utils/save_vtk.jl @@ -1,12 +1,19 @@ """ - save_vtk(setup, u, p, filename = "output/solution") + save_vtk(setup, u, filename = "output/solution"; fieldnames = [:velocity], psolver) Save velocity and pressure field to a VTK file. In the case of a 2D setup, the velocity field is saved as a 3D vector with a z-component of zero, as this seems to be preferred by ParaView. """ -function save_vtk(setup, u, p, filename = "output/solution") +function save_vtk( + setup, + u, + t, + filename = "output/solution"; + fieldnames = [:velocity], + psolver, +) parts = split(filename, "/") path = join(parts[1:end-1], "/") isdir(path) || mkpath(path) @@ -26,7 +33,7 @@ function save_vtk(setup, u, p, filename = "output/solution") ωp = Array.(ωp) end vtk["velocity"] = Array.(up) - vtk["pressure"] = Array(p) + :pressure in fieldnames && (vtk["pressure"] = Array(pressure(u, t, setup; psolver))) vtk["vorticity"] = ωp end end From e460d57b6cc99c120f228374e83e9ac0b2b405fb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Sat, 20 Apr 2024 15:41:38 +0200 Subject: [PATCH 287/379] Fix make --- docs/make.jl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/make.jl b/docs/make.jl index 35fb9b49e..73785cb84 100644 --- a/docs/make.jl +++ b/docs/make.jl @@ -1,8 +1,9 @@ # Check number of threads aviailable on GitHub Actions @info "" Threads.nthreads() -# Load examples environment +# Load environments push!(LOAD_PATH, joinpath(@__DIR__, "..", "examples")) +push!(LOAD_PATH, joinpath(@__DIR__, "..", "libs", "NeuralClosure")) using IncompressibleNavierStokes using NeuralClosure From 5f66af2903046efe6610ce5f644fa28438392c9c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Sat, 20 Apr 2024 16:09:44 +0200 Subject: [PATCH 288/379] Up --- docs/Project.toml | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/Project.toml b/docs/Project.toml index 190e805f4..fe265ad8d 100644 --- a/docs/Project.toml +++ b/docs/Project.toml @@ -3,4 +3,3 @@ Documenter = "e30172f5-a6a5-5a46-863b-614d45cd2de4" DocumenterCitations = "daee34ce-89f3-4625-b898-19384cb65244" IncompressibleNavierStokes = "5e318141-6589-402b-868d-77d7df8c442e" Literate = "98b081ad-f1c9-55d3-8b20-4c87d4299306" -NeuralClosure = "099dac27-d7f2-4047-93d5-0baee36b9c25" From 026b76b7ee3ff44b42f935edd7cb03c8f0934446 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Sat, 20 Apr 2024 18:24:49 +0200 Subject: [PATCH 289/379] Fix make --- .github/workflows/CI.yml | 2 ++ docs/make.jl | 13 +++++++++---- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 522f84cee..8506f4306 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -52,6 +52,8 @@ jobs: version: '1' - uses: julia-actions/julia-buildpkg@v1 - uses: julia-actions/julia-docdeploy@v1 + with: + install-package: false env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | diff --git a/docs/make.jl b/docs/make.jl index 73785cb84..c72e4096b 100644 --- a/docs/make.jl +++ b/docs/make.jl @@ -1,9 +1,14 @@ -# Check number of threads aviailable on GitHub Actions -@info "" Threads.nthreads() - # Load environments push!(LOAD_PATH, joinpath(@__DIR__, "..", "examples")) -push!(LOAD_PATH, joinpath(@__DIR__, "..", "libs", "NeuralClosure")) + +if haskey(ENV, "GithubActions") + @info "" Threads.nthreads() + using Pkg + cd(@__DIR__) + Pkg.activate(".") + pkg"dev .. ../libs/NeuralClosure" + Pkg.instantiate() +end using IncompressibleNavierStokes using NeuralClosure From 83302cc10d3ddd3b110021895a550ffd5b2bce7f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Sat, 20 Apr 2024 18:29:56 +0200 Subject: [PATCH 290/379] Fix make --- docs/make.jl | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/docs/make.jl b/docs/make.jl index c72e4096b..2c4b2f4c2 100644 --- a/docs/make.jl +++ b/docs/make.jl @@ -6,7 +6,10 @@ if haskey(ENV, "GithubActions") using Pkg cd(@__DIR__) Pkg.activate(".") - pkg"dev .. ../libs/NeuralClosure" + Pkg.develop([ + PackageSpec(; path = ".."), + PackageSpec(; path = "../libs/NeuralClosure"), + ]) Pkg.instantiate() end From e3aa518a562f68b288c7ec73087dbd29260e98e6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Sat, 20 Apr 2024 18:53:52 +0200 Subject: [PATCH 291/379] Up --- docs/make.jl | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/docs/make.jl b/docs/make.jl index 2c4b2f4c2..9561fcf7f 100644 --- a/docs/make.jl +++ b/docs/make.jl @@ -1,17 +1,16 @@ # Load environments push!(LOAD_PATH, joinpath(@__DIR__, "..", "examples")) -if haskey(ENV, "GithubActions") - @info "" Threads.nthreads() - using Pkg - cd(@__DIR__) - Pkg.activate(".") - Pkg.develop([ - PackageSpec(; path = ".."), - PackageSpec(; path = "../libs/NeuralClosure"), - ]) - Pkg.instantiate() -end +@info "" Threads.nthreads() + +using Pkg +cd(@__DIR__) +Pkg.activate(".") +Pkg.develop([ + PackageSpec(; path = ".."), + PackageSpec(; path = "../libs/NeuralClosure"), +]) +Pkg.instantiate() using IncompressibleNavierStokes using NeuralClosure From 5d4bcfaeb00a747036055c65c355edbdcdaa06b4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Sat, 20 Apr 2024 23:22:24 +0200 Subject: [PATCH 292/379] Fix make --- docs/make.jl | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/docs/make.jl b/docs/make.jl index 9561fcf7f..471493c9a 100644 --- a/docs/make.jl +++ b/docs/make.jl @@ -1,8 +1,6 @@ -# Load environments -push!(LOAD_PATH, joinpath(@__DIR__, "..", "examples")) - @info "" Threads.nthreads() +# Load environments using Pkg cd(@__DIR__) Pkg.activate(".") @@ -10,6 +8,7 @@ Pkg.develop([ PackageSpec(; path = ".."), PackageSpec(; path = "../libs/NeuralClosure"), ]) +push!(LOAD_PATH, joinpath(@__DIR__, "..", "examples")) Pkg.instantiate() using IncompressibleNavierStokes From 7c8822ed9cb0cd23fdd0c3f878d6beda6f00309c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Sun, 21 Apr 2024 10:48:55 +0200 Subject: [PATCH 293/379] Fix make --- docs/Project.toml | 2 ++ docs/make.jl | 12 +++++++++--- examples/Project.toml | 5 +++++ examples/src/Examples.jl | 5 +++++ 4 files changed, 21 insertions(+), 3 deletions(-) create mode 100644 examples/src/Examples.jl diff --git a/docs/Project.toml b/docs/Project.toml index fe265ad8d..e08de019d 100644 --- a/docs/Project.toml +++ b/docs/Project.toml @@ -1,5 +1,7 @@ [deps] Documenter = "e30172f5-a6a5-5a46-863b-614d45cd2de4" DocumenterCitations = "daee34ce-89f3-4625-b898-19384cb65244" +Examples = "318dbb63-4243-420f-99f2-d56058123f9d" IncompressibleNavierStokes = "5e318141-6589-402b-868d-77d7df8c442e" Literate = "98b081ad-f1c9-55d3-8b20-4c87d4299306" +NeuralClosure = "099dac27-d7f2-4047-93d5-0baee36b9c25" diff --git a/docs/make.jl b/docs/make.jl index 471493c9a..3c6ee773b 100644 --- a/docs/make.jl +++ b/docs/make.jl @@ -1,16 +1,22 @@ +# Show number of threads on GitHub Actions @info "" Threads.nthreads() -# Load environments -using Pkg +# Make paths relative to this file cd(@__DIR__) + +# Build docs environment +using Pkg Pkg.activate(".") Pkg.develop([ PackageSpec(; path = ".."), PackageSpec(; path = "../libs/NeuralClosure"), + PackageSpec(; path = "../examples"), ]) -push!(LOAD_PATH, joinpath(@__DIR__, "..", "examples")) Pkg.instantiate() +# Get access to example dependencies +push!(LOAD_PATH, joinpath(@__DIR__, "..", "examples")) + using IncompressibleNavierStokes using NeuralClosure using Literate diff --git a/examples/Project.toml b/examples/Project.toml index fd652a236..477423491 100644 --- a/examples/Project.toml +++ b/examples/Project.toml @@ -1,3 +1,8 @@ +name = "Examples" +uuid = "318dbb63-4243-420f-99f2-d56058123f9d" +authors = ["Syver Døving Agdestein "] +version = "0.1.0" + [deps] CUDA = "052768ef-5323-5732-b1bb-66c8b64840ba" CairoMakie = "13f3f980-e62b-5c42-98c6-ff1f3baf88f0" diff --git a/examples/src/Examples.jl b/examples/src/Examples.jl new file mode 100644 index 000000000..7d9f903f1 --- /dev/null +++ b/examples/src/Examples.jl @@ -0,0 +1,5 @@ +""" +Empty module for examples environment. +This allows the docs to load example dependencies. +""" +module Examples end From 049ac3aca9f03292d1b0571a55eada96c69054dd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Sun, 21 Apr 2024 10:49:34 +0200 Subject: [PATCH 294/379] Fix examples --- examples/Actuator3D.jl | 2 +- examples/BackwardFacingStep3D.jl | 2 +- examples/DecayingTurbulence3D.jl | 2 +- examples/LidDrivenCavity3D.jl | 2 +- examples/MultiActuator.jl | 2 +- examples/PlaneJets2D.jl | 2 +- examples/ShearLayer2D.jl | 2 +- examples/TaylorGreenVortex3D.jl | 2 +- src/utils/save_vtk.jl | 2 +- 9 files changed, 9 insertions(+), 9 deletions(-) diff --git a/examples/Actuator3D.jl b/examples/Actuator3D.jl index afb65984d..fb0dfd508 100644 --- a/examples/Actuator3D.jl +++ b/examples/Actuator3D.jl @@ -104,4 +104,4 @@ u₀ = create_initial_conditions(setup, (dim, x, y, z) -> dim() == 1 ? one(x) : outputs.rtp # Export to VTK -save_vtk(setup, u, p, "$output/solution") +save_vtk(setup, u, t, "$output/solution") diff --git a/examples/BackwardFacingStep3D.jl b/examples/BackwardFacingStep3D.jl index f9deb774f..e57741c95 100644 --- a/examples/BackwardFacingStep3D.jl +++ b/examples/BackwardFacingStep3D.jl @@ -87,7 +87,7 @@ state, outputs = solve_unsteady( # We may visualize or export the computed fields # Export to VTK -save_vtk(setup, state.u, state.p, "$output/solution") +save_vtk(setup, state.u, state.t, "$output/solution") # Plot pressure fieldplot(state; setup, fieldname = :pressure) diff --git a/examples/DecayingTurbulence3D.jl b/examples/DecayingTurbulence3D.jl index 1f7e0deb5..4f30e115c 100644 --- a/examples/DecayingTurbulence3D.jl +++ b/examples/DecayingTurbulence3D.jl @@ -77,4 +77,4 @@ outputs.ehist outputs.espec # Export to VTK -save_vtk(setup, u, p, "$output/solution") +save_vtk(setup, u, t, "$output/solution") diff --git a/examples/LidDrivenCavity3D.jl b/examples/LidDrivenCavity3D.jl index 8605fc9bd..d49a66d5e 100644 --- a/examples/LidDrivenCavity3D.jl +++ b/examples/LidDrivenCavity3D.jl @@ -73,7 +73,7 @@ u₀ = create_initial_conditions(setup, (dim, x, y, z) -> zero(x)) # We may visualize or export the computed fields `(V, p)` # Export to VTK -save_vtk(setup, u, p, "$output/solution") +save_vtk(setup, u, t, "$output/solution") # Energy history outputs.ehist diff --git a/examples/MultiActuator.jl b/examples/MultiActuator.jl index d56eebf04..13d01492e 100644 --- a/examples/MultiActuator.jl +++ b/examples/MultiActuator.jl @@ -144,7 +144,7 @@ state, outputs = solve_unsteady( # We may visualize or export the computed fields `(u, p)`. # Export to VTK -save_vtk(setup, state.u, state.p, "$output/solution") +save_vtk(setup, state.u, state.t, "$output/solution") # Plot pressure fig = fieldplot(state; setup, fieldname = :pressure, psolver) diff --git a/examples/PlaneJets2D.jl b/examples/PlaneJets2D.jl index c4d83cb98..111856413 100644 --- a/examples/PlaneJets2D.jl +++ b/examples/PlaneJets2D.jl @@ -180,7 +180,7 @@ state, outputs = solve_unsteady( outputs.rtp # Export to VTK -save_vtk(setup, state.u, state.p, "$output/solution") +save_vtk(setup, state.u, state.t, "$output/solution") # Plot pressure fieldplot(state; setup, fieldname = :pressure) diff --git a/examples/ShearLayer2D.jl b/examples/ShearLayer2D.jl index 746a037b3..2225290bc 100644 --- a/examples/ShearLayer2D.jl +++ b/examples/ShearLayer2D.jl @@ -81,7 +81,7 @@ state, outputs = solve_unsteady( outputs.rtp # Export to VTK -save_vtk(setup, state.u, state.p, "$output/solution") +save_vtk(setup, state.u, state.t, "$output/solution"; psolver) # Plot pressure fieldplot(state; setup, fieldname = :pressure) diff --git a/examples/TaylorGreenVortex3D.jl b/examples/TaylorGreenVortex3D.jl index 44cf7cdd5..5bed0ea73 100644 --- a/examples/TaylorGreenVortex3D.jl +++ b/examples/TaylorGreenVortex3D.jl @@ -79,4 +79,4 @@ outputs.ehist outputs.espec # Export to VTK -save_vtk(setup, u, p, "$output/solution") +save_vtk(setup, u, t, "$output/solution"; psolver) diff --git a/src/utils/save_vtk.jl b/src/utils/save_vtk.jl index 7535ded6d..c11ade3e6 100644 --- a/src/utils/save_vtk.jl +++ b/src/utils/save_vtk.jl @@ -12,7 +12,7 @@ function save_vtk( t, filename = "output/solution"; fieldnames = [:velocity], - psolver, + psolver = DirectPressureSolver(setup), ) parts = split(filename, "/") path = join(parts[1:end-1], "/") From 04c48ff060a4068701eac0889351d35eaa1c3681 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Sun, 21 Apr 2024 12:16:12 +0200 Subject: [PATCH 295/379] chore: Format files --- PaperDC/src/PaperDC.jl | 6 +----- docs/make.jl | 2 +- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/PaperDC/src/PaperDC.jl b/PaperDC/src/PaperDC.jl index d04b91782..e7e309d4b 100644 --- a/PaperDC/src/PaperDC.jl +++ b/PaperDC/src/PaperDC.jl @@ -4,11 +4,7 @@ Utility functions for scripts. module PaperDC using IncompressibleNavierStokes -using IncompressibleNavierStokes: - momentum!, - divergence!, - project!, - apply_bc_u! +using IncompressibleNavierStokes: momentum!, divergence!, project!, apply_bc_u! using Observables using LinearAlgebra diff --git a/docs/make.jl b/docs/make.jl index 3c6ee773b..18c929896 100644 --- a/docs/make.jl +++ b/docs/make.jl @@ -58,7 +58,7 @@ for e ∈ examples end makedocs(; - modules = [IncompressibleNavierStokes, NeuralClosure], + modules = [IncompressibleNavierStokes, NeuralClosure], plugins = [bib], authors = "Syver Døving Agdestein, Benjamin Sanderse, and contributors", repo = "https://github.com/agdestein/IncompressibleNavierStokes.jl/blob/{commit}{path}#{line}", From badd0ebc4037c106c6c5d0a5af6581e3c58d1249 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Mon, 22 Apr 2024 15:50:33 +0200 Subject: [PATCH 296/379] Add 2D invariants --- src/operators.jl | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/src/operators.jl b/src/operators.jl index 5a85ff66c..4765ee362 100644 --- a/src/operators.jl +++ b/src/operators.jl @@ -930,8 +930,20 @@ Note that `B[1]` corresponds to ``T_0`` in the paper, and `V` to ``I``. """ function tensorbasis!(B, V, u, setup) (; grid, workgroupsize) = setup - (; Np, Ip, Δ, Δu) = grid - @kernel function basis!(B, V, u, I0) + (; Np, Ip, Δ, Δu, dimension) = grid + @kernel function basis!(::Dimension{2}, B, V, u, I0) + I = @index(Global, Cartesian) + I = I + I0 + ∇u = ∇(u, I, Δ, Δu) + S = (∇u + ∇u') / 2 + R = (∇u - ∇u') / 2 + B[1][I] = idtensor(u, I) + B[2][I] = S + B[3][I] = S * R - R * S + V[1][I] = tr(S * S) + V[2][I] = tr(R * R) + end + @kernel function basis!(::Dimension{3}, B, V, u, I0) I = @index(Global, Cartesian) I = I + I0 ∇u = ∇(u, I, Δ, Δu) @@ -956,7 +968,7 @@ function tensorbasis!(B, V, u, setup) end I0 = first(Ip) I0 -= oneunit(I0) - basis!(get_backend(u[1]), workgroupsize)(B, V, u, I0; ndrange = Np) + basis!(get_backend(u[1]), workgroupsize)(dimension, B, V, u, I0; ndrange = Np) B, V end From 8f4e9195c83eee970acaa05125f59e2dca096f35 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Mon, 22 Apr 2024 15:51:13 +0200 Subject: [PATCH 297/379] Add missing conditios --- src/solvers/pressure/solvers.jl | 25 ++++++++++++++++++------- 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/src/solvers/pressure/solvers.jl b/src/solvers/pressure/solvers.jl index d4d738863..a5d554d21 100644 --- a/src/solvers/pressure/solvers.jl +++ b/src/solvers/pressure/solvers.jl @@ -58,8 +58,11 @@ struct DirectPressureSolver{T,S,F,A} <: AbstractPressureSolver{T} f = zeros(T, prod(Np) + 1) p = zeros(T, prod(Np) + 1) L = laplacian_mat(setup) - e = ones(T, size(L, 2)) - L = [L e; e' 0] + if any(bc -> bc[1] isa PressureBC || bc[2] isa PressureBC, boundary_conditions) + else + e = ones(T, size(L, 2)) + L = [L e; e' 0] + end # fact = lu(L) # fact = ldlt(Symmetric(L)) fact = factorize(L) @@ -87,20 +90,28 @@ struct CUDSSPressureSolver{T,S,F,A} <: AbstractPressureSolver{T} f::A p::A function CUDSSPressureSolver(setup) - (; grid, ArrayType) = setup + (; grid, boundary_conditions, ArrayType) = setup (; x, Np) = grid T = eltype(x[1]) @assert x[1] isa CuArray "CUDSSPressureSolver only works for CuArray." f = fill!(similar(x[1], prod(Np) + 1), 0) p = fill!(similar(x[1], prod(Np) + 1), 0) L = laplacian_mat(setup) - e = ones(T, size(L, 2)) - L = [L e; e' 0] + if any(bc -> bc[1] isa PressureBC || bc[2] isa PressureBC, boundary_conditions) + # structure = "SPD" # Symmetric positive definite + structure = "G" # General matrix + view = 'F' # Full matrix representation + else + e = ones(T, size(L, 2)) + L = [L e; e' 0] + structure = "S" # Symmetric (not positive definite) + view = 'L' # Lower triangular representation + end L = CuSparseMatrixCSR(L) solver = CudssSolver( L, - "S", # Symmetric (not positive definite) - 'L', # Lower triangular representation + structure, + view, # Lower triangular representation ) cudss("analysis", solver, p, f) cudss("factorization", solver, p, f) # Compute factorization From 6a26d73c6200891f1808e5b5c69a08707b7a7dd5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Mon, 22 Apr 2024 15:51:35 +0200 Subject: [PATCH 298/379] Bump version --- Project.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Project.toml b/Project.toml index 655ff39d5..2173b5028 100644 --- a/Project.toml +++ b/Project.toml @@ -25,7 +25,7 @@ WriteVTK = "64499a7a-5c06-52f2-abe2-ccb03c286192" [compat] Adapt = "3, 4" CUDA = "5" -CUDSS = "0.1" +CUDSS = "0.2" FFTW = "1" IterativeSolvers = "0.9" KernelAbstractions = "0.9" From f4d29e522b480c26d351e508f740c7ac991d367d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Mon, 22 Apr 2024 17:59:05 +0200 Subject: [PATCH 299/379] Update basis --- src/operators.jl | 10 +++++----- src/processors/real_time_plot.jl | 21 +++++++++++++++++++++ 2 files changed, 26 insertions(+), 5 deletions(-) diff --git a/src/operators.jl b/src/operators.jl index 4765ee362..5c0af08e4 100644 --- a/src/operators.jl +++ b/src/operators.jl @@ -951,9 +951,9 @@ function tensorbasis!(B, V, u, setup) R = (∇u - ∇u') / 2 B[1][I] = idtensor(u, I) B[2][I] = S - B[3][I] = S * S - B[4][I] = R * R - B[5][I] = S * R - R * S + B[3][I] = S * R - R * S + B[4][I] = S * S + B[5][I] = R * R B[6][I] = S * S * R - R * S * S B[7][I] = S * R * R + R * R * S B[8][I] = R * S * R * R - R * R * S * R @@ -981,8 +981,8 @@ function tensorbasis(u, setup) T = eltype(u[1]) D = setup.grid.dimension() tensorbasis!( - ntuple(k -> similar(u[1], SMatrix{D,D,T,D * D}, setup.grid.N), 11), - ntuple(k -> similar(u[1], setup.grid.N), 5), + ntuple(k -> similar(u[1], SMatrix{D,D,T,D * D}, setup.grid.N), D == 2 ? 3 : 11), + ntuple(k -> similar(u[1], setup.grid.N), D == 2 ? 2 : 5), u, setup, ) diff --git a/src/processors/real_time_plot.jl b/src/processors/real_time_plot.jl index 20380d217..dbf54198b 100644 --- a/src/processors/real_time_plot.jl +++ b/src/processors/real_time_plot.jl @@ -116,6 +116,12 @@ function fieldplot( F = zero.(u) div = zero(u[1]) p = zero(u[1]) + elseif fieldname == :V1 + B, V = tensorbasis(u, setup) + V[1] + elseif fieldname == :V2 + B, V = tensorbasis(u, setup) + V[2] end _f = Array(_f)[Ip] field = lift(state) do (; u, t) @@ -134,6 +140,12 @@ function fieldplot( get_streamfunction!(setup, ψ, u, t) elseif fieldname == :pressure pressure!(p, u, t, setup; psolver, F, div) + elseif fieldname == :V1 + tensorbasis!(B, V, u, setup) + V[1] + elseif fieldname == :V2 + tensorbasis!(B, V, u, setup) + -V[2] end # Array(f)[Ip] copyto!(_f, view(f, Ip)) @@ -238,6 +250,12 @@ function fieldplot( Q = similar(u[1]) elseif fieldname == :eig2field λ = similar(u[1]) + elseif fieldname in union(Symbol.(["B$i" for i in 1:11]), Symbol.(["V$i" for i in 1:5])) + sym = string(fieldname)[1] + sym = sym == 'B' ? 1 : 2 + idx = parse(Int, string(fieldname)[2:end]) + tb = tensorbasis(u, setup) + tb[sym][idx] else error("Unknown fieldname") end @@ -269,6 +287,9 @@ function fieldplot( λin = view(λ, Ip) @. λin .= log(max(logtol, -λin)) λ + elseif fieldname in union(Symbol.(["B$i" for i in 1:11]), Symbol.(["V$i" for i in 1:5])) + tensorbasis!(tb..., u, setup) + tb[sym][idx] end Array(f)[Ip] end From e300aad38a2b6ae9ae2c96014d70f21fdf9dfc97 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Mon, 22 Apr 2024 18:00:57 +0200 Subject: [PATCH 300/379] Add script --- TensorClosure/Project.toml | 19 ++++++++++++ TensorClosure/main.jl | 59 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 78 insertions(+) create mode 100644 TensorClosure/Project.toml create mode 100644 TensorClosure/main.jl diff --git a/TensorClosure/Project.toml b/TensorClosure/Project.toml new file mode 100644 index 000000000..316021e0a --- /dev/null +++ b/TensorClosure/Project.toml @@ -0,0 +1,19 @@ +[deps] +Adapt = "79e6a3ab-5dfb-504d-930d-738a2a938a0e" +CUDA = "052768ef-5323-5732-b1bb-66c8b64840ba" +CairoMakie = "13f3f980-e62b-5c42-98c6-ff1f3baf88f0" +FFTW = "7a1cc6ca-52ef-59f5-83cd-3a7055c09341" +GLMakie = "e9467ef8-e4e7-5192-8a1a-b1aee30e663a" +IncompressibleNavierStokes = "5e318141-6589-402b-868d-77d7df8c442e" +JLD2 = "033835bb-8acc-5ee8-8aae-3f567f8a3819" +LaTeXStrings = "b964fa9f-0449-5b57-a5c2-d3ea65f4040f" +LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" +Lux = "b2108857-7c20-44ae-9111-449ecde12c47" +LuxCUDA = "d0bbae9a-e099-4d5b-a835-1c6931763bda" +NNlib = "872c559c-99b0-510c-b3b7-b6c96a88d5cd" +NeuralClosure = "099dac27-d7f2-4047-93d5-0baee36b9c25" +Observables = "510215fc-4207-5dde-b226-833fc4488ee2" +Optimisers = "3bd65402-5787-11e9-1adc-39752487f4e2" +Printf = "de0858da-6303-5e67-8744-51eddeeeb8d7" +Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" +SparseArrays = "2f01184e-e22b-5df5-ae63-d93ebab69eaf" diff --git a/TensorClosure/main.jl b/TensorClosure/main.jl new file mode 100644 index 000000000..57df0c5fd --- /dev/null +++ b/TensorClosure/main.jl @@ -0,0 +1,59 @@ +using CairoMakie +using IncompressibleNavierStokes + +# For running on GPU +using CUDA; +CUDA.allowscalar(false); +ArrayType = CuArray; +T = Float32 + +# Setup +Re = T(10_000) +n = 1024 +lims = T(0), T(1) +x = LinRange(lims..., n + 1), LinRange(lims..., n + 1) +setup = Setup(x...; Re, ArrayType); +psolver = SpectralPressureSolver(setup); +u₀ = random_field(setup, T(0); psolver); + +u = u₀ + +B, V = IncompressibleNavierStokes.tensorbasis(u, setup) +B |> length +CUDA.@allowscalar B[2][5, 10] +getindex.(B[2], 2) + +heatmap(Array(u[1])) +u[1] |> Array |> heatmap +getindex.(B[3], 1, 1) |> Array |> heatmap +getindex.(B[3], 2, 1) |> Array |> heatmap +getindex.(B[3], 2, 2) |> Array |> heatmap +V[1] |> Array |> heatmap +V[2] |> Array |> heatmap +V[2] |> Array |> contourf + +# Solve unsteady problem +state, outputs = solve_unsteady( + setup, + u₀, + (T(0), T(1)); + Δt = T(1e-4), + psolver, + processors = ( + # rtp = realtimeplotter(; setup, nupdate = 1), + log = timelogger(; nupdate = 10), + ), +); +(; u) = state + +makeplot(u, fieldname, time) = save( + "output/fieldplots/$fieldname$time.png", + fieldplot((; u, t = T(0)); setup, fieldname, docolorbar = false, size = (500, 500)), +) + +makeplot(u₀, :vorticity, 0) +makeplot(u₀, :S2, 0) +makeplot(u₀, :R2, 0) +makeplot(u, :vorticity, 1) +makeplot(u, :S2, 1) +makeplot(u, :R2, 1) From e2b660288ef3371335e80b88f358b186e6915416 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Thu, 25 Apr 2024 11:57:27 +0200 Subject: [PATCH 301/379] Move weird integrator to paper only --- PaperDC/postanalysis.jl | 208 +++--------------- PaperDC/src/PaperDC.jl | 3 + PaperDC/src/rk.jl | 149 +++++++++++++ libs/NeuralClosure/src/training.jl | 1 - .../step_explicit_runge_kutta.jl | 42 +--- 5 files changed, 185 insertions(+), 218 deletions(-) create mode 100644 PaperDC/src/rk.jl diff --git a/PaperDC/postanalysis.jl b/PaperDC/postanalysis.jl index 0dc8ef23a..822637701 100644 --- a/PaperDC/postanalysis.jl +++ b/PaperDC/postanalysis.jl @@ -278,8 +278,7 @@ closure(device(io_train[1, 1].u[:, :, :, 1:50]), device(θ₀)); # A-priori training ########################################################### -ispath("$savepath/priortraining") || mkpath("$savepath/priortraining") -for ifil = 1:1, ig = 4:4 +for ifil = 1:2, ig = 4:4 starttime = time() println("ig = $ig, ifil = $ifil") d = create_dataloader_prior(io_train[ig, ifil]; batchsize = 50, device) @@ -307,14 +306,14 @@ for ifil = 1:1, ig = 4:4 ) θ = callbackstate.θmin # Use best θ instead of last θ prior = (; θ = Array(θ), comptime = time() - starttime, callbackstate.hist) - jldsave("$savepath/priortraining/ifilter$(ifil)_igrid$(ig).jld2"; prior) + jldsave("$savepath/prior_ifilter$(ifil)_igrid$(ig).jld2"; prior) end clean() # Load trained parameters prior = map(CartesianIndices(size(io_train))) do I ig, ifil = I.I - name = "$savepath/priortraining/ifilter$(ifil)_igrid$(ig).jld2" + name = "$path/prior_ifilter$(ifil)_igrid$(ig).jld2" load(name)["prior"] end; θ_cnn_prior = [copyto!(device(θ₀), p.θ) for p in prior]; @@ -343,17 +342,13 @@ let loss = IncompressibleNavierStokes.create_loss_post(; setup, psolver, + method = RKProject(RK44(; T), getorder(iorder)), closure, nupdate = 2, - projectorder = getorder(iorder), ) data = [(; u = d.data[ig, ifil].u, d.t) for d in data_train] d = create_dataloader_post(data; device, nunroll = 20) - # θ = T(1e-1) * copy(θ_cnn_prior[ig, ifil]) θ = copy(θ_cnn_prior[ig, ifil]) - # θ = copy(θ_cnn_post) - # θ = copy(θ_cnn_post[ig, ifil, iorder]) - # θ = device(θ₀) opt = Optimisers.setup(Adam(T(1.0e-3)), θ) it = 1:30 data = data_valid[1] @@ -363,8 +358,8 @@ let data, setup, psolver, + method = RKProject(RK44(; T), getorder(iorder)), closure_model = wrappedclosure(closure, setup), - projectorder = getorder(iorder), nupdate = 2, ); θ, @@ -395,7 +390,7 @@ map(p -> p.comptime, post) |> sum map(p -> p.comptime, post) |> sum |> x -> x / 60 map(p -> p.comptime, post) |> sum |> x -> x / 3600 -# Train Smagorinsky model with Lpost (grid search) +# Train Smagorinsky model with a-posteriori error grid search smag = map(CartesianIndices((size(io_train, 2), 2))) do I starttime = time() ifil, iorder = I.I @@ -418,8 +413,8 @@ smag = map(CartesianIndices((size(io_train, 2), 2))) do I data, setup, psolver, + method = RKProject(RK44(; T), getorder(iorder)), closure_model = smagorinsky_closure(setup), - projectorder, nupdate, ) e += err(θ) @@ -448,8 +443,6 @@ smag = load("$outdir/smag.jld2")["smag"]; map(s -> s.comptime, smag) map(s -> s.comptime, smag) |> sum -# lines(LinRange(T(0), T(1), 100), e_smag) - # Compute a-priori errors ################################################### eprior = let @@ -476,19 +469,17 @@ eprior.post |> x -> reshape(x, :, 2) |> x -> round.(x; digits = 2) # Compute posterior errors #################################################### -e_nm, e_smag, e_cnn, e_cnn_post = let +(; e_nm, e_smag, e_cnn, e_cnn_post) = let e_nm = zeros(T, size(data_test.data)...) e_smag = zeros(T, size(data_test.data)..., 2) e_cnn = zeros(T, size(data_test.data)..., 2) e_cnn_post = zeros(T, size(data_test.data)..., 2) for iorder = 1:2, ifil = 1:2, ig = 1:size(data_test.data, 1) - # (ig, ifil, iorder) == (2, 2, 2) || continue println("iorder = $iorder, ifil = $ifil, ig = $ig") projectorder = getorder(iorder) setup = setups_test[ig] psolver = SpectralPressureSolver(setup) data = (; u = device.(data_test.data[ig, ifil].u), t = data_test.t) - # nupdate = ig > 3 ? 4 : 2 nupdate = 2 # No model # Only for closurefirst, since projectfirst is the same @@ -501,8 +492,8 @@ e_nm, e_smag, e_cnn, e_cnn_post = let data, setup, psolver, + method = RKProject(RK44(; T), getorder(iorder)), closure_model = smagorinsky_closure(setup), - projectorder, nupdate, ) e_smag[ig, ifil, iorder] = err(θ_smag[ifil, iorder]) @@ -513,22 +504,17 @@ e_nm, e_smag, e_cnn, e_cnn_post = let data, setup, psolver, + method = RKProject(RK44(; T), getorder(iorder)), closure_model = wrappedclosure(closure, setup), - projectorder, nupdate, - # nupdate = 50, ) e_cnn[ig, ifil, iorder] = err(θ_cnn_prior[ig, ifil]) e_cnn_post[ig, ifil, iorder] = err(θ_cnn_post[ig, ifil, iorder]) end end - e_nm, e_smag, e_cnn, e_cnn_post + (; e_nm, e_smag, e_cnn, e_cnn_post) end clean() -e_nm -e_smag -e_cnn -e_cnn_post round.( [e_nm[:] reshape(e_smag, :, 2) reshape(e_cnn, :, 2) reshape(e_cnn_post, :, 2)][ @@ -546,31 +532,19 @@ GLMakie.activate!() # Plot a-priori errors ######################################################## -fig = with_theme(; - # linewidth = 5, - # markersize = 10, - # markersize = 20, - # fontsize = 20, - palette, -) do +fig = with_theme(; palette) do nles = [n[1] for n in params_test.nles][1:3] ifil = 1 fig = Figure(; size = (500, 400)) ax = Axis( fig[1, 1]; xscale = log10, - # yscale = log10, xticks = nles, xlabel = "Resolution", - # title = "Relative a-priori error", title = "Relative a-priori error $(ifil == 1 ? " (FA)" : " (VA)")", ) linestyle = :solid - # for ifil = 1:2 - # linestyle = ifil == 1 ? :solid : :dash label = "No closure" - # label = label * (ifil == 1 ? " (FA)" : " (VA)") - # ifil == 2 && (label = nothing) scatterlines!( nles, ones(T, length(nles)); @@ -579,26 +553,7 @@ fig = with_theme(; marker = :circle, label, ) - # end - # for ifil = 1:2 - # linestyle = ifil == 1 ? :solid : :dash - # label = "Smagorinsky" - # # label = label * (ifil == 1 ? " (FA)" : " (VA)") - # ifil == 2 && (label = nothing) - # scatterlines!( - # nles, - # eprior.smag[:, ifil, iorder]; - # color = Cycled(2), - # linestyle, - # marker = :utriangle, - # label, - # ) - # end - # for ifil = 1:2 - # linestyle = ifil == 1 ? :solid : :dash label = "CNN (Lprior)" - # label = label * (ifil == 1 ? " (FA)" : " (VA)") - # ifil == 2 && (label = nothing) scatterlines!( nles, eprior.prior[:, ifil]; @@ -607,12 +562,7 @@ fig = with_theme(; marker = :utriangle, label, ) - # end - # for ifil = 1:2 - # linestyle = ifil == 1 ? :solid : :dash label = "CNN (Lpost, DIF)" - # label = label * (ifil == 1 ? " (FA)" : " (VA)") - # ifil == 2 && (label = nothing) scatterlines!( nles, eprior.post[:, ifil, 1]; @@ -621,12 +571,7 @@ fig = with_theme(; marker = :rect, label, ) - # end - # for ifil = 1:2 - # linestyle = ifil == 1 ? :solid : :dash label = "CNN (Lpost, DCF)" - # label = label * (ifil == 1 ? " (FA)" : " (VA)") - # ifil == 2 && (label = nothing) scatterlines!( nles, eprior.post[:, ifil, 2]; @@ -635,17 +580,8 @@ fig = with_theme(; marker = :diamond, label, ) - # end - # lines!( - # collect(extrema(nles[4:end])), - # n -> 2e4 * n^-2.0; - # linestyle = :dash, - # label = "n⁻²", - # color = Cycled(1), - # ) axislegend(; position = :lb) ylims!(ax, (T(-0.05), T(1.05))) - # iorder == 2 && limits!(ax, (T(60), T(1050)), (T(2e-2), T(1e1))) name = "$plotdir/convergence" ispath(name) || mkpath(name) save("$name/$(mname)_prior_ifilter$ifil.pdf", fig) @@ -654,19 +590,10 @@ end # Plot a-posteriori errors ################################################### -with_theme(; - # linewidth = 5, - # markersize = 10, - # markersize = 20, - # fontsize = 20, - palette, -) do +with_theme(; palette) do iorder = 2 - # lesmodel = iorder == 1 ? "project-then-closure" : "closure-then-project" - # lesmodel = iorder == 1 ? "project-then-closure" : "closure-then-project" lesmodel = iorder == 1 ? "DIF" : "DCF" ntrain = size(data_train[1].data, 1) - # nles = [n[1] for n in params_test.nles] nles = [n[1] for n in params_test.nles][1:ntrain] fig = Figure(; size = (500, 400)) ax = Axis( @@ -675,19 +602,14 @@ with_theme(; yscale = log10, xticks = nles, xlabel = "Resolution", - # xlabel = "n", - # xlabel = L"\bar{n}", - # title = "Relative error (DNS: n = $(params_test.ndns[1]))", title = "Relative error ($lesmodel)", ) for ifil = 1:2 linestyle = ifil == 1 ? :solid : :dash label = "No closure" - # label = label * (ifil == 1 ? " (FA)" : " (VA)") ifil == 2 && (label = nothing) scatterlines!( nles, - # e_nm[:, ifil]; e_nm[1:ntrain, ifil]; color = Cycled(1), linestyle, @@ -698,11 +620,9 @@ with_theme(; for ifil = 1:2 linestyle = ifil == 1 ? :solid : :dash label = "Smagorinsky" - # label = label * (ifil == 1 ? " (FA)" : " (VA)") ifil == 2 && (label = nothing) scatterlines!( nles, - # e_smag[:, ifil, iorder]; e_smag[1:ntrain, ifil, iorder]; color = Cycled(2), linestyle, @@ -713,7 +633,6 @@ with_theme(; for ifil = 1:2 linestyle = ifil == 1 ? :solid : :dash label = "CNN (prior)" - # label = label * (ifil == 1 ? " (FA)" : " (VA)") ifil == 2 && (label = nothing) scatterlines!( nles[1:ntrain], @@ -727,7 +646,6 @@ with_theme(; for ifil = 1:2 linestyle = ifil == 1 ? :solid : :dash label = "CNN (post)" - # label = label * (ifil == 1 ? " (FA)" : " (VA)") ifil == 2 && (label = nothing) scatterlines!( nles[1:ntrain], @@ -738,17 +656,8 @@ with_theme(; label, ) end - # lines!( - # collect(extrema(nles[4:end])), - # n -> 2e4 * n^-2.0; - # linestyle = :dash, - # label = "n⁻²", - # color = Cycled(1), - # ) - # axislegend(; position = :rt) axislegend(; position = :lb) ylims!(ax, (T(0.025), T(1.00))) - # iorder == 2 && limits!(ax, (T(60), T(1050)), (T(2e-2), T(1e1))) name = "$plotdir/convergence" ispath(name) || mkpath(name) save("$name/$(mname)_iorder$iorder.pdf", fig) @@ -772,7 +681,6 @@ kineticenergy = let t = data_test.t u₀ = data_test.data[ig, ifil].u[1] |> device tlims = (t[1], t[end]) - # nupdate = 50 nupdate = 2 Δt = (t[2] - t[1]) / nupdate T = eltype(u₀[1]) @@ -844,24 +752,14 @@ kineticenergy = let end; clean(); -kineticenergy.ke_ref[1] -kineticenergy.ke_nomodel[1] -kineticenergy.ke_smag[1] -kineticenergy.ke_cnn_prior[1] -kineticenergy.ke_cnn_post[1] +# Plot energy evolution ######################################################## CairoMakie.activate!() -# Plot energy evolution ######################################################## - with_theme(; palette) do - # t = data_test.t[2:end] t = data_test.t for iorder = 1:2, ifil = 1:2, igrid = 1:3 println("iorder = $iorder, ifil = $ifil, igrid = $igrid") - # reflevel = kineticenergy.ke_ref[igrid, ifil][2:end] - reflevel = copy(kineticenergy.ke_ref[igrid, ifil]) - reflevel = fill!(reflevel, 1) lesmodel = iorder == 1 ? "DIF" : "DCF" fil = ifil == 1 ? "FA" : "VA" nles = params_test.nles[igrid] @@ -870,14 +768,12 @@ with_theme(; palette) do fig[1, 1]; xlabel = "t", ylabel = "E(t)", - # title = "Kinetic energy: $lesmodel, $fil, $nles", title = "Kinetic energy: $lesmodel, $fil", ) lines!( ax, t, - # kineticenergy.ke_ref[igrid, ifil][2:end] ./ reflevel; - kineticenergy.ke_ref[igrid, ifil] ./ reflevel; + kineticenergy.ke_ref[igrid, ifil]; color = Cycled(1), linestyle = :dash, label = "Reference", @@ -885,35 +781,33 @@ with_theme(; palette) do lines!( ax, t, - kineticenergy.ke_nomodel[igrid, ifil] ./ reflevel; + kineticenergy.ke_nomodel[igrid, ifil]; color = Cycled(1), label = "No closure", ) lines!( ax, t, - kineticenergy.ke_smag[igrid, ifil, iorder] ./ reflevel; + kineticenergy.ke_smag[igrid, ifil, iorder]; color = Cycled(2), label = "Smagorinsky", ) lines!( ax, t, - kineticenergy.ke_cnn_prior[igrid, ifil, iorder] ./ reflevel; + kineticenergy.ke_cnn_prior[igrid, ifil, iorder]; color = Cycled(3), label = "CNN (prior)", ) lines!( ax, t, - kineticenergy.ke_cnn_post[igrid, ifil, iorder] ./ reflevel; + kineticenergy.ke_cnn_post[igrid, ifil, iorder]; color = Cycled(4), label = "CNN (post)", ) iorder == 1 && axislegend(; position = :lt) iorder == 2 && axislegend(; position = :lb) - # axislegend(; position = :lb) - # axislegend() name = "$plotdir/energy_evolution/$mname/" ispath(name) || mkpath(name) save("$(name)/iorder$(iorder)_ifilter$(ifil)_igrid$(igrid).pdf", fig) @@ -937,7 +831,6 @@ divs = let t = data_test.t u₀ = data_test.data[ig, ifil].u[1] |> device tlims = (t[1], t[end]) - # nupdate = 50 nupdate = 2 Δt = (t[2] - t[1]) / nupdate T = eltype(u₀[1]) @@ -970,9 +863,10 @@ divs = let iorder_use = iorder == 3 ? 2 : iorder d_nomodel[ig, ifil, iorder] = solve_unsteady( - (setup..., projectorder = getorder(iorder)), + setup, u₀, tlims; + method = RKProject(RK44(; T), getorder(iorder)), Δt, processors, psolver, @@ -981,11 +875,11 @@ divs = let solve_unsteady( (; setup..., - projectorder = getorder(iorder), closure_model = smagorinsky_closure(setup), ), u₀, tlims; + method = RKProject(RK44(; T), getorder(iorder)), Δt, processors, psolver, @@ -995,11 +889,11 @@ divs = let solve_unsteady( (; setup..., - projectorder = getorder(iorder), closure_model = wrappedclosure(closure, setup), ), u₀, tlims; + method = RKProject(RK44(; T), getorder(iorder)), Δt, processors, psolver, @@ -1009,11 +903,11 @@ divs = let solve_unsteady( (; setup..., - projectorder = getorder(iorder), closure_model = wrappedclosure(closure, setup), ), u₀, tlims; + method = RKProject(RK44(; T), getorder(iorder)), Δt, processors, psolver, @@ -1054,7 +948,6 @@ with_theme(; t = data_test.t for islog in (true, false) for iorder = 1:3, ifil = 1:2, igrid = 1:3 - # println("iorder = $iorder, igrid = $igrid") println("iorder = $iorder, ifil = $ifil, igrid = $igrid") lesmodel = if iorder == 1 "DIF" @@ -1071,10 +964,7 @@ with_theme(; fig[1, 1]; yscale, xlabel = "t", - # ylabel = "Dv", - # title = "Divergence: $lesmodel, $nles", title = "Divergence: $lesmodel, $fil, $nles", - # title = "Divergence: $lesmodel, $fil", ) linestyle = ifil == 1 ? :solid : :dash lines!( @@ -1113,15 +1003,10 @@ with_theme(; color = Cycled(4), label = "CNN (post)", ) - # axislegend() - # iorder == 1 && axislegend(; position = :lt) - # iorder == 2 && axislegend(; position = :lb) iorder == 2 && ifil == 1 && axislegend(; position = :rt) - # axislegend() islog && ylims!(ax, (T(1e-6), T(1e3))) name = "$plotdir/divergence/$mname/$(islog ? "log" : "lin")" ispath(name) || mkpath(name) - # save("$(name)/iorder$(iorder)_igrid$(igrid).pdf", fig) save("$(name)/iorder$(iorder)_ifilter$(ifil)_igrid$(igrid).pdf", fig) end end @@ -1158,11 +1043,11 @@ ufinal = let solve_unsteady( (; setup..., - projectorder = getorder(iorder), closure_model = smagorinsky_closure(setup), ), u₀, tlims; + method = RKProject(RK44(; T), getorder(iorder)), Δt, psolver, θ = θ_smag[ifil, iorder], @@ -1171,11 +1056,11 @@ ufinal = let solve_unsteady( (; setup..., - projectorder = getorder(iorder), closure_model = wrappedclosure(closure, setup), ), u₀, tlims; + method = RKProject(RK44(; T), getorder(iorder)), Δt, psolver, θ = θ_cnn_prior[igrid, ifil], @@ -1184,11 +1069,11 @@ ufinal = let solve_unsteady( (; setup..., - projectorder = getorder(iorder), closure_model = wrappedclosure(closure, setup), ), u₀, tlims; + method = RKProject(RK44(; T), getorder(iorder)), Δt, psolver, θ = θ_cnn_post[igrid, ifil, iorder], @@ -1198,39 +1083,10 @@ ufinal = let end; clean(); -ufinal.u_ref[1][2] -ufinal.u_cnn_prior[3, 1, 1][1] -ufinal.u_cnn_post[3, 1, 1][1] - jldsave("$savepath/ufinal.jld2"; ufinal) ufinal = load("$savepath/ufinal.jld2")["ufinal"]; -markers_labels = [ - (:circle, ":circle"), - (:rect, ":rect"), - (:diamond, ":diamond"), - (:hexagon, ":hexagon"), - (:cross, ":cross"), - (:xcross, ":xcross"), - (:utriangle, ":utriangle"), - (:dtriangle, ":dtriangle"), - (:ltriangle, ":ltriangle"), - (:rtriangle, ":rtriangle"), - (:pentagon, ":pentagon"), - (:star4, ":star4"), - (:star5, ":star5"), - (:star6, ":star6"), - (:star8, ":star8"), - (:vline, ":vline"), - (:hline, ":hline"), - ('a', "'a'"), - ('B', "'B'"), - ('↑', "'\\uparrow'"), - ('😄', "'\\:smile:'"), - ('✈', "'\\:airplane:'"), -] - # Plot spectra ############################################################### fig = with_theme(; palette) do @@ -1251,14 +1107,11 @@ fig = with_theme(; palette) do (; Ip) = setup.grid (; A, κ, K) = IncompressibleNavierStokes.spectral_stuff(setup) specs = map(fields) do u - # up = interpolate_u_p(u, setup) up = u e = sum(up) do u u = u[Ip] uhat = fft(u)[ntuple(α -> 1:K[α], 2)...] - # abs2.(uhat) abs2.(uhat) ./ (2 * prod(size(u))^2) - # abs2.(uhat) ./ size(u, 1) end e = A * reshape(e, :) # e = max.(e, eps(T)) # Avoid log(0) @@ -1279,9 +1132,7 @@ fig = with_theme(; palette) do ax = Axis( fig[1, 1]; xticks, - # xlabel = "k", xlabel = "κ", - # ylabel = "e(κ)", xscale = log10, yscale = log10, limits = (1, kmax, T(1e-8), T(1)), @@ -1293,10 +1144,8 @@ fig = with_theme(; palette) do lines!(ax, κ, specs[5]; color = Cycled(4), label = "CNN (post)") lines!(ax, κ, specs[1]; color = Cycled(1), linestyle = :dash, label = "Reference") lines!(ax, krange, inertia; color = Cycled(1), label = slopelabel, linestyle = :dot) - # axislegend(ax; position = :lb) axislegend(ax; position = :cb) autolimits!(ax) - # limits!(ax, (T(0.8), T(800)), (T(1e-10), T(1))) ylims!(ax, (T(1e-3), T(0.35))) name = "$plotdir/energy_spectra/$mname" ispath(name) || mkpath(name) @@ -1320,7 +1169,6 @@ with_theme(; fontsize = 25, palette) do Point2f(x2, y2), Point2f(x1, y2), Point2f(x1, y1), - # Point2f(x2, y1), ] path = "$plotdir/les_fields/$mname" ispath(path) || mkpath(path) @@ -1335,8 +1183,6 @@ with_theme(; fontsize = 25, palette) do (; u, t = T(0)); setup, title, - # type = image, - # colormap = :viridis, docolorbar = false, size = (500, 500), ) diff --git a/PaperDC/src/PaperDC.jl b/PaperDC/src/PaperDC.jl index e7e309d4b..a03a863f7 100644 --- a/PaperDC/src/PaperDC.jl +++ b/PaperDC/src/PaperDC.jl @@ -88,6 +88,9 @@ observe_u(dns, psolver_dns, filters; nupdate = 1) = results end +include("rk.jl") + export observe_u, observe_v +export RKProject end # module PaperDC diff --git a/PaperDC/src/rk.jl b/PaperDC/src/rk.jl new file mode 100644 index 000000000..42caf3e06 --- /dev/null +++ b/PaperDC/src/rk.jl @@ -0,0 +1,149 @@ +""" + RKProject(rk, projectorder) + +Runge-Kutta method with different projection order. +The Runge-Kutta method `rk` can be for example `RK44()`. + +- `projetorder = :first`: Project RHS before applying closure term. +- `projetorder = :second`: Project RHS after applying closure term. +- `projetorder = :last`: Project solution instead of RHS (same as `rk`). +""" +struct RKProject{T,R} <: IncompressibleNavierStokes.AbstractODEMethod{T} + rk::R + projectorder::Symbol + RKProject(rk, projectorder) = new{eltype(rk.A),typeof(rk)}(rk, projectorder) +end + +ode_method_cache(method::RKProject, setup, u) = ode_method_cache(method.rk, setup, u) + +create_stepper(method::RKProject; setup, psolver, u, t, n = 0) = + create_stepper(method.rk; setup, psolver, u, t, n) + +function timestep!(method::RKProject, stepper, Δt; θ = nothing, cache) + (; setup, psolver, u, t, n) = stepper + (; grid, closure_model) = setup + (; dimension, Iu) = grid + (; rk, projectorder) = method + (; A, b, c) = rk + (; u₀, ku, div, p) = cache + D = dimension() + nstage = length(b) + m = closure_model + projectorder ∈ (:first, :second, :last) || error("Unknown projectorder: $projectorder") + + # Update current solution + t₀ = t + copyto!.(u₀, u) + + for i = 1:nstage + # Compute force at current stage i + apply_bc_u!(u, t, setup) + momentum!(ku[i], u, t, setup) + + # Project F first + if projectorder == :first + apply_bc_u!(ku[i], t, setup; dudt = true) + project!(ku[i], setup; psolver, div, p) + end + + # Add closure term + isnothing(m) || map((k, m) -> k .+= m, ku[i], m(u, θ)) + + # Project F second + if projectorder == :second + apply_bc_u!(ku[i], t, setup; dudt = true) + project!(ku[i], setup; psolver, div, p) + end + + # Intermediate time step + t = t₀ + c[i] * Δt + + # Apply stage forces + for α = 1:D + u[α] .= u₀[α] + for j = 1:i + @. u[α] += Δt * A[i, j] * ku[j][α] + # @. u[α][Iu[α]] += Δt * A[i, j] * ku[j][α][Iu[α]] + end + end + + # Project stage u directly + # Make velocity divergence free at time t + if projectorder == :last + apply_bc_u!(u, t, setup) + project!(u, setup; psolver, div, p) + end + end + + # This is redundant, but Neumann BC need to have _exact_ copies + # since we divide by an infinitely thin (eps(T)) volume width in the + # diffusion term + apply_bc_u!(u, t, setup) + + create_stepper(method; setup, psolver, u, t, n = n + 1) +end + +function timestep(method::RKProject, stepper, Δt; θ = nothing) + (; setup, psolver, u, t, n) = stepper + (; grid, closure_model) = setup + (; dimension) = grid + (; rk, projectorder) = method + (; A, b, c) = rk + D = dimension() + nstage = length(b) + m = closure_model + projectorder ∈ (:first, :second, :last) || error("Unknown projectorder: $projectorder") + + # Update current solution (does not depend on previous step size) + t₀ = t + u₀ = u + ku = () + + for i = 1:nstage + # Compute force at current stage i + u = apply_bc_u(u, t, setup) + F = momentum(u, t, setup) + + # Project F first + if projectorder == :first + F = apply_bc_u(F, t, setup; dudt = true) + F = project(F, setup; psolver) + end + + # Add closure term + isnothing(m) || (F = F .+ m(u, θ)) + + # Project F second + if projectorder == :second + F = apply_bc_u(F, t, setup; dudt = true) + F = project(F, setup; psolver) + end + + # Store right-hand side of stage i + ku = (ku..., F) + + # Intermediate time step + t = t₀ + c[i] * Δt + + # Apply stage forces + u = u₀ + for j = 1:i + u = @. u + Δt * A[i, j] * ku[j] + # u = tupleadd(u, @.(Δt * A[i, j] * ku[j])) + end + + # Project stage u directly + # Make velocity divergence free at time t + if projectorder == :last + u = apply_bc_u(u, t, setup) + u = project(u, setup; psolver) + end + end + + # This is redundant, but Neumann BC need to have _exact_ copies + # since we divide by an infinitely thin (eps(T)) volume width in the + # diffusion term + u = apply_bc_u(u, t, setup) + + create_stepper(method; setup, psolver, u, t, n = n + 1) +end diff --git a/libs/NeuralClosure/src/training.jl b/libs/NeuralClosure/src/training.jl index 35e559616..9b30a3df0 100644 --- a/libs/NeuralClosure/src/training.jl +++ b/libs/NeuralClosure/src/training.jl @@ -150,7 +150,6 @@ end psolver, closure_model, nupdate = 1, - projectorder = :last, ) Create a-posteriori relative error. diff --git a/src/time_steppers/step_explicit_runge_kutta.jl b/src/time_steppers/step_explicit_runge_kutta.jl index 21f5f5ba4..bf2f77751 100644 --- a/src/time_steppers/step_explicit_runge_kutta.jl +++ b/src/time_steppers/step_explicit_runge_kutta.jl @@ -3,14 +3,13 @@ create_stepper(::ExplicitRungeKuttaMethod; setup, psolver, u, t, n = 0) = function timestep!(method::ExplicitRungeKuttaMethod, stepper, Δt; θ = nothing, cache) (; setup, psolver, u, t, n) = stepper - (; grid, closure_model, projectorder) = setup + (; grid, closure_model) = setup (; dimension, Iu) = grid (; A, b, c) = method (; u₀, ku, div, p) = cache D = dimension() nstage = length(b) m = closure_model - projectorder ∈ (:first, :second, :last) || error("Unknown projectorder: $projectorder") # Update current solution t₀ = t @@ -21,21 +20,9 @@ function timestep!(method::ExplicitRungeKuttaMethod, stepper, Δt; θ = nothing, apply_bc_u!(u, t, setup) momentum!(ku[i], u, t, setup) - # Project F first - if projectorder == :first - apply_bc_u!(ku[i], t, setup; dudt = true) - project!(ku[i], setup; psolver, div, p) - end - # Add closure term isnothing(m) || map((k, m) -> k .+= m, ku[i], m(u, θ)) - # Project F second - if projectorder == :second - apply_bc_u!(ku[i], t, setup; dudt = true) - project!(ku[i], setup; psolver, div, p) - end - # Intermediate time step t = t₀ + c[i] * Δt @@ -50,10 +37,8 @@ function timestep!(method::ExplicitRungeKuttaMethod, stepper, Δt; θ = nothing, # Project stage u directly # Make velocity divergence free at time t - if projectorder == :last - apply_bc_u!(u, t, setup) - project!(u, setup; psolver, div, p) - end + apply_bc_u!(u, t, setup) + project!(u, setup; psolver, div, p) end # This is redundant, but Neumann BC need to have _exact_ copies @@ -66,13 +51,12 @@ end function timestep(method::ExplicitRungeKuttaMethod, stepper, Δt; θ = nothing) (; setup, psolver, u, t, n) = stepper - (; grid, closure_model, projectorder) = setup + (; grid, closure_model) = setup (; dimension) = grid (; A, b, c) = method D = dimension() nstage = length(b) m = closure_model - projectorder ∈ (:first, :second, :last) || error("Unknown projectorder: $projectorder") # Update current solution (does not depend on previous step size) t₀ = t @@ -84,21 +68,9 @@ function timestep(method::ExplicitRungeKuttaMethod, stepper, Δt; θ = nothing) u = apply_bc_u(u, t, setup) F = momentum(u, t, setup) - # Project F first - if projectorder == :first - F = apply_bc_u(F, t, setup; dudt = true) - F = project(F, setup; psolver) - end - # Add closure term isnothing(m) || (F = F .+ m(u, θ)) - # Project F second - if projectorder == :second - F = apply_bc_u(F, t, setup; dudt = true) - F = project(F, setup; psolver) - end - # Store right-hand side of stage i ku = (ku..., F) @@ -114,10 +86,8 @@ function timestep(method::ExplicitRungeKuttaMethod, stepper, Δt; θ = nothing) # Project stage u directly # Make velocity divergence free at time t - if projectorder == :last - u = apply_bc_u(u, t, setup) - u = project(u, setup; psolver) - end + u = apply_bc_u(u, t, setup) + u = project(u, setup; psolver) end # This is redundant, but Neumann BC need to have _exact_ copies From 30b2bba33d3754f52632ae2e0ee07308208cdde8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Thu, 25 Apr 2024 11:57:58 +0200 Subject: [PATCH 302/379] Fix naming conflict --- src/operators.jl | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/operators.jl b/src/operators.jl index 5c0af08e4..90a225af2 100644 --- a/src/operators.jl +++ b/src/operators.jl @@ -931,7 +931,7 @@ Note that `B[1]` corresponds to ``T_0`` in the paper, and `V` to ``I``. function tensorbasis!(B, V, u, setup) (; grid, workgroupsize) = setup (; Np, Ip, Δ, Δu, dimension) = grid - @kernel function basis!(::Dimension{2}, B, V, u, I0) + @kernel function basis2!(B, V, u, I0) I = @index(Global, Cartesian) I = I + I0 ∇u = ∇(u, I, Δ, Δu) @@ -943,7 +943,7 @@ function tensorbasis!(B, V, u, setup) V[1][I] = tr(S * S) V[2][I] = tr(R * R) end - @kernel function basis!(::Dimension{3}, B, V, u, I0) + @kernel function basis3!(B, V, u, I0) I = @index(Global, Cartesian) I = I + I0 ∇u = ∇(u, I, Δ, Δu) @@ -968,6 +968,7 @@ function tensorbasis!(B, V, u, setup) end I0 = first(Ip) I0 -= oneunit(I0) + basis! = D == 2 ? basis2! : basis3! basis!(get_backend(u[1]), workgroupsize)(dimension, B, V, u, I0; ndrange = Np) B, V end From 07779a8b01e30612048d7fb7adca25c9eff91561 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Thu, 25 Apr 2024 11:58:46 +0200 Subject: [PATCH 303/379] chore: Format --- PaperDC/postanalysis.jl | 30 ++++++------------------------ 1 file changed, 6 insertions(+), 24 deletions(-) diff --git a/PaperDC/postanalysis.jl b/PaperDC/postanalysis.jl index 822637701..bb1b006c0 100644 --- a/PaperDC/postanalysis.jl +++ b/PaperDC/postanalysis.jl @@ -873,10 +873,7 @@ divs = let )[2].dwriter d_smag[ig, ifil, iorder] = solve_unsteady( - (; - setup..., - closure_model = smagorinsky_closure(setup), - ), + (; setup..., closure_model = smagorinsky_closure(setup)), u₀, tlims; method = RKProject(RK44(; T), getorder(iorder)), @@ -887,10 +884,7 @@ divs = let )[2].dwriter d_cnn_prior[ig, ifil, iorder] = solve_unsteady( - (; - setup..., - closure_model = wrappedclosure(closure, setup), - ), + (; setup..., closure_model = wrappedclosure(closure, setup)), u₀, tlims; method = RKProject(RK44(; T), getorder(iorder)), @@ -901,10 +895,7 @@ divs = let )[2].dwriter d_cnn_post[ig, ifil, iorder] = solve_unsteady( - (; - setup..., - closure_model = wrappedclosure(closure, setup), - ), + (; setup..., closure_model = wrappedclosure(closure, setup)), u₀, tlims; method = RKProject(RK44(; T), getorder(iorder)), @@ -1041,10 +1032,7 @@ ufinal = let end u_smag[igrid, ifil, iorder] = solve_unsteady( - (; - setup..., - closure_model = smagorinsky_closure(setup), - ), + (; setup..., closure_model = smagorinsky_closure(setup)), u₀, tlims; method = RKProject(RK44(; T), getorder(iorder)), @@ -1054,10 +1042,7 @@ ufinal = let )[1].u .|> Array u_cnn_prior[igrid, ifil, iorder] = solve_unsteady( - (; - setup..., - closure_model = wrappedclosure(closure, setup), - ), + (; setup..., closure_model = wrappedclosure(closure, setup)), u₀, tlims; method = RKProject(RK44(; T), getorder(iorder)), @@ -1067,10 +1052,7 @@ ufinal = let )[1].u .|> Array u_cnn_post[igrid, ifil, iorder] = solve_unsteady( - (; - setup..., - closure_model = wrappedclosure(closure, setup), - ), + (; setup..., closure_model = wrappedclosure(closure, setup)), u₀, tlims; method = RKProject(RK44(; T), getorder(iorder)), From 2c42dbb56a7fb3ee653846f4a97a501318984936 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Thu, 25 Apr 2024 13:18:23 +0200 Subject: [PATCH 304/379] refactor(PaperDC): Split into file --- PaperDC/src/PaperDC.jl | 81 +----------------------------------------- PaperDC/src/observe.jl | 79 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 80 insertions(+), 80 deletions(-) create mode 100644 PaperDC/src/observe.jl diff --git a/PaperDC/src/PaperDC.jl b/PaperDC/src/PaperDC.jl index a03a863f7..b23d5b724 100644 --- a/PaperDC/src/PaperDC.jl +++ b/PaperDC/src/PaperDC.jl @@ -8,86 +8,7 @@ using IncompressibleNavierStokes: momentum!, divergence!, project!, apply_bc_u! using Observables using LinearAlgebra -function observe_v(dnsobs, Φ, les, compression, psolver) - (; grid) = les - (; dimension, N, Iu, Ip) = grid - D = dimension() - Mα = N[1] - 2 - v = zero.(Φ(dnsobs[].u, les, compression)) - Pv = zero.(v) - p = zero(v[1]) - div = zero(p) - ΦPF = zero.(v) - PFΦ = zero.(v) - c = zero.(v) - T = eltype(v[1]) - results = (; - Φ, - Mα, - t = zeros(T, 0), - Dv = zeros(T, 0), - Pv = zeros(T, 0), - Pc = zeros(T, 0), - c = zeros(T, 0), - ) - on(dnsobs) do (; u, PF, t) - push!(results.t, t) - - Φ(v, u, les, compression) - apply_bc_u!(v, t, les) - Φ(ΦPF, PF, les, compression) - momentum!(PFΦ, v, t, les) - apply_bc_u!(PFΦ, t, les; dudt = true) - project!(PFΦ, les; psolver, div, p) - foreach(α -> c[α] .= ΦPF[α] .- PFΦ[α], 1:D) - apply_bc_u!(c, t, les) - divergence!(div, v, les) - norm_Du = norm(div[Ip]) - norm_v = sqrt(sum(α -> sum(abs2, v[α][Iu[α]]), 1:D)) - push!(results.Dv, norm_Du / norm_v) - - copyto!.(Pv, v) - project!(Pv, les; psolver, div, p) - foreach(α -> Pv[α] .= Pv[α] .- v[α], 1:D) - norm_vmPv = sqrt(sum(α -> sum(abs2, Pv[α][Iu[α]]), 1:D)) - push!(results.Pv, norm_vmPv / norm_v) - - Pc = Pv - copyto!.(Pc, c) - project!(Pc, les; psolver, div, p) - foreach(α -> Pc[α] .= Pc[α] .- c[α], 1:D) - norm_cmPc = sqrt(sum(α -> sum(abs2, Pc[α][Iu[α]]), 1:D)) - norm_c = sqrt(sum(α -> sum(abs2, c[α][Iu[α]]), 1:D)) - push!(results.Pc, norm_cmPc / norm_c) - - norm_ΦPF = sqrt(sum(α -> sum(abs2, ΦPF[α][Iu[α]]), 1:D)) - push!(results.c, norm_c / norm_ΦPF) - end - results -end - -observe_u(dns, psolver_dns, filters; nupdate = 1) = - processor() do state - PF = zero.(state[].u) - div = zero(state[].u[1]) - p = zero(state[].u[1]) - dnsobs = Observable((; state[].u, PF, state[].t)) - results = [ - observe_v(dnsobs, Φ, setup, compression, psolver) for - (; setup, Φ, compression, psolver) in filters - ] - on(state) do (; u, t, n) - n % nupdate == 0 || return - apply_bc_u!(u, t, dns) - momentum!(PF, u, t, dns) - apply_bc_u!(PF, t, dns; dudt = true) - project!(PF, dns; psolver = psolver_dns, div, p) - dnsobs[] = (; u, PF, t) - end - # state[] = state[] # Save initial conditions - results - end - +include("observe.jl") include("rk.jl") export observe_u, observe_v diff --git a/PaperDC/src/observe.jl b/PaperDC/src/observe.jl new file mode 100644 index 000000000..8fa53f59e --- /dev/null +++ b/PaperDC/src/observe.jl @@ -0,0 +1,79 @@ +function observe_v(dnsobs, Φ, les, compression, psolver) + (; grid) = les + (; dimension, N, Iu, Ip) = grid + D = dimension() + Mα = N[1] - 2 + v = zero.(Φ(dnsobs[].u, les, compression)) + Pv = zero.(v) + p = zero(v[1]) + div = zero(p) + ΦPF = zero.(v) + PFΦ = zero.(v) + c = zero.(v) + T = eltype(v[1]) + results = (; + Φ, + Mα, + t = zeros(T, 0), + Dv = zeros(T, 0), + Pv = zeros(T, 0), + Pc = zeros(T, 0), + c = zeros(T, 0), + ) + on(dnsobs) do (; u, PF, t) + push!(results.t, t) + + Φ(v, u, les, compression) + apply_bc_u!(v, t, les) + Φ(ΦPF, PF, les, compression) + momentum!(PFΦ, v, t, les) + apply_bc_u!(PFΦ, t, les; dudt = true) + project!(PFΦ, les; psolver, div, p) + foreach(α -> c[α] .= ΦPF[α] .- PFΦ[α], 1:D) + apply_bc_u!(c, t, les) + divergence!(div, v, les) + norm_Du = norm(div[Ip]) + norm_v = sqrt(sum(α -> sum(abs2, v[α][Iu[α]]), 1:D)) + push!(results.Dv, norm_Du / norm_v) + + copyto!.(Pv, v) + project!(Pv, les; psolver, div, p) + foreach(α -> Pv[α] .= Pv[α] .- v[α], 1:D) + norm_vmPv = sqrt(sum(α -> sum(abs2, Pv[α][Iu[α]]), 1:D)) + push!(results.Pv, norm_vmPv / norm_v) + + Pc = Pv + copyto!.(Pc, c) + project!(Pc, les; psolver, div, p) + foreach(α -> Pc[α] .= Pc[α] .- c[α], 1:D) + norm_cmPc = sqrt(sum(α -> sum(abs2, Pc[α][Iu[α]]), 1:D)) + norm_c = sqrt(sum(α -> sum(abs2, c[α][Iu[α]]), 1:D)) + push!(results.Pc, norm_cmPc / norm_c) + + norm_ΦPF = sqrt(sum(α -> sum(abs2, ΦPF[α][Iu[α]]), 1:D)) + push!(results.c, norm_c / norm_ΦPF) + end + results +end + +observe_u(dns, psolver_dns, filters; nupdate = 1) = + processor() do state + PF = zero.(state[].u) + div = zero(state[].u[1]) + p = zero(state[].u[1]) + dnsobs = Observable((; state[].u, PF, state[].t)) + results = [ + observe_v(dnsobs, Φ, setup, compression, psolver) for + (; setup, Φ, compression, psolver) in filters + ] + on(state) do (; u, t, n) + n % nupdate == 0 || return + apply_bc_u!(u, t, dns) + momentum!(PF, u, t, dns) + apply_bc_u!(PF, t, dns; dudt = true) + project!(PF, dns; psolver = psolver_dns, div, p) + dnsobs[] = (; u, PF, t) + end + # state[] = state[] # Save initial conditions + results + end From 08d41f5f1f6a4bcfe5612933c81d96a2c13682b9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Thu, 25 Apr 2024 13:20:49 +0200 Subject: [PATCH 305/379] Move subprojects --- {PaperDC => libs/PaperDC}/LICENSE | 0 {PaperDC => libs/PaperDC}/Project.toml | 0 {PaperDC => libs/PaperDC}/README.md | 0 {PaperDC => libs/PaperDC}/postanalysis.jl | 0 {PaperDC => libs/PaperDC}/prioranalysis.jl | 0 {PaperDC => libs/PaperDC}/src/PaperDC.jl | 0 {PaperDC => libs/PaperDC}/src/observe.jl | 0 {PaperDC => libs/PaperDC}/src/rk.jl | 0 {TensorClosure => libs/TensorClosure}/Project.toml | 0 {TensorClosure => libs/TensorClosure}/main.jl | 0 10 files changed, 0 insertions(+), 0 deletions(-) rename {PaperDC => libs/PaperDC}/LICENSE (100%) rename {PaperDC => libs/PaperDC}/Project.toml (100%) rename {PaperDC => libs/PaperDC}/README.md (100%) rename {PaperDC => libs/PaperDC}/postanalysis.jl (100%) rename {PaperDC => libs/PaperDC}/prioranalysis.jl (100%) rename {PaperDC => libs/PaperDC}/src/PaperDC.jl (100%) rename {PaperDC => libs/PaperDC}/src/observe.jl (100%) rename {PaperDC => libs/PaperDC}/src/rk.jl (100%) rename {TensorClosure => libs/TensorClosure}/Project.toml (100%) rename {TensorClosure => libs/TensorClosure}/main.jl (100%) diff --git a/PaperDC/LICENSE b/libs/PaperDC/LICENSE similarity index 100% rename from PaperDC/LICENSE rename to libs/PaperDC/LICENSE diff --git a/PaperDC/Project.toml b/libs/PaperDC/Project.toml similarity index 100% rename from PaperDC/Project.toml rename to libs/PaperDC/Project.toml diff --git a/PaperDC/README.md b/libs/PaperDC/README.md similarity index 100% rename from PaperDC/README.md rename to libs/PaperDC/README.md diff --git a/PaperDC/postanalysis.jl b/libs/PaperDC/postanalysis.jl similarity index 100% rename from PaperDC/postanalysis.jl rename to libs/PaperDC/postanalysis.jl diff --git a/PaperDC/prioranalysis.jl b/libs/PaperDC/prioranalysis.jl similarity index 100% rename from PaperDC/prioranalysis.jl rename to libs/PaperDC/prioranalysis.jl diff --git a/PaperDC/src/PaperDC.jl b/libs/PaperDC/src/PaperDC.jl similarity index 100% rename from PaperDC/src/PaperDC.jl rename to libs/PaperDC/src/PaperDC.jl diff --git a/PaperDC/src/observe.jl b/libs/PaperDC/src/observe.jl similarity index 100% rename from PaperDC/src/observe.jl rename to libs/PaperDC/src/observe.jl diff --git a/PaperDC/src/rk.jl b/libs/PaperDC/src/rk.jl similarity index 100% rename from PaperDC/src/rk.jl rename to libs/PaperDC/src/rk.jl diff --git a/TensorClosure/Project.toml b/libs/TensorClosure/Project.toml similarity index 100% rename from TensorClosure/Project.toml rename to libs/TensorClosure/Project.toml diff --git a/TensorClosure/main.jl b/libs/TensorClosure/main.jl similarity index 100% rename from TensorClosure/main.jl rename to libs/TensorClosure/main.jl From dc8f4f9b91737bfeffe61bcee2ea7ca88be26c9f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Thu, 25 Apr 2024 13:24:42 +0200 Subject: [PATCH 306/379] Update READMEs --- examples/README.md | 4 ++-- libs/PaperDC/README.md | 6 +++--- libs/TensorClosure/README.md | 27 +++++++++++++++++++++++++++ 3 files changed, 32 insertions(+), 5 deletions(-) create mode 100644 libs/TensorClosure/README.md diff --git a/examples/README.md b/examples/README.md index c736e914d..e77f9e279 100644 --- a/examples/README.md +++ b/examples/README.md @@ -20,6 +20,6 @@ or interactively from a Julia REPL: ```julia-repl julia> ] (v1.10) pkg> activate . -(examples) pkg> dev .. -(examples) pkg> instantiate +(Examples) pkg> dev .. +(Examples) pkg> instantiate ``` diff --git a/libs/PaperDC/README.md b/libs/PaperDC/README.md index 0c14fbc75..4df9d7cc7 100644 --- a/libs/PaperDC/README.md +++ b/libs/PaperDC/README.md @@ -11,8 +11,8 @@ From this directory, run: julia --project -e ' using Pkg Pkg.develop([ - PackageSpec(; path = ".."), - PackageSpec(; path = "../libs/NeuralClosure"), + PackageSpec(; path = "../.."), + PackageSpec(; path = "../NeuralClosure"), ]) Pkg.instantiate()' ' @@ -23,7 +23,7 @@ or interactively from a Julia REPL: ```julia-repl julia> ] (v1.10) pkg> activate . -(PaperDC) pkg> dev .. ../libs/NeuralClosure +(PaperDC) pkg> dev ../.. ../NeuralClosure (PaperDC) pkg> instantiate ``` diff --git a/libs/TensorClosure/README.md b/libs/TensorClosure/README.md new file mode 100644 index 000000000..ec76c711d --- /dev/null +++ b/libs/TensorClosure/README.md @@ -0,0 +1,27 @@ +# TensorClosure + +Tensor closure scripts. + +## Set up environment + +From this directory, run: + +```sh +julia --project -e ' +using Pkg +Pkg.develop([ + PackageSpec(; path = "../.."), + PackageSpec(; path = "../NeuralClosure"), +]) +Pkg.instantiate()' +' +``` + +or interactively from a Julia REPL: + +```julia-repl +julia> ] +(v1.10) pkg> activate . +(TensorClosure) pkg> dev ../.. ../NeuralClosure +(TensorClosure) pkg> instantiate +``` From 4f07f1d7101f7bcd885168b9f7549893181ca768 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Thu, 25 Apr 2024 13:30:39 +0200 Subject: [PATCH 307/379] Update gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 60eb6c8dd..a4a7cb627 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ +.vscode *.jl.*.cov *.jl.cov *.jl.mem From 1ef69692127af2681011683655db591d68e53298 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Thu, 25 Apr 2024 13:30:56 +0200 Subject: [PATCH 308/379] Update READMEs --- README.md | 5 +++-- examples/README.md | 2 +- libs/PaperDC/README.md | 2 +- libs/TensorClosure/README.md | 2 +- 4 files changed, 6 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index dd55ea26b..58b71e1dd 100644 --- a/README.md +++ b/README.md @@ -19,8 +19,8 @@ large eddy simulation. To install IncompressibleNavierStokes, open up a Julia-REPL, type `]` to get into Pkg-mode, and type: -``` -add IncompressibleNavierStokes +```julia-repl +(v1.10) pkg> add IncompressibleNavierStokes ``` which will install the package and all dependencies to your local environment. @@ -53,6 +53,7 @@ with an unsteady inflow. It simulates a wind turbine (actuator) under varying wind conditions. ```julia +# using Pkg; Pkg.add(["GLMakie", "IncompressibleNavierStokes"])) using GLMakie using IncompressibleNavierStokes diff --git a/examples/README.md b/examples/README.md index e77f9e279..82a545fb9 100644 --- a/examples/README.md +++ b/examples/README.md @@ -11,7 +11,7 @@ From this directory, run: julia --project -e ' using Pkg Pkg.develop(PackageSpec(; path = "..")) -Pkg.instantiate()' +Pkg.instantiate() ' ``` diff --git a/libs/PaperDC/README.md b/libs/PaperDC/README.md index 4df9d7cc7..5b6d4c8c1 100644 --- a/libs/PaperDC/README.md +++ b/libs/PaperDC/README.md @@ -14,7 +14,7 @@ Pkg.develop([ PackageSpec(; path = "../.."), PackageSpec(; path = "../NeuralClosure"), ]) -Pkg.instantiate()' +Pkg.instantiate() ' ``` diff --git a/libs/TensorClosure/README.md b/libs/TensorClosure/README.md index ec76c711d..1e4a0210b 100644 --- a/libs/TensorClosure/README.md +++ b/libs/TensorClosure/README.md @@ -13,7 +13,7 @@ Pkg.develop([ PackageSpec(; path = "../.."), PackageSpec(; path = "../NeuralClosure"), ]) -Pkg.instantiate()' +Pkg.instantiate() ' ``` From 800eddc286b3beb43e41983ba3a710f8184f7ed4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Thu, 25 Apr 2024 14:13:29 +0200 Subject: [PATCH 309/379] Fix tests --- Project.toml | 3 +++ src/solvers/pressure/solvers.jl | 2 +- test/Project.toml | 4 +++- 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/Project.toml b/Project.toml index 2173b5028..1d76fa150 100644 --- a/Project.toml +++ b/Project.toml @@ -24,9 +24,12 @@ WriteVTK = "64499a7a-5c06-52f2-abe2-ccb03c286192" [compat] Adapt = "3, 4" +CairoMakie = "0.11" +ChainRulesCore = "1" CUDA = "5" CUDSS = "0.2" FFTW = "1" +GLMakie = "0.9" IterativeSolvers = "0.9" KernelAbstractions = "0.9" LinearAlgebra = "1" diff --git a/src/solvers/pressure/solvers.jl b/src/solvers/pressure/solvers.jl index a5d554d21..e640ee9e4 100644 --- a/src/solvers/pressure/solvers.jl +++ b/src/solvers/pressure/solvers.jl @@ -16,7 +16,7 @@ struct DirectPressureSolver{T,S,F,A} <: AbstractPressureSolver{T} f::A p::A function DirectPressureSolver(setup) - (; grid, ArrayType) = setup + (; grid, boundary_conditions, ArrayType) = setup (; x, Np) = grid if false #ArrayType == CuArray # T = eltype(x[1]) diff --git a/test/Project.toml b/test/Project.toml index b62ce12ba..4533703fc 100644 --- a/test/Project.toml +++ b/test/Project.toml @@ -1,4 +1,6 @@ [deps] Aqua = "4c88cf16-eb10-579e-8560-4a9242c79595" CairoMakie = "13f3f980-e62b-5c42-98c6-ff1f3baf88f0" -IncompressibleNavierStokes = "5e318141-6589-402b-868d-77d7df8c442e" +LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" +Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2" +Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" From 379ba6d384429049fbd6710d30f2ed81ba488de0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Thu, 25 Apr 2024 14:13:54 +0200 Subject: [PATCH 310/379] chore: Format --- src/processors/real_time_plot.jl | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/processors/real_time_plot.jl b/src/processors/real_time_plot.jl index dbf54198b..b54567015 100644 --- a/src/processors/real_time_plot.jl +++ b/src/processors/real_time_plot.jl @@ -250,7 +250,7 @@ function fieldplot( Q = similar(u[1]) elseif fieldname == :eig2field λ = similar(u[1]) - elseif fieldname in union(Symbol.(["B$i" for i in 1:11]), Symbol.(["V$i" for i in 1:5])) + elseif fieldname in union(Symbol.(["B$i" for i = 1:11]), Symbol.(["V$i" for i = 1:5])) sym = string(fieldname)[1] sym = sym == 'B' ? 1 : 2 idx = parse(Int, string(fieldname)[2:end]) @@ -287,7 +287,8 @@ function fieldplot( λin = view(λ, Ip) @. λin .= log(max(logtol, -λin)) λ - elseif fieldname in union(Symbol.(["B$i" for i in 1:11]), Symbol.(["V$i" for i in 1:5])) + elseif fieldname in + union(Symbol.(["B$i" for i = 1:11]), Symbol.(["V$i" for i = 1:5])) tensorbasis!(tb..., u, setup) tb[sym][idx] end From 3ee9d9bb4e8201a7ae77026016f72d72dbe1867f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Thu, 25 Apr 2024 14:19:38 +0200 Subject: [PATCH 311/379] Rename variable --- src/boundary_conditions.jl | 76 +++++++++++++++++++------------------- 1 file changed, 38 insertions(+), 38 deletions(-) diff --git a/src/boundary_conditions.jl b/src/boundary_conditions.jl index ff1efb632..fc2731c0e 100644 --- a/src/boundary_conditions.jl +++ b/src/boundary_conditions.jl @@ -75,11 +75,11 @@ ghost_a!(::PressureBC, x) = pushfirst!(x, x[1], x[1]) ghost_b!(::PressureBC, x) = push!(x, x[end]) """ - offset_u(bc, isnormal, atend) + offset_u(bc, isnormal, isright) Number of non-DOF velocity components at boundary. If `isnormal`, then the velocity is normal to the boundary, else parallel. -If `atend`, it is at the end/right/rear/top boundary, otherwise beginning. +If `isright`, it is at the end/right/rear/top boundary, otherwise beginning. """ function offset_u end @@ -90,17 +90,17 @@ Number of non-DOF pressure components at boundary. """ function offset_p end -offset_u(::PeriodicBC, isnormal, atend) = 1 -offset_p(::PeriodicBC, atend) = 1 +offset_u(::PeriodicBC, isnormal, isright) = 1 +offset_p(::PeriodicBC, isright) = 1 -offset_u(::DirichletBC, isnormal, atend) = 1 + isnormal * atend -offset_p(::DirichletBC, atend) = 1 +offset_u(::DirichletBC, isnormal, isright) = 1 + isnormal * isright +offset_p(::DirichletBC, isright) = 1 -offset_u(::SymmetricBC, isnormal, atend) = 1 + isnormal * atend -offset_p(::SymmetricBC, atend) = 1 +offset_u(::SymmetricBC, isnormal, isright) = 1 + isnormal * isright +offset_p(::SymmetricBC, isright) = 1 -offset_u(::PressureBC, isnormal, atend) = 1 + !isnormal * !atend -offset_p(::PressureBC, atend) = 1 + !atend +offset_u(::PressureBC, isnormal, isright) = 1 + !isnormal * !isright +offset_p(::PressureBC, isright) = 1 + !isright function apply_bc_u! end function apply_bc_p! end @@ -134,8 +134,8 @@ function apply_bc_u!(u, t, setup; kwargs...) (; boundary_conditions) = setup D = length(u) for β = 1:D - apply_bc_u!(boundary_conditions[β][1], u, β, t, setup; atend = false, kwargs...) - apply_bc_u!(boundary_conditions[β][2], u, β, t, setup; atend = true, kwargs...) + apply_bc_u!(boundary_conditions[β][1], u, β, t, setup; isright = false, kwargs...) + apply_bc_u!(boundary_conditions[β][2], u, β, t, setup; isright = true, kwargs...) end u end @@ -151,7 +151,7 @@ function apply_bc_u_pullback!(φbar, t, setup; kwargs...) β, t, setup; - atend = false, + isright = false, kwargs..., ) apply_bc_u_pullback!( @@ -160,7 +160,7 @@ function apply_bc_u_pullback!(φbar, t, setup; kwargs...) β, t, setup; - atend = true, + isright = true, kwargs..., ) end @@ -172,8 +172,8 @@ function apply_bc_p!(p, t, setup; kwargs...) (; dimension) = grid D = dimension() for β = 1:D - apply_bc_p!(boundary_conditions[β][1], p, β, t, setup; atend = false) - apply_bc_p!(boundary_conditions[β][2], p, β, t, setup; atend = true) + apply_bc_p!(boundary_conditions[β][1], p, β, t, setup; isright = false) + apply_bc_p!(boundary_conditions[β][2], p, β, t, setup; isright = true) end p end @@ -183,13 +183,13 @@ function apply_bc_p_pullback!(φbar, t, setup; kwargs...) (; dimension) = grid D = dimension() for β = 1:D - apply_bc_p_pullback!(boundary_conditions[β][1], φbar, β, t, setup; atend = false) - apply_bc_p_pullback!(boundary_conditions[β][2], φbar, β, t, setup; atend = true) + apply_bc_p_pullback!(boundary_conditions[β][1], φbar, β, t, setup; isright = false) + apply_bc_p_pullback!(boundary_conditions[β][2], φbar, β, t, setup; isright = true) end φbar end -function apply_bc_u!(::PeriodicBC, u, β, t, setup; atend, kwargs...) +function apply_bc_u!(::PeriodicBC, u, β, t, setup; isright, kwargs...) (; grid, workgroupsize) = setup (; dimension, N) = grid D = dimension() @@ -204,7 +204,7 @@ function apply_bc_u!(::PeriodicBC, u, β, t, setup; atend, kwargs...) end ndrange = ntuple(γ -> γ == β ? 1 : N[γ], D) for α = 1:D - if atend + if isright _bc_b!(get_backend(u[1]), workgroupsize)(u, Val(α), Val(β); ndrange) else _bc_a!(get_backend(u[1]), workgroupsize)(u, Val(α), Val(β); ndrange) @@ -213,7 +213,7 @@ function apply_bc_u!(::PeriodicBC, u, β, t, setup; atend, kwargs...) u end -function apply_bc_u_pullback!(::PeriodicBC, φbar, β, t, setup; atend, kwargs...) +function apply_bc_u_pullback!(::PeriodicBC, φbar, β, t, setup; isright, kwargs...) (; grid, workgroupsize) = setup (; dimension, N) = grid D = dimension() @@ -230,7 +230,7 @@ function apply_bc_u_pullback!(::PeriodicBC, φbar, β, t, setup; atend, kwargs.. end ndrange = ntuple(γ -> γ == β ? 1 : N[γ], D) for α = 1:D - if atend + if isright adj_b!(get_backend(φbar[1]), workgroupsize)(φbar, Val(α), Val(β); ndrange) else adj_a!(get_backend(φbar[1]), workgroupsize)(φbar, Val(α), Val(β); ndrange) @@ -239,7 +239,7 @@ function apply_bc_u_pullback!(::PeriodicBC, φbar, β, t, setup; atend, kwargs.. φbar end -function apply_bc_p!(::PeriodicBC, p, β, t, setup; atend, kwargs...) +function apply_bc_p!(::PeriodicBC, p, β, t, setup; isright, kwargs...) (; grid, workgroupsize) = setup (; dimension, N) = grid D = dimension() @@ -253,7 +253,7 @@ function apply_bc_p!(::PeriodicBC, p, β, t, setup; atend, kwargs...) p[I+(N[β]-1)*δ(β)] = p[I+δ(β)] end ndrange = ntuple(γ -> γ == β ? 1 : N[γ], D) - if atend + if isright _bc_b(get_backend(p), workgroupsize)(p, Val(β); ndrange) else _bc_a(get_backend(p), workgroupsize)(p, Val(β); ndrange) @@ -261,7 +261,7 @@ function apply_bc_p!(::PeriodicBC, p, β, t, setup; atend, kwargs...) p end -function apply_bc_p_pullback!(::PeriodicBC, φbar, β, t, setup; atend, kwargs...) +function apply_bc_p_pullback!(::PeriodicBC, φbar, β, t, setup; isright, kwargs...) (; grid, workgroupsize) = setup (; dimension, N) = grid D = dimension() @@ -277,7 +277,7 @@ function apply_bc_p_pullback!(::PeriodicBC, φbar, β, t, setup; atend, kwargs.. φ[I+(N[β]-1)*δ(β)] = 0 end ndrange = ntuple(γ -> γ == β ? 1 : N[γ], D) - if atend + if isright adj_b!(get_backend(φbar), workgroupsize)(φbar, Val(β); ndrange) else adj_a!(get_backend(φbar), workgroupsize)(φbar, Val(β); ndrange) @@ -285,14 +285,14 @@ function apply_bc_p_pullback!(::PeriodicBC, φbar, β, t, setup; atend, kwargs.. φbar end -function apply_bc_u!(bc::DirichletBC, u, β, t, setup; atend, dudt = false, kwargs...) +function apply_bc_u!(bc::DirichletBC, u, β, t, setup; isright, dudt = false, kwargs...) (; dimension, x, xp, N) = setup.grid D = dimension() δ = Offset{D}() # isnothing(bc.u) && return bcfunc = dudt ? bc.dudt : bc.u for α = 1:D - I = if atend + I = if isright CartesianIndices( ntuple(γ -> γ == β ? α == β ? (N[γ]-1:N[γ]-1) : (N[γ]:N[γ]) : (1:N[γ]), D), ) @@ -317,11 +317,11 @@ function apply_bc_u!(bc::DirichletBC, u, β, t, setup; atend, dudt = false, kwar u end -function apply_bc_p!(::DirichletBC, p, β, t, setup; atend, kwargs...) +function apply_bc_p!(::DirichletBC, p, β, t, setup; isright, kwargs...) (; dimension, N) = setup.grid D = dimension() δ = Offset{D}() - if atend + if isright I = CartesianIndices(ntuple(γ -> γ == β ? (N[γ]:N[γ]) : (1:N[γ]), D)) p[I] .= p[I.-δ(β)] else @@ -331,13 +331,13 @@ function apply_bc_p!(::DirichletBC, p, β, t, setup; atend, kwargs...) p end -function apply_bc_u!(::SymmetricBC, u, β, t, setup; atend, kwargs...) +function apply_bc_u!(::SymmetricBC, u, β, t, setup; isright, kwargs...) (; dimension, N) = setup.grid D = dimension() δ = Offset{D}() for α = 1:D if α != β - if atend + if isright I = CartesianIndices(ntuple(γ -> γ == β ? (N[γ]:N[γ]) : (1:N[γ]), D)) u[α][I] .= u[α][I.-δ(β)] else @@ -349,11 +349,11 @@ function apply_bc_u!(::SymmetricBC, u, β, t, setup; atend, kwargs...) u end -function apply_bc_p!(::SymmetricBC, p, β, t, setup; atend, kwargs...) +function apply_bc_p!(::SymmetricBC, p, β, t, setup; isright, kwargs...) (; dimension, N) = setup.grid D = dimension() δ = Offset{D}() - if atend + if isright I = CartesianIndices(ntuple(γ -> γ == β ? (N[γ]:N[γ]) : (1:N[γ]), D)) p[I] .= p[I.-δ(β)] else @@ -363,7 +363,7 @@ function apply_bc_p!(::SymmetricBC, p, β, t, setup; atend, kwargs...) p end -function apply_bc_u!(bc::PressureBC, u, β, t, setup; atend, kwargs...) +function apply_bc_u!(bc::PressureBC, u, β, t, setup; isright, kwargs...) (; grid, workgroupsize) = setup (; dimension, N, Nu, Iu) = grid D = dimension() @@ -380,7 +380,7 @@ function apply_bc_u!(bc::PressureBC, u, β, t, setup; atend, kwargs...) end ndrange = (N[1:β-1]..., 1, N[β+1:end]...) for α = 1:D - if atend + if isright I0 = CartesianIndex(ntuple(γ -> γ == β ? N[β] : 1, D)) I0 -= oneunit(I0) _bc_b!(get_backend(u[1]), workgroupsize)(u, Val(α), Val(β), I0; ndrange) @@ -393,10 +393,10 @@ function apply_bc_u!(bc::PressureBC, u, β, t, setup; atend, kwargs...) u end -function apply_bc_p!(bc::PressureBC, p, β, t, setup; atend, kwargs...) +function apply_bc_p!(bc::PressureBC, p, β, t, setup; isright, kwargs...) (; dimension, N) = setup.grid D = dimension() - I = if atend + I = if isright CartesianIndices(ntuple(γ -> γ == β ? (N[γ]:N[γ]) : (1:N[γ]), D)) else CartesianIndices(ntuple(γ -> γ == β ? (2:2) : (1:N[γ]), D)) From c6b55e177a259e715b20e92afa4673614a73f859 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Thu, 25 Apr 2024 14:44:54 +0200 Subject: [PATCH 312/379] Fix dimension in definite case --- src/solvers/pressure/poisson.jl | 25 ++++++++++++++++++++----- src/solvers/pressure/solvers.jl | 6 ++++-- 2 files changed, 24 insertions(+), 7 deletions(-) diff --git a/src/solvers/pressure/poisson.jl b/src/solvers/pressure/poisson.jl index 40447347e..387e3bda5 100644 --- a/src/solvers/pressure/poisson.jl +++ b/src/solvers/pressure/poisson.jl @@ -28,7 +28,8 @@ function poisson! end function poisson!(solver::DirectPressureSolver, p, f) (; setup, fact) = solver - (; Ip) = setup.grid + (; grid, boundary_conditions) = setup + (; Ip) = grid T = eltype(p) # solver.f .= view(view(f, Ip), :) # copyto!(solver.f, view(view(f, Ip), :)) @@ -48,9 +49,16 @@ function poisson!(solver::DirectPressureSolver, p, f) mul!(a, F.Q, b) copyto!(pp, view(a, 1:length(a)-1)) else - copyto!(view(solver.f, 1:length(solver.f)-1), Array(view(view(f, Ip), :))) + if any(bc -> bc[1] isa PressureBC || bc[2] isa PressureBC, boundary_conditions) + # No extra DOF + viewrange = (:) + else + # With extra DOF + viewrange = 1:length(solver.p)-1 + end + copyto!(view(solver.f, viewrange), Array(view(view(f, Ip), :))) solver.p .= fact \ solver.f - copyto!(pp, T.(view(solver.p, 1:length(solver.p)-1))) + copyto!(pp, T.(view(solver.p, viewrange))) end # @infiltrate # ldiv!(solver.p, fact, solver.f) @@ -63,10 +71,17 @@ function poisson!(solver::CUDSSPressureSolver, p, f) (; setup) = solver (; Ip) = setup.grid T = eltype(p) + if any(bc -> bc[1] isa PressureBC || bc[2] isa PressureBC, boundary_conditions) + # No extra DOF + viewrange = (:) + else + # With extra DOF + viewrange = 1:length(solver.p)-1 + end pp = view(view(p, Ip), :) - copyto!(view(solver.f, 1:length(solver.f)-1), view(view(f, Ip), :)) + copyto!(view(solver.f, viewrange), view(view(f, Ip), :)) cudss("solve", solver.solver, solver.p, solver.f) - copyto!(pp, view(solver.p, 1:length(solver.p)-1)) + copyto!(pp, view(solver.p, viewrange)) p end diff --git a/src/solvers/pressure/solvers.jl b/src/solvers/pressure/solvers.jl index e640ee9e4..0d3865176 100644 --- a/src/solvers/pressure/solvers.jl +++ b/src/solvers/pressure/solvers.jl @@ -55,11 +55,13 @@ struct DirectPressureSolver{T,S,F,A} <: AbstractPressureSolver{T} # T = eltype(x[1]) T = Float64 backend = get_backend(x[1]) - f = zeros(T, prod(Np) + 1) - p = zeros(T, prod(Np) + 1) L = laplacian_mat(setup) if any(bc -> bc[1] isa PressureBC || bc[2] isa PressureBC, boundary_conditions) + f = zeros(T, prod(Np)) + p = zeros(T, prod(Np)) else + f = zeros(T, prod(Np) + 1) + p = zeros(T, prod(Np) + 1) e = ones(T, size(L, 2)) L = [L e; e' 0] end From 9f1482df680133446a271fdc0d28fbeea46929f4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Thu, 25 Apr 2024 15:11:12 +0200 Subject: [PATCH 313/379] fix(NeuralClosure): Missing dependencies --- libs/NeuralClosure/Project.toml | 4 +- libs/NeuralClosure/src/NeuralClosure.jl | 3 +- libs/PaperDC/postanalysis.jl | 55 ++++++++++--------------- 3 files changed, 26 insertions(+), 36 deletions(-) diff --git a/libs/NeuralClosure/Project.toml b/libs/NeuralClosure/Project.toml index c2becd591..82e8e9c58 100644 --- a/libs/NeuralClosure/Project.toml +++ b/libs/NeuralClosure/Project.toml @@ -9,16 +9,18 @@ ComponentArrays = "b0b7db55-cfe3-40fc-9ded-d10e2dbeff66" IncompressibleNavierStokes = "5e318141-6589-402b-868d-77d7df8c442e" Lux = "b2108857-7c20-44ae-9111-449ecde12c47" NNlib = "872c559c-99b0-510c-b3b7-b6c96a88d5cd" +Observables = "510215fc-4207-5dde-b226-833fc4488ee2" Optimisers = "3bd65402-5787-11e9-1adc-39752487f4e2" Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" Tullio = "bc48ee85-29a4-5162-ae0b-a64e1601d4bc" Zygote = "e88e6eb3-aa80-5325-afca-941959d7151f" [compat] -ComponentArrays = "0.15" CUDA = "5" +ComponentArrays = "0.15" Lux = "0.5" NNlib = "0.9" +Observables = "0.5" Optimisers = "0.3" Random = "1" Tullio = "0.3" diff --git a/libs/NeuralClosure/src/NeuralClosure.jl b/libs/NeuralClosure/src/NeuralClosure.jl index ae4c44fdb..048cc9622 100644 --- a/libs/NeuralClosure/src/NeuralClosure.jl +++ b/libs/NeuralClosure/src/NeuralClosure.jl @@ -6,9 +6,10 @@ module NeuralClosure using CUDA using ComponentArrays: ComponentArray using IncompressibleNavierStokes -using IncompressibleNavierStokes: Dimension +using IncompressibleNavierStokes: Dimension, momentum!, apply_bc_u!, project! using Lux using NNlib +using Observables using Random using Tullio using Zygote diff --git a/libs/PaperDC/postanalysis.jl b/libs/PaperDC/postanalysis.jl index bb1b006c0..52061ddd1 100644 --- a/libs/PaperDC/postanalysis.jl +++ b/libs/PaperDC/postanalysis.jl @@ -1,7 +1,10 @@ -# # Train closure model +# # A-posteriori analysis: Large Eddy Simulation (2D) # -# Here, we consider a periodic box ``[0, 1]^2``. It is discretized with a -# uniform Cartesian grid with square cells. +# Generate filtered DNS data, train closure model, compare filters, closure +# models and projection orders. +# +# The data is saved and can be loaded in a subesequent sesssion. +# The CNN parameters are also saved. using Adapt using GLMakie @@ -11,15 +14,17 @@ using JLD2 using LaTeXStrings using LinearAlgebra using Lux +using NeuralClosure using NNlib using Optimisers using Random using SparseArrays using FFTW -# palette = (; color = ["#3366cc", "#cc0000", "#669900", "#ffcc00"]) +# Color palette for consistent theme throughout paper palette = (; color = ["#3366cc", "#cc0000", "#669900", "#ff9900"]) +# Encode projection order ("close first, then project" etc) getorder(i) = if i == 1 :first @@ -31,34 +36,27 @@ getorder(i) = error("Unknown order: $i") end -GLMakie.activate!() - -set_theme!(; GLMakie = (; scalefactor = 1.5)) - -plotdir = "../SupervisedClosure/figures" -# outdir = "output/postanalysis" -outdir = "output/divfree" +# Choose where to put output +plotdir = "output/postanalysis/plots" +outdir = "output/postanalysis" +ispath(plotdir) || mkpath(plotdir) ispath(outdir) || mkpath(outdir) # Random number generator rng = Random.default_rng() Random.seed!(rng, 12345) -# Floating point precision -T = Float64 - -# Array type +# For running on CPU. +# Consider reducing the sizes of DNS, LES, and CNN layers if +# you want to test run on a laptop. +T = Float32 ArrayType = Array device = identity clean() = nothing -## using CUDA; ArrayType = CuArray -## using AMDGPU; ArrayType = ROCArray -## using oneAPI; ArrayType = oneArray -## using Metal; ArrayType = MtlArray +# For running on a CUDA compatible GPU using LuxCUDA using CUDA; -# T = Float64; T = Float32; ArrayType = CuArray; CUDA.allowscalar(false); @@ -73,8 +71,6 @@ get_params(nlesscalar) = (; tsim = T(0.5), Δt = T(5e-5), nles = map(n -> (n, n), nlesscalar), - # ndns = (n -> (n, n))(1024), - # ndns = (n -> (n, n))(2048), ndns = (n -> (n, n))(4096), filters = (FaceAverage(), VolumeAverage()), ArrayType, @@ -88,15 +84,12 @@ get_params(nlesscalar) = (; ), ) +# Get parameters for multiple LES resolutions params_train = (; get_params([64, 128, 256])..., tsim = T(0.5), savefreq = 10); params_valid = (; get_params([64, 128, 256])..., tsim = T(0.1), savefreq = 40); params_test = (; get_params([64, 128, 256, 512, 1024])..., tsim = T(0.1), savefreq = 10); -# params_train = (; get_params([64, 128, 256, 512])..., tsim = T(0.5), savefreq = 10); -# params_valid = (; get_params([64, 128, 256, 512])..., tsim = T(0.1), savefreq = 40); -# params_test = (; get_params([64, 128, 256, 512])..., tsim = T(0.1), savefreq = 10); - -# Create LES data from DNS +# Create filtered DNS data data_train = [create_les_data(; params_train...) for _ = 1:5]; data_valid = [create_les_data(; params_valid...) for _ = 1:1]; data_test = create_les_data(; params_test...); @@ -111,21 +104,15 @@ data_train = load("$outdir/data_train.jld2", "data_train"); data_valid = load("$outdir/data_valid.jld2", "data_valid"); data_test = load("$outdir/data_test.jld2", "data_test"); +# Computational time data_train[5].comptime data_valid[1].comptime data_test.comptime - map(d -> d.comptime, data_train) - sum(d -> d.comptime, data_train) / 60 - data_test.comptime / 60 - (sum(d -> d.comptime, data_train) + sum(d -> d.comptime, data_valid) + data_test.comptime) -data_train[1].data[1].u[1][1] -data_test.data[6].u[1][1] - # Build LES setup and assemble operators getsetups(params) = [ Setup( From 64979accfca0d66d4e488b78b640503010bcae07 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Thu, 25 Apr 2024 16:17:20 +0200 Subject: [PATCH 314/379] feat(PaperDC): Clean up script --- libs/PaperDC/postanalysis.jl | 204 +++++++++++++++-------------------- 1 file changed, 89 insertions(+), 115 deletions(-) diff --git a/libs/PaperDC/postanalysis.jl b/libs/PaperDC/postanalysis.jl index 52061ddd1..3c44dbb8c 100644 --- a/libs/PaperDC/postanalysis.jl +++ b/libs/PaperDC/postanalysis.jl @@ -1,10 +1,10 @@ # # A-posteriori analysis: Large Eddy Simulation (2D) # -# Generate filtered DNS data, train closure model, compare filters, closure -# models and projection orders. +# Generate filtered DNS data, train closure model, compare filters, +# closure models, and projection orders. # -# The data is saved and can be loaded in a subesequent sesssion. -# The CNN parameters are also saved. +# The filtered DNS data is saved and can be loaded in a subesequent sesssion. +# The learned CNN parameters are also saved. using Adapt using GLMakie @@ -70,15 +70,14 @@ get_params(nlesscalar) = (; tburn = T(0.05), tsim = T(0.5), Δt = T(5e-5), - nles = map(n -> (n, n), nlesscalar), - ndns = (n -> (n, n))(4096), + nles = map(n -> (n, n), nlesscalar), # LES resolutions + ndns = (n -> (n, n))(4096), # DNS resolution filters = (FaceAverage(), VolumeAverage()), ArrayType, PSolver = SpectralPressureSolver, icfunc = (setup, psolver) -> random_field( setup, zero(eltype(setup.grid.x[1])); - # A = 1, kp = 20, psolver, ), @@ -125,19 +124,27 @@ setups_train = getsetups(params_train); setups_valid = getsetups(params_valid); setups_test = getsetups(params_test); +# Example data inspection data_train[1].t data_train[1].data |> size data_train[1].data[1, 1].u[end][1] -# Create input/output arrays +# Create input/output arrays for a-priori training (ubar vs c) io_train = create_io_arrays(data_train, setups_train); io_valid = create_io_arrays(data_valid, setups_valid); io_test = create_io_arrays([data_test], setups_test); +# # Save IO arrays # jldsave("$outdir/io_train.jld2"; io_train) # jldsave("$outdir/io_valid.jld2"; io_valid) # jldsave("$outdir/io_test.jld2"; io_test) +# +# # Load IO arrays +# io_train = load("$outdir/io_train.jld2"; "io_train") +# io_valid = load("$outdir/io_valid.jld2"; "io_valid") +# io_test = load("$outdir/io_test.jld2"; "io_test") +# Check that data is reasonably bounded io_train[1].u |> extrema io_train[1].c |> extrema io_valid[1].u |> extrema @@ -145,7 +152,8 @@ io_valid[1].c |> extrema io_test[1].u |> extrema io_test[1].c |> extrema -# Inspect data +# Inspect data (live animation with GLMakie) +GLMakie.activate!() let ig = 2 ifil = 1 @@ -167,75 +175,7 @@ let end end -GLMakie.activate!() -CairoMakie.activate!() - -# Training data plot -ifil = 1 -boxx = T(0.3), T(0.5) -boxy = T(0.5), T(0.7) -box = [ - Point2f(boxx[1], boxy[1]), - Point2f(boxx[2], boxy[1]), - Point2f(boxx[2], boxy[2]), - Point2f(boxx[1], boxy[2]), - Point2f(boxx[1], boxy[1]), -] -# fig = with_theme() do -fig = with_theme(; palette = (; color = ["#3366cc", "#cc0000", "#669900", "#ffcc00"])) do - sample = data_train[1] - fig = Figure() - for (i, it) in enumerate((1, length(sample.t))) - # for (j, ig) in enumerate((1, 2, 3)) - for (j, ig) in enumerate((1, 2)) - setup = setups_train[ig] - xf = Array.(getindex.(setup.grid.xp, setup.grid.Ip.indices)) - u = sample.data[ig, ifil].u[it] |> device - ωp = - IncompressibleNavierStokes.interpolate_ω_p( - IncompressibleNavierStokes.vorticity(u, setup), - setup, - )[setup.grid.Ip] |> Array - colorrange = IncompressibleNavierStokes.get_lims(ωp) - opts = (; - xticksvisible = false, - xticklabelsvisible = false, - yticklabelsvisible = false, - yticksvisible = false, - ) - i == 2 && ( - opts = (; - opts..., - xlabel = "x", - xticksvisible = true, - xticklabelsvisible = true, - ) - ) - j == 1 && ( - opts = (; - opts..., - ylabel = "y", - yticklabelsvisible = true, - yticksvisible = true, - ) - ) - ax = Axis( - fig[i, j]; - opts..., - title = "n = $(params_train.nles[ig]), t = $(round(sample.t[it]; digits = 1))", - aspect = DataAspect(), - limits = (T(0), T(1), T(0), T(1)), - ) - heatmap!(ax, xf..., ωp; colorrange) - # lines!(ax, box; color = Cycled(2)) - end - end - fig -end - -save("$plotdir/training_data.pdf", fig) - -# Architecture 1 +# CNN architecture 1 mname = "balzac" closure, θ₀ = cnn(; setup = setups_train[1], @@ -247,7 +187,7 @@ closure, θ₀ = cnn(; ); closure.chain -# Architecture 2 +# CNN architecture 2 mname = "rimbaud" closure, θ₀ = cnn(; setup = setups_train[1], @@ -258,14 +198,24 @@ closure, θ₀ = cnn(; rng, ); closure.chain + +# Save-path for CNN savepath = "$outdir/$mname" ispath(savepath) || mkpath(savepath) +# Give the CNN a test run +# Note: Data and parameters are stored on the CPU, and +# must be moved to the GPU before running (`device`) closure(device(io_train[1, 1].u[:, :, :, 1:50]), device(θ₀)); # A-priori training ########################################################### +# +# Train one set of CNN parameters for each of the filter types and grid sizes. +# Save parameters to disk after each run. +# Plot training progress (for a validation data batch). for ifil = 1:2, ig = 4:4 + clean() starttime = time() println("ig = $ig, ifil = $ifil") d = create_dataloader_prior(io_train[ig, ifil]; batchsize = 50, device) @@ -278,7 +228,7 @@ for ifil = 1:2, ig = 4:4 create_relerr_prior(closure, validset...); θ, displayref = true, - display_each_iteration = false, + display_each_iteration = false, # Set to `true` if using CairoMakie ) (; opt, θ, callbackstate) = train( [d], @@ -286,7 +236,6 @@ for ifil = 1:2, ig = 4:4 opt, θ; niter = 10_000, - # niter = 100, ncallback = 20, callbackstate, callback, @@ -297,7 +246,7 @@ for ifil = 1:2, ig = 4:4 end clean() -# Load trained parameters +# Load learned parameters and training times prior = map(CartesianIndices(size(io_train))) do I ig, ifil = I.I name = "$path/prior_ifilter$(ifil)_igrid$(ig).jld2" @@ -305,24 +254,30 @@ prior = map(CartesianIndices(size(io_train))) do I end; θ_cnn_prior = [copyto!(device(θ₀), p.θ) for p in prior]; +# Check that parameters are within reasonable bounds θ_cnn_prior .|> extrema +# Training times map(p -> p.comptime, prior) map(p -> p.comptime, prior) |> vec -map(p -> p.comptime, prior) |> sum -map(p -> p.comptime, prior) |> sum |> x -> x / 60 -map(p -> p.comptime, prior) |> sum |> x -> x / 3600 +map(p -> p.comptime, prior) |> sum # Seconds +map(p -> p.comptime, prior) |> sum |> x -> x / 60 # Minutes +map(p -> p.comptime, prior) |> sum |> x -> x / 3600 # Hours # A-posteriori training ###################################################### +# +# Train one set of CNN parameters for each +# projection order, filter type and grid size. +# Save parameters to disk after each combination. +# Plot training progress (for a validation data batch). +# +# The time stepper `RKProject` allows for choosing when to project. let ngrid, nfilter = size(io_train) for iorder = 1:2, ifil = 1:nfilter, ig = 1:ngrid clean() starttime = time() - # (ig, ifil, iorder) == (3, 1, 1) || continue - # (ifil, iorder) == (1, 1) && continue - # (ifil, iorder) == (2, 2) || continue println("iorder = $iorder, ifil = $ifil, ig = $ig") setup = setups_train[ig] psolver = SpectralPressureSolver(setup) @@ -331,7 +286,7 @@ let psolver, method = RKProject(RK44(; T), getorder(iorder)), closure, - nupdate = 2, + nupdate = 2, # Time steps per loss evaluation ) data = [(; u = d.data[ig, ifil].u, d.t) for d in data_train] d = create_dataloader_post(data; device, nunroll = 20) @@ -361,6 +316,7 @@ let clean() end +# Load learned parameters and training times post = map(CartesianIndices((size(io_train)..., 2))) do I ig, ifil, iorder = I.I name = "$savepath/post_iorder$(iorder)_ifil$(ifil)_ig$(ig).jld2" @@ -368,8 +324,10 @@ post = map(CartesianIndices((size(io_train)..., 2))) do I end; θ_cnn_post = [copyto!(device(θ₀), p.θ) for p in post]; +# Check that parameters are within reasonable bounds θ_cnn_post .|> extrema +# Training times map(p -> p.comptime, post) map(p -> p.comptime, post) |> x -> reshape(x, 6, 2) map(p -> p.comptime, post) ./ 60 @@ -377,7 +335,14 @@ map(p -> p.comptime, post) |> sum map(p -> p.comptime, post) |> sum |> x -> x / 60 map(p -> p.comptime, post) |> sum |> x -> x / 3600 -# Train Smagorinsky model with a-posteriori error grid search +# Train Smagorinsky model #################################################### +# +# Use a-posteriori error grid search to determine +# the optimal Smagorinsky constant. +# Find one constant for each projection order and filter type. but +# The constant is shared for all grid sizes, since the filter +# width (=grid size) is part of the model definition separately. + smag = map(CartesianIndices((size(io_train, 2), 2))) do I starttime = time() ifil, iorder = I.I @@ -427,10 +392,14 @@ smag = load("$outdir/smag.jld2")["smag"]; # Extract coefficients θ_smag = map(s -> s.θ, smag) +# Computational time map(s -> s.comptime, smag) map(s -> s.comptime, smag) |> sum # Compute a-priori errors ################################################### +# +# Note that it is still interesting to compute the a-priori errors for the +# a-posteriori trained CNN. eprior = let prior = zeros(T, 3, 2) @@ -454,7 +423,7 @@ eprior.post eprior.prior |> x -> reshape(x, :) |> x -> round.(x; digits = 2) eprior.post |> x -> reshape(x, :, 2) |> x -> round.(x; digits = 2) -# Compute posterior errors #################################################### +# Compute a-posteriori errors ################################################# (; e_nm, e_smag, e_cnn, e_cnn_post) = let e_nm = zeros(T, size(data_test.data)...) @@ -511,13 +480,10 @@ round.( sigdigits = 2, ) -data_train[1].t[2] - data_train[1].t[1] -data_test.t[2] - data_test.t[1] +# Plot a-priori errors ######################################################## +# Better for PDF export CairoMakie.activate!() -GLMakie.activate!() - -# Plot a-priori errors ######################################################## fig = with_theme(; palette) do nles = [n[1] for n in params_test.nles][1:3] @@ -577,6 +543,9 @@ end # Plot a-posteriori errors ################################################### +# Better for PDF export +CairoMakie.activate!() + with_theme(; palette) do iorder = 2 lesmodel = iorder == 1 ? "DIF" : "DCF" @@ -652,6 +621,8 @@ with_theme(; palette) do end # Energy evolution ########################################################### +# +# Compute total kinetic energy as a function of time. kineticenergy = let clean() @@ -741,6 +712,7 @@ clean(); # Plot energy evolution ######################################################## +# Better for PDF export CairoMakie.activate!() with_theme(; palette) do @@ -802,6 +774,8 @@ with_theme(; palette) do end # Compute Divergence ########################################################## +# +# Compute divergence as a function of time. divs = let clean() @@ -896,29 +870,18 @@ divs = let end; clean(); +# Check that divergence is within reasonable bounds divs.d_ref .|> extrema divs.d_nomodel .|> extrema divs.d_smag .|> extrema divs.d_cnn_prior .|> extrema divs.d_cnn_post .|> extrema -divs.d_ref[1, 1] |> lines -divs.d_ref[1, 2] |> lines! - -divs.d_nomodel[1, 1] |> lines -divs.d_nomodel[1, 2] |> lines! - -divs.d_cnn_post[1, 1, 3] |> lines -divs.d_cnn_post[1, 2, 3] |> lines! - -divs.d_smag[1, 2, 3] -divs.d_smag[1, 1, 3] -divs.d_smag[1, 2, 3] +# Plot Divergence ############################################################# +# Better for PDF export CairoMakie.activate!() -# Plot Divergence ############################################################# - with_theme(; # fontsize = 20, palette, @@ -931,7 +894,7 @@ with_theme(; "DIF" elseif iorder == 2 "DCF" - else + elseif iorder == 3 "DCF-RHS" end fil = ifil == 1 ? "FA" : "VA" @@ -1052,11 +1015,18 @@ ufinal = let end; clean(); -jldsave("$savepath/ufinal.jld2"; ufinal) - -ufinal = load("$savepath/ufinal.jld2")["ufinal"]; +# # Save solution +# jldsave("$savepath/ufinal.jld2"; ufinal) +# +# # Load solution +# ufinal = load("$savepath/ufinal.jld2")["ufinal"]; # Plot spectra ############################################################### +# +# Plot kinetic energy spectra at final time. + +# Better for PDF export +CairoMakie.activate!() fig = with_theme(; palette) do for iorder = 1:2, ifil = 1:2, igrid = 1:3 @@ -1125,9 +1095,13 @@ clean(); # Plot fields ################################################################ +# Export to PNG, otherwise each volume gets represented +# as a separate rectangle in the PDF +# (takes time to load in the article PDF) GLMakie.activate!() with_theme(; fontsize = 25, palette) do + # Reference box for eddy comparison x1 = 0.3 x2 = 0.5 y1 = 0.5 @@ -1155,7 +1129,7 @@ with_theme(; fontsize = 25, palette) do docolorbar = false, size = (500, 500), ) - lines!(box; linewidth = 5, color = Cycled(2)) + lines!(box; linewidth = 5, color = Cycled(2)) # Red in palette fname = "$(name)_$(suffix).png" save(fname, fig) # run(`convert $fname -trim $fname`) # Requires imagemagick From ebc27a17fc33f2549b15995c7c314fbd243491e3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Thu, 25 Apr 2024 17:02:44 +0200 Subject: [PATCH 315/379] feat(PaperDC): Structure RNGs transparently --- libs/NeuralClosure/src/create_les_data.jl | 25 ++++--- libs/NeuralClosure/src/training.jl | 12 ++-- libs/PaperDC/postanalysis.jl | 82 ++++++++++++++++++----- 3 files changed, 86 insertions(+), 33 deletions(-) diff --git a/libs/NeuralClosure/src/create_les_data.jl b/libs/NeuralClosure/src/create_les_data.jl index 3ea1d072c..7be062472 100644 --- a/libs/NeuralClosure/src/create_les_data.jl +++ b/libs/NeuralClosure/src/create_les_data.jl @@ -91,17 +91,21 @@ filtersaver(dns, les, filters, compression, psolver_dns, psolver_les; nupdate = """ create_les_data( - T; D = 2, - Re = T(2_000), - lims = (T(0), T(1)), - nles = [64], - ndns = 256, + Re = 2e3, + lims = ntuple(α -> (typeof(Re)(0), typeof(Re)(1)), D), + nles = [ntuple(α -> 64, D)], + ndns = ntuple(α -> 256, D), filters = (FaceAverage(),), - tburn = T(0.1), - tsim = T(0.1), - Δt = T(1e-4), + tburn = typeof(Re)(0.1), + tsim = typeof(Re)(0.1), + Δt = typeof(Re)(1e-4), + PSolver = SpectralPressureSolver, + savefreq = 1, ArrayType = Array, + icfunc = (setup, psolver) -> random_field(setup, typeof(Re)(0); psolver), + rng, + kwargs..., ) Create filtered DNS data. @@ -119,7 +123,8 @@ function create_les_data(; PSolver = SpectralPressureSolver, savefreq = 1, ArrayType = Array, - icfunc = (setup, psolver) -> random_field(setup, typeof(Re)(0); psolver), + icfunc = (setup, psolver, rng) -> random_field(setup, typeof(Re)(0); psolver, rng), + rng, kwargs..., ) T = typeof(Re) @@ -165,7 +170,7 @@ function create_les_data(; @info "Generating $datasize Mb of filtered DNS data" # Initial conditions - u₀ = icfunc(dns, psolver) + u₀ = icfunc(dns, psolver, rng) any(u -> any(isnan, u), u₀) && @warn "Initial conditions contain NaNs" diff --git a/libs/NeuralClosure/src/training.jl b/libs/NeuralClosure/src/training.jl index 9b30a3df0..d1b2fe815 100644 --- a/libs/NeuralClosure/src/training.jl +++ b/libs/NeuralClosure/src/training.jl @@ -1,31 +1,31 @@ """ - createdataloader(data; nuse = 50, device = identity) + createdataloader(data; nuse = 50, device = identity, rng) Create dataloader that uses a batch of `batchsize` random samples from `data` at each evaluation. The batch is moved to `device`. """ -create_dataloader_prior(data; batchsize = 50, device = identity) = function dataloader() +create_dataloader_prior(data; batchsize = 50, device = identity, rng) = function dataloader() x, y = data nsample = size(x)[end] d = ndims(x) - i = sort(shuffle(1:nsample)[1:batchsize]) + i = sort(shuffle(rng, 1:nsample)[1:batchsize]) xuse = device(Array(selectdim(x, d, i))) yuse = device(Array(selectdim(y, d, i))) xuse, yuse end """ - create_dataloader_post(trajectories; nunroll = 10, device = identity) + create_dataloader_post(trajectories; nunroll = 10, device = identity, rng) Create trajectory dataloader. """ -create_dataloader_post(trajectories; nunroll = 10, device = identity) = +create_dataloader_post(trajectories; nunroll = 10, device = identity, rng) = function dataloader() (; u, t) = rand(trajectories) nt = length(t) @assert nt ≥ nunroll - istart = rand(1:nt-nunroll) + istart = rand(rng, 1:nt-nunroll) it = istart:istart+nunroll (; u = device.(u[it]), t = t[it]) end diff --git a/libs/PaperDC/postanalysis.jl b/libs/PaperDC/postanalysis.jl index 3c44dbb8c..43217798e 100644 --- a/libs/PaperDC/postanalysis.jl +++ b/libs/PaperDC/postanalysis.jl @@ -42,9 +42,29 @@ outdir = "output/postanalysis" ispath(plotdir) || mkpath(plotdir) ispath(outdir) || mkpath(outdir) -# Random number generator -rng = Random.default_rng() -Random.seed!(rng, 12345) +# Random number generator seeds ################################################ +# +# Use a new RNG with deterministic seed for each code "section" +# so that e.g. training batch selection does not depend on whether we +# generated fresh filtered DNS data or loaded existing one (the +# generation of which would change the state of a global RNG). +# +# Note: Using `rng = Random.default_rng()` twice seems to point to the +# same RNG, and mutating one also mutates the other. +# `rng = Random.Xoshiro()` creates an independent copy each time +# (we could also do `rng = deepcopy(Random.default_rng())`). +# +# We define all the seeds here so that we don't accidentally type the same seed +# twice. + +seeds = (; + dns = 123, # Initial conditions + θ₀ = 234, # Initial CNN parameters + prior = 345, # A-priori training batch selection + post = 456, # A-posteriori training batch selection +) + +# Hardware selection ######################################################## # For running on CPU. # Consider reducing the sizes of DNS, LES, and CNN layers if @@ -63,6 +83,17 @@ CUDA.allowscalar(false); device = x -> adapt(CuArray, x) clean() = (GC.gc(); CUDA.reclaim()) +# Data generation ########################################################### +# +# Create filtered DNS data for training, validation, and testing. + +# Random number generator for initial conditions. +# Important: Created and seeded first, then shared for all initial conditions. +# After each initial condition generation, it is mutated and creates different +# IC for the next iteration. +rng = Random.Xoshiro() +Random.seed!(rng, seeds.dns) + # Parameters get_params(nlesscalar) = (; D = 2, @@ -81,6 +112,7 @@ get_params(nlesscalar) = (; kp = 20, psolver, ), + rng, ) # Get parameters for multiple LES resolutions @@ -175,17 +207,25 @@ let end end -# CNN architecture 1 -mname = "balzac" -closure, θ₀ = cnn(; - setup = setups_train[1], - radii = [2, 2, 2, 2], - channels = [20, 20, 20, params_train.D], - activations = [leakyrelu, leakyrelu, leakyrelu, identity], - use_bias = [true, true, true, false], - rng, -); -closure.chain +# CNN closure model ########################################################## + +# Random number generator for initial CNN parameters. +# All training sessions will start from the same θ₀ +# for a fair comparison. +rng = Random.Xoshiro() +Random.seed!(rng, seeds.θ₀) + +# # CNN architecture 1 +# mname = "balzac" +# closure, θ₀ = cnn(; +# setup = setups_train[1], +# radii = [2, 2, 2, 2], +# channels = [20, 20, 20, params_train.D], +# activations = [leakyrelu, leakyrelu, leakyrelu, identity], +# use_bias = [true, true, true, false], +# rng, +# ); +# closure.chain # CNN architecture 2 mname = "rimbaud" @@ -214,15 +254,19 @@ closure(device(io_train[1, 1].u[:, :, :, 1:50]), device(θ₀)); # Save parameters to disk after each run. # Plot training progress (for a validation data batch). +# Random number generator for batch selection +rng = Random.Xoshiro() +Random.seed!(rng, seeds.prior) + for ifil = 1:2, ig = 4:4 clean() starttime = time() println("ig = $ig, ifil = $ifil") - d = create_dataloader_prior(io_train[ig, ifil]; batchsize = 50, device) + d = create_dataloader_prior(io_train[ig, ifil]; batchsize = 50, device, rng) θ = T(1.0e0) * device(θ₀) loss = create_loss_prior(mean_squared_error, closure) opt = Optimisers.setup(Adam(T(1.0e-3)), θ) - it = rand(1:size(io_valid[ig, ifil].u, 4), 50) + it = rand(rng, 1:size(io_valid[ig, ifil].u, 4), 50) validset = device(map(v -> v[:, :, :, it], io_valid[ig, ifil])) (; callbackstate, callback) = create_callback( create_relerr_prior(closure, validset...); @@ -274,6 +318,10 @@ map(p -> p.comptime, prior) |> sum |> x -> x / 3600 # Hours # The time stepper `RKProject` allows for choosing when to project. let + # Random number generator for batch selection + rng = Random.Xoshiro() + Random.seed!(rng, seeds.post) + ngrid, nfilter = size(io_train) for iorder = 1:2, ifil = 1:nfilter, ig = 1:ngrid clean() @@ -289,7 +337,7 @@ let nupdate = 2, # Time steps per loss evaluation ) data = [(; u = d.data[ig, ifil].u, d.t) for d in data_train] - d = create_dataloader_post(data; device, nunroll = 20) + d = create_dataloader_post(data; device, nunroll = 20, rng) θ = copy(θ_cnn_prior[ig, ifil]) opt = Optimisers.setup(Adam(T(1.0e-3)), θ) it = 1:30 From fea03e91fb0d4df3517af745b2f58959c0e71c45 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Thu, 25 Apr 2024 17:14:13 +0200 Subject: [PATCH 316/379] Fix typo --- libs/PaperDC/postanalysis.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libs/PaperDC/postanalysis.jl b/libs/PaperDC/postanalysis.jl index 43217798e..33fcff431 100644 --- a/libs/PaperDC/postanalysis.jl +++ b/libs/PaperDC/postanalysis.jl @@ -3,7 +3,7 @@ # Generate filtered DNS data, train closure model, compare filters, # closure models, and projection orders. # -# The filtered DNS data is saved and can be loaded in a subesequent sesssion. +# The filtered DNS data is saved and can be loaded in a subesequent session. # The learned CNN parameters are also saved. using Adapt From c0cb20f9ae5dfee883443d89a3cf704012599845 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Thu, 25 Apr 2024 17:14:51 +0200 Subject: [PATCH 317/379] chore: Format --- libs/NeuralClosure/src/training.jl | 19 ++++++++++--------- libs/PaperDC/postanalysis.jl | 20 ++++---------------- 2 files changed, 14 insertions(+), 25 deletions(-) diff --git a/libs/NeuralClosure/src/training.jl b/libs/NeuralClosure/src/training.jl index d1b2fe815..63aada1e0 100644 --- a/libs/NeuralClosure/src/training.jl +++ b/libs/NeuralClosure/src/training.jl @@ -5,15 +5,16 @@ Create dataloader that uses a batch of `batchsize` random samples from `data` at each evaluation. The batch is moved to `device`. """ -create_dataloader_prior(data; batchsize = 50, device = identity, rng) = function dataloader() - x, y = data - nsample = size(x)[end] - d = ndims(x) - i = sort(shuffle(rng, 1:nsample)[1:batchsize]) - xuse = device(Array(selectdim(x, d, i))) - yuse = device(Array(selectdim(y, d, i))) - xuse, yuse -end +create_dataloader_prior(data; batchsize = 50, device = identity, rng) = + function dataloader() + x, y = data + nsample = size(x)[end] + d = ndims(x) + i = sort(shuffle(rng, 1:nsample)[1:batchsize]) + xuse = device(Array(selectdim(x, d, i))) + yuse = device(Array(selectdim(y, d, i))) + xuse, yuse + end """ create_dataloader_post(trajectories; nunroll = 10, device = identity, rng) diff --git a/libs/PaperDC/postanalysis.jl b/libs/PaperDC/postanalysis.jl index 33fcff431..0db99fc71 100644 --- a/libs/PaperDC/postanalysis.jl +++ b/libs/PaperDC/postanalysis.jl @@ -106,12 +106,8 @@ get_params(nlesscalar) = (; filters = (FaceAverage(), VolumeAverage()), ArrayType, PSolver = SpectralPressureSolver, - icfunc = (setup, psolver) -> random_field( - setup, - zero(eltype(setup.grid.x[1])); - kp = 20, - psolver, - ), + icfunc = (setup, psolver) -> + random_field(setup, zero(eltype(setup.grid.x[1])); kp = 20, psolver), rng, ) @@ -274,16 +270,8 @@ for ifil = 1:2, ig = 4:4 displayref = true, display_each_iteration = false, # Set to `true` if using CairoMakie ) - (; opt, θ, callbackstate) = train( - [d], - loss, - opt, - θ; - niter = 10_000, - ncallback = 20, - callbackstate, - callback, - ) + (; opt, θ, callbackstate) = + train([d], loss, opt, θ; niter = 10_000, ncallback = 20, callbackstate, callback) θ = callbackstate.θmin # Use best θ instead of last θ prior = (; θ = Array(θ), comptime = time() - starttime, callbackstate.hist) jldsave("$savepath/prior_ifilter$(ifil)_igrid$(ig).jld2"; prior) From c049672d5bb9f9eb3dfda2b16f05ce7cd397d10d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Mon, 29 Apr 2024 10:51:18 +0200 Subject: [PATCH 318/379] Update old code --- src/time_steppers/step_one_leg.jl | 184 +++++++++++------------ src/time_steppers/time_stepper_caches.jl | 16 +- 2 files changed, 95 insertions(+), 105 deletions(-) diff --git a/src/time_steppers/step_one_leg.jl b/src/time_steppers/step_one_leg.jl index 77eef22c0..e84331a1c 100644 --- a/src/time_steppers/step_one_leg.jl +++ b/src/time_steppers/step_one_leg.jl @@ -2,177 +2,169 @@ create_stepper( method::OneLegMethod; setup, psolver, - bc_vectors, - V, - p, + u, t, n = 0, + p = pressure(u, t, setup; psolver), # For the first step, these are not used - Vₙ = copy(V), - pₙ = copy(p), - tₙ = t, -) = (; setup, psolver, bc_vectors, V, p, t, n, Vₙ, pₙ, tₙ) + uold = copy.(u), + pold = copy(p), + told = t, +) = (; setup, psolver, u, p, t, n, uold, pold, told) -function timestep(method::OneLegMethod, stepper, Δt) - (; setup, psolver, bc_vectors, V, p, t, n, Vₙ, pₙ, tₙ) = stepper +function timestep(method::OneLegMethod, stepper, Δt; θ = nothing) + (; setup, psolver, u, p, t, n, uold, pold, told) = stepper (; p_add_solve, β, method_startup) = method - (; grid, operators, boundary_conditions) = setup - (; bc_unsteady) = boundary_conditions - (; G, M) = operators + (; grid, boundary_conditions) = setup (; Ω) = grid + T = typeof(Δt) # One-leg requires state at previous time step, which is not available at # the first iteration. Do one startup step instead if n == 0 - stepper_startup = - create_stepper(method_startup; setup, psolver, bc_vectors, V, p, t) - n += 1 - Vₙ = V - pₙ = p - tₙ = t - (; V, p, t) = timestep(method_startup, stepper_startup, Δt) - return create_stepper(method; setup, psolver, bc_vectors, V, p, t, n, Vₙ, pₙ, tₙ) + stepper_startup = create_stepper(method_startup; setup, psolver, u, t) + (; u, t, n) = timestep(method_startup, stepper_startup, Δt) + p = pressure(u, t, setup; psolver) + return create_stepper(method; setup, psolver, u, p, t, n, uold, pold, told) end - # Update current solution - Δtₙ₋₁ = t - tₙ - n += 1 - Vₙ₋₁ = Vₙ - pₙ₋₁ = pₙ - Vₙ = V - pₙ = p - tₙ = t - Δtₙ = Δt - # One-leg requires fixed time step - @assert Δtₙ ≈ Δtₙ₋₁ + @assert Δt ≈ t - told # Intermediate ("offstep") velocities - t = tₙ + β * Δtₙ - V = @. (1 + β) * Vₙ - β * Vₙ₋₁ - p = @. (1 + β) * pₙ - β * pₙ₋₁ + tβ = t + β * Δt + uβ = ntuple(α -> @.((1 + β) * u[α] - β * uold[α]), length(u)) + pβ = @. (1 + β) * p - β * pold # Right-hand side of the momentum equation - F, = momentum(V, V, p, t, setup; bc_vectors) + F = momentum(uβ, tβ, setup) + G = pressuregradient(pβ, setup) # Take a time step with this right-hand side, this gives an intermediate velocity field # (not divergence free) - V = @. (2β * Vₙ - (β - 1 // 2) * Vₙ₋₁ + Δtₙ / Ω * F) / (β + 1 // 2) + unew = ntuple( + α -> @.( + (2β * u[α] - (β - T(1) / 2) * uold[α] + Δt * (F[α] - G[α])) / (β + T(1) / 2) + ), + length(u), + ) # To make the velocity field uₙ₊₁ at tₙ₊₁ divergence-free we need the boundary # conditions at tₙ₊₁ - if bc_unsteady - bc_vectors = get_bc_vectors(setup, tₙ + Δtₙ) - end - (; yM) = bc_vectors + unew = apply_bc_u(unew, t + Δt, setup) # Adapt time step for pressure calculation - Δtᵦ = Δtₙ / (β + 1 // 2) + Δtᵦ = Δt / (β + T(1) / 2) # Divergence of intermediate velocity field - f = (M * V + yM) / Δtᵦ + div = divergence(unew, setup) / Δtᵦ + div = @. div * Ω # Solve the Poisson equation for the pressure - Δp = poisson(psolver, f) - GΔp = G * Δp + Δp = poisson(psolver, div) + Δp = apply_bc_p(Δp, t + Δtᵦ, setup) + GΔp = pressuregradient(Δp, setup) # Update velocity field - V = @. V - Δtᵦ / Ω * GΔp + unew = ntuple(α -> @.(unew[α] - Δtᵦ * GΔp[α]), length(u)) + unew = apply_bc_u(unew, t + Δt, setup) # Update pressure (second order) - p = @. 2pₙ - pₙ₋₁ + 4 // 3 * Δp + pnew = @. 2 * p - pold + T(4) / 3 * Δp + pnew = apply_bc_p(pnew, t + Δt, setup) # Alternatively, do an additional Poisson solve if p_add_solve - p = pressure(psolver, V, p, tₙ + Δtₙ, setup; bc_vectors) + pnew = pressure(unew, t + Δt, setup; psolver) end - t = tₙ + Δtₙ + told = t + pold = p + uold = u + t = t + Δt + p = pnew + u = unew - create_stepper(method; setup, psolver, bc_vectors, V, p, t, n, Vₙ, pₙ, tₙ) + create_stepper(method; setup, psolver, u, p, t, n, uold, pold, told) end -function timestep!(method::OneLegMethod, stepper, Δt; cache, momentum_cache) - (; setup, psolver, bc_vectors, n, V, p, t, Vₙ, pₙ, tₙ) = stepper +function timestep!(method::OneLegMethod, stepper, Δt; θ = nothing, cache) + (; setup, psolver, u, p, t, n, uold, pold, told) = stepper (; p_add_solve, β, method_startup) = method - (; grid, operators, boundary_conditions) = setup - (; bc_unsteady) = boundary_conditions - (; G, M) = operators + (; grid, boundary_conditions) = setup (; Ω) = grid - (; Vₙ₋₁, pₙ₋₁, F, f, Δp, GΔp) = cache + (; unew, pnew, div, F, Δp) = cache + T = typeof(Δt) # One-leg requires state at previous time step, which is not available at # the first iteration. Do one startup step instead if n == 0 - stepper_startup = - create_stepper(method_startup; setup, psolver, bc_vectors, V, p, t) - n += 1 - Vₙ = V - pₙ = p - tₙ = t - - # Note: We do one out-of-place step here, with a few allocations - (; V, p, t) = timestep(method_startup, stepper_startup, Δt) - return create_stepper(method; setup, psolver, bc_vectors, V, p, t, n, Vₙ, pₙ, tₙ) + stepper_startup = create_stepper(method_startup; setup, psolver, u, t) + (; u, t, n) = timestep(method_startup, stepper_startup, Δt) + pressure!(p, u, t, setup; psolver, F, div) + return create_stepper(method; setup, psolver, u, p, t, n, uold, pold, told) end - # Update current solution - Δtₙ₋₁ = t - tₙ - n += 1 - Vₙ₋₁ .= Vₙ - pₙ₋₁ .= pₙ - Vₙ .= V - pₙ .= p - tₙ = t - Δtₙ = Δt - # One-leg requires fixed time step - @assert Δtₙ ≈ Δtₙ₋₁ + @assert Δt ≈ t - told # Intermediate ("offstep") velocities - t = tₙ + β * Δtₙ - @. V = (1 + β) * Vₙ - β * Vₙ₋₁ - @. p = (1 + β) * pₙ - β * pₙ₋₁ + tnew = t + β * Δt + for α = 1:length(u) + @. unew[α] = (1 + β) * u[α] - β * uold[α] + end + @. pnew = (1 + β) * p - β * pold # Right-hand side of the momentum equation - momentum!(F, nothing, V, V, p, t, setup, momentum_cache) + momentum!(F, unew, tnew, setup) + applypressure!(F, pnew, setup) # Take a time step with this right-hand side, this gives an intermediate velocity field # (not divergence free) - @. V = (2β * Vₙ - (β - 1 // 2) * Vₙ₋₁ + Δtₙ / Ω * F) / (β + 1 // 2) + for α = 1:length(u) + @. unew[α] = 2β * u[α] - (β - T(1) / 2) * uold[α] + Δt * F[α] / (β + T(1) / 2) + end # To make the velocity field uₙ₊₁ at tₙ₊₁ divergence-free we need the boundary # conditions at tₙ₊₁ - if bc_unsteady - bc_vectors = get_bc_vectors(setup, tₙ + Δtₙ) - end - (; yM) = bc_vectors + apply_bc_u!(unew, t + Δt, setup) # Adapt time step for pressure calculation - Δtᵦ = Δtₙ / (β + 1 // 2) + Δtᵦ = Δt / (β + T(1) / 2) # Divergence of intermediate velocity field - f .= yM - mul!(f, M, V, 1 / Δtᵦ, 1 / Δtᵦ) - # f .= (M * V + yM) / Δtᵦ + divergence!(div, unew, setup) + @. div *= Ω # Solve the Poisson equation for the pressure - poisson!(psolver, Δp, f) - mul!(GΔp, G, Δp) + poisson!(psolver, Δp, div) + apply_bc_p!(Δp, t + Δtᵦ, setup) # Update velocity field - @. V -= Δtᵦ / Ω * GΔp + applypressure!(unew, Δp, setup) + apply_bc_u!(unew, t + Δt, setup) # Update pressure (second order) - @. p = 2pₙ - pₙ₋₁ + 4 // 3 * Δp + @. pnew = 2 * p - pold + T(4) / 3 * Δp / Δtᵦ + apply_bc_p!(pnew, t + Δt, setup) # Alternatively, do an additional Poisson solve if p_add_solve - pressure!(psolver, V, p, tₙ + Δtₙ, setup, momentum_cache, F, f, Δp; bc_vectors) + pressure!(pnew, unew, t + Δt, setup; psolver, F, div) end - t = tₙ + Δtₙ + n += 1 + told = t + pold .= p + for α = 1:length(u) + uold[α] .= u[α] + end + t = t + Δt + p .= pnew + for α = 1:length(u) + u[α] .= unew[α] + end - create_stepper(method; setup, psolver, bc_vectors, V, p, t, n, Vₙ, pₙ, tₙ) + create_stepper(method; setup, psolver, u, p, t, n, uold, pold, told) end diff --git a/src/time_steppers/time_stepper_caches.jl b/src/time_steppers/time_stepper_caches.jl index 20a6ea9bc..b9ee9e741 100644 --- a/src/time_steppers/time_stepper_caches.jl +++ b/src/time_steppers/time_stepper_caches.jl @@ -23,15 +23,13 @@ function ode_method_cache(::AdamsBashforthCrankNicolsonMethod, setup, V, p) (; c₀, c₋₁, F, f, Δp, Rr, b, b₀, b₁, yDiff₀, yDiff₁, Gp₀) end -function ode_method_cache(::OneLegMethod{T}, setup, V, p) where {T} - (; NV, Np) = setup.grid - u₋₁ = zero(V) - p₋₁ = zero(p) - F = zero(V) - f = zero(p) - Δp = zero(p) - GΔp = zero(V) - (; u₋₁, p₋₁, F, f, Δp, GΔp) +function ode_method_cache(::OneLegMethod{T}, setup, u) where {T} + unew = zero.(u) + pnew = zero(u[1]) + div = zero(u[1]) + F = zero.(u) + Δp = zero(u[1]) + (; unew, pnew, F, div, Δp) end function ode_method_cache(method::ExplicitRungeKuttaMethod{T}, setup, u) where {T} From e30a0bc880006647d2b98a4987ec7e8cd49a5d0a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Mon, 29 Apr 2024 10:51:58 +0200 Subject: [PATCH 319/379] Force zero average --- src/solvers/pressure/poisson.jl | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/solvers/pressure/poisson.jl b/src/solvers/pressure/poisson.jl index 387e3bda5..d661eaeb4 100644 --- a/src/solvers/pressure/poisson.jl +++ b/src/solvers/pressure/poisson.jl @@ -190,6 +190,12 @@ function poisson!(solver::SpectralPressureSolver, p, f) # Solve for coefficients in Fourier space @. fhat = -phat / Ahat + # Pressure is determined up to constant. We set this to 0 (instead of + # phat[1] / 0 = Inf) + # Note use of singleton range 1:1 instead of scalar index 1 + # (otherwise CUDA gets annoyed) + fhat[1:1] .= 0 + # Transform back ldiv!(phat, plan, fhat) @. p[Ip] = real(phat) From e61d33d605d7efcb4451135ad7645e38fd591801 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Mon, 29 Apr 2024 10:52:41 +0200 Subject: [PATCH 320/379] Minor change --- src/processors/processors.jl | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/src/processors/processors.jl b/src/processors/processors.jl index e1b17f463..a045540b5 100644 --- a/src/processors/processors.jl +++ b/src/processors/processors.jl @@ -46,13 +46,14 @@ processor(initialize, finalize = (initialized, state) -> initialized) = Create processor that logs time step information. """ -timelogger(; nupdate = 1) = processor(function (state) - on(state) do (; t, n) - n % nupdate == 0 || return - @printf "Iteration %d\tt = %g\n" n t +timelogger(; nupdate = 1) = + processor() do state + on(state) do (; t, n) + n % nupdate == 0 || return + @printf "Iteration %d\tt = %g\n" n t + end + nothing end - nothing -end) """ vtk_writer(; From 063bd0fcd7c0cb5c9745152f0e8251f0cd3738fa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Mon, 29 Apr 2024 10:52:51 +0200 Subject: [PATCH 321/379] docs(PaperDC): Add title --- libs/PaperDC/prioranalysis.jl | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/libs/PaperDC/prioranalysis.jl b/libs/PaperDC/prioranalysis.jl index 71f99b5ce..850c0b4b6 100644 --- a/libs/PaperDC/prioranalysis.jl +++ b/libs/PaperDC/prioranalysis.jl @@ -1,3 +1,7 @@ +# # A-priori analysis: Filtered DNS (2D or 3D) +# +# Generate filtered DNS data and inspect its properties. + using CairoMakie using FFTW using GLMakie From eada869456e1fde4fe1a97189fce9fe7b8b40aa8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Mon, 29 Apr 2024 10:56:41 +0200 Subject: [PATCH 322/379] Add old changes --- scratch/energy.jl | 89 ++++++++++++++++++++++++++------ scratch/multigrid.jl | 10 ++-- src/processors/real_time_plot.jl | 5 +- 3 files changed, 81 insertions(+), 23 deletions(-) diff --git a/scratch/energy.jl b/scratch/energy.jl index 57046bb82..e37139b89 100644 --- a/scratch/energy.jl +++ b/scratch/energy.jl @@ -5,7 +5,8 @@ if isdefined(@__MODULE__, :LanguageServer) #src using .IncompressibleNavierStokes #src end #src -using GLMakie +# using GLMakie +using CairoMakie using IncompressibleNavierStokes using IncompressibleNavierStokes: apply_bc_u!, total_kinetic_energy, diffusion! @@ -30,13 +31,15 @@ clean() = (GC.gc(); CUDA.reclaim()) set_theme!() set_theme!(; GLMakie = (; scalefactor = 1.5)) -# 2D +# 2D T = Float64; -Re = T(4_000) -# ndns = 1024 +Re = T(10_000) +# ndns = 4096 +ndns = 1024 +# nles = 128 +# ndns = 256 # nles = 128 -ndns = 256 -nles = 32 +nles = 64 compression = ndns ÷ nles D = 2 kp = 20 @@ -75,6 +78,9 @@ observe_u(dns, les, compression; Δt, nupdate = 1) = apply_bc_u!(v, t, les.setup) Ku = total_kinetic_energy(u, dns.setup) Kv = total_kinetic_energy(v, les.setup) + push!(results.t, t) + push!(results.Ku, Ku) + push!(results.Kv, Kv) if n == 0 push!(results.Kuref, Ku) push!(results.Kvref, Kv) @@ -99,14 +105,12 @@ observe_u(dns, les, compression; Δt, nupdate = 1) = results.Kvref[end] + nupdate * Δt * sum(sum.(diffv)) / nles^D, ) end - push!(results.t, t) - push!(results.Ku, Ku) - push!(results.Kv, Kv) end state[] = state[] # Save initial conditions results end +# Δt = 5e-5 Δt = 1e-4 # Solve unsteady problem @@ -114,12 +118,12 @@ observe_u(dns, les, compression; Δt, nupdate = 1) = dns.setup, u₀, # state.u, - (T(0), T(1e-1)); + (T(0), T(5e-1)); Δt, docopy = true, dns.psolver, processors = ( - # rtp = realtimeplotter(; dns.setup, nupdate = 5), + rtp = realtimeplotter(; dns.setup, displayupdates = true, nupdate = 50), obs = observe_u(dns, les, compression; Δt, nupdate = 1), log = timelogger(; nupdate = 1), ), @@ -136,13 +140,64 @@ using CairoMakie fig = with_theme() do fig = Figure() - ax = Axis(fig[1, 1]) - lines!(ax, outputs.obs.t, outputs.obs.Ku; label = "Ku") - lines!(ax, outputs.obs.t, outputs.obs.Kuref; label = "Kuref") - lines!(ax, outputs.obs.t, outputs.obs.Kv; label = "Kv") - lines!(ax, outputs.obs.t, outputs.obs.Kvref; label = "Kvref") - axislegend() + ax = Axis(fig[1, 1]; xlabel = "t", title = "Kinetic energy") + lines!( + ax, + outputs.obs.t, + outputs.obs.Ku; + color = Cycled(1), + linestyle = :solid, + label = "DNS", + ) + lines!( + ax, + outputs.obs.t, + outputs.obs.Kuref; + color = Cycled(1), + linestyle = :dash, + label = "DNS (reference)", + ) + lines!( + ax, + outputs.obs.t, + outputs.obs.Kv; + color = Cycled(2), + linestyle = :solid, + label = "Filtered DNS", + ) + lines!( + ax, + outputs.obs.t, + outputs.obs.Kvref; + color = Cycled(2), + linestyle = :dash, + label = "Filtered DNS (reference)", + ) + axislegend(ax) fig end save("$output/energy.pdf", fig) + +fig = with_theme() do + fig = Figure() + ax = Axis(fig[1, 1]) + lines!( + ax, + outputs.obs.t, + outputs.obs.Kv ./ outputs.obs.Ku; + color = Cycled(2), + linestyle = :solid, + label = "Filtered DNS", + ) + lines!( + ax, + outputs.obs.t, + outputs.obs.Kvref ./ outputs.obs.Ku; + color = Cycled(2), + linestyle = :dash, + label = "Filtered DNS (theoretical)", + ) + # axislegend(ax) + fig +end diff --git a/scratch/multigrid.jl b/scratch/multigrid.jl index 1e470d10e..b3d33e3f9 100644 --- a/scratch/multigrid.jl +++ b/scratch/multigrid.jl @@ -97,10 +97,10 @@ io_train = create_io_arrays(data_train, setups_train); io_valid = create_io_arrays(data_valid, setups_valid); # Inspect data -ig = 5 -# field, setup = data_train[1].u[ig], setups_train[ig]; +ig = 3 +field, setup = data_train[1].u[ig], setups_train[ig]; # field, setup = data_valid[1].u[ig], setups_valid[ig]; -field, setup = data_test.u[ig], setups_test[ig]; +# field, setup = data_test.u[ig], setups_test[ig]; u = device(field[1]); o = Observable((; u, t = nothing)); energy_spectrum_plot(o; setup) @@ -189,8 +189,8 @@ fig = with_theme(; palette = (; color = ["#3366cc", "#cc0000", "#669900", "#ffcc k = reshape(k, :) # Sum or average wavenumbers between k and k+1 nk = ceil(Int, maximum(k)) - # kmax = minimum(K) - 1 - kmax = nk - 1 + kmax = minimum(K) - 1 + # kmax = nk - 1 kint = 1:kmax ia = similar(xp[1], Int, 0) ib = sortperm(k) diff --git a/src/processors/real_time_plot.jl b/src/processors/real_time_plot.jl index b54567015..35b2d92ea 100644 --- a/src/processors/real_time_plot.jl +++ b/src/processors/real_time_plot.jl @@ -369,6 +369,7 @@ function energy_spectrum_plot( D = dimension() (; A, κ, K) = spectral_stuff(setup; npoint, a) + # (; masks, κ, K) = get_spectrum(setup; npoint, a) # Mask kmax = maximum(κ) # Energy @@ -379,10 +380,12 @@ function energy_spectrum_plot( e = sum(up) do u u = u[Ip] uhat = fft(u)[ntuple(α -> 1:K[α], D)...] + # uhat = fft(u)[ntuple(α -> 1:K, D)...] # Mask abs2.(uhat) ./ (2 * prod(size(uhat))^2) end e = A * reshape(e, :) - # e = max.(e, eps(T)) # Avoid log(0) + # e = [sum(e[m]) for m in masks] # Mask + e = max.(e, eps(T)) # Avoid log(0) Array(e) end From 6fabaced8ccd077abbee8ba3d3a1284dfc53cd4c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Mon, 29 Apr 2024 11:22:31 +0200 Subject: [PATCH 323/379] docs(PaperDC): Remove statement --- libs/PaperDC/postanalysis.jl | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/libs/PaperDC/postanalysis.jl b/libs/PaperDC/postanalysis.jl index 0db99fc71..1d9998d7b 100644 --- a/libs/PaperDC/postanalysis.jl +++ b/libs/PaperDC/postanalysis.jl @@ -51,8 +51,7 @@ ispath(outdir) || mkpath(outdir) # # Note: Using `rng = Random.default_rng()` twice seems to point to the # same RNG, and mutating one also mutates the other. -# `rng = Random.Xoshiro()` creates an independent copy each time -# (we could also do `rng = deepcopy(Random.default_rng())`). +# `rng = Random.Xoshiro()` creates an independent copy each time. # # We define all the seeds here so that we don't accidentally type the same seed # twice. From 28dc31ea865e104209eb4671e922792c5c486e7b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Mon, 29 Apr 2024 14:06:24 +0200 Subject: [PATCH 324/379] clean(PaperDC): Remove duplicate names --- libs/PaperDC/postanalysis.jl | 209 +++++++++++++---------------------- 1 file changed, 78 insertions(+), 131 deletions(-) diff --git a/libs/PaperDC/postanalysis.jl b/libs/PaperDC/postanalysis.jl index 1d9998d7b..c95bb527f 100644 --- a/libs/PaperDC/postanalysis.jl +++ b/libs/PaperDC/postanalysis.jl @@ -249,40 +249,52 @@ closure(device(io_train[1, 1].u[:, :, :, 1:50]), device(θ₀)); # Save parameters to disk after each run. # Plot training progress (for a validation data batch). -# Random number generator for batch selection -rng = Random.Xoshiro() -Random.seed!(rng, seeds.prior) +priornames = map(CartesianIndices(io_train)) do I + ig, ifil = I.I + "$savepath/prior_ifilter$(ifil)_igrid$(ig).jld2" +end -for ifil = 1:2, ig = 4:4 +# Train +let + # Random number generator for batch selection + rng = Random.Xoshiro() + Random.seed!(rng, seeds.prior) + ngrid, nfilter = size(io_train) + for ifil = 1:nfilter, ig = 1:ngrid + clean() + starttime = time() + println("ig = $ig, ifil = $ifil") + d = create_dataloader_prior(io_train[ig, ifil]; batchsize = 50, device, rng) + θ = T(1.0e0) * device(θ₀) + loss = create_loss_prior(mean_squared_error, closure) + opt = Optimisers.setup(Adam(T(1.0e-3)), θ) + it = rand(rng, 1:size(io_valid[ig, ifil].u, 4), 50) + validset = device(map(v -> v[:, :, :, it], io_valid[ig, ifil])) + (; callbackstate, callback) = create_callback( + create_relerr_prior(closure, validset...); + θ, + displayref = true, + display_each_iteration = false, # Set to `true` if using CairoMakie + ) + (; opt, θ, callbackstate) = train( + [d], + loss, + opt, + θ; + niter = 10_000, + ncallback = 20, + callbackstate, + callback, + ) + θ = callbackstate.θmin # Use best θ instead of last θ + prior = (; θ = Array(θ), comptime = time() - starttime, callbackstate.hist) + jldsave(priorfiles[ig, ifil]; prior) + end clean() - starttime = time() - println("ig = $ig, ifil = $ifil") - d = create_dataloader_prior(io_train[ig, ifil]; batchsize = 50, device, rng) - θ = T(1.0e0) * device(θ₀) - loss = create_loss_prior(mean_squared_error, closure) - opt = Optimisers.setup(Adam(T(1.0e-3)), θ) - it = rand(rng, 1:size(io_valid[ig, ifil].u, 4), 50) - validset = device(map(v -> v[:, :, :, it], io_valid[ig, ifil])) - (; callbackstate, callback) = create_callback( - create_relerr_prior(closure, validset...); - θ, - displayref = true, - display_each_iteration = false, # Set to `true` if using CairoMakie - ) - (; opt, θ, callbackstate) = - train([d], loss, opt, θ; niter = 10_000, ncallback = 20, callbackstate, callback) - θ = callbackstate.θmin # Use best θ instead of last θ - prior = (; θ = Array(θ), comptime = time() - starttime, callbackstate.hist) - jldsave("$savepath/prior_ifilter$(ifil)_igrid$(ig).jld2"; prior) end -clean() # Load learned parameters and training times -prior = map(CartesianIndices(size(io_train))) do I - ig, ifil = I.I - name = "$path/prior_ifilter$(ifil)_igrid$(ig).jld2" - load(name)["prior"] -end; +prior = map(f -> load(f)["prior"], priorfiles) θ_cnn_prior = [copyto!(device(θ₀), p.θ) for p in prior]; # Check that parameters are within reasonable bounds @@ -304,11 +316,16 @@ map(p -> p.comptime, prior) |> sum |> x -> x / 3600 # Hours # # The time stepper `RKProject` allows for choosing when to project. +postfiles = map(CartesianIndices((size(io_train)..., 2))) do I + ig, ifil, iorder = I.I + "$savepath/post_iorder$(iorder)_ifil$(ifil)_ig$(ig).jld2" +end + +# Train let # Random number generator for batch selection rng = Random.Xoshiro() Random.seed!(rng, seeds.post) - ngrid, nfilter = size(io_train) for iorder = 1:2, ifil = 1:nfilter, ig = 1:ngrid clean() @@ -346,17 +363,13 @@ let train([d], loss, opt, θ; niter = 2000, ncallback = 10, callbackstate, callback) θ = callbackstate.θmin # Use best θ instead of last θ post = (; θ = Array(θ), comptime = time() - starttime) - jldsave("$savepath/post_iorder$(iorder)_ifil$(ifil)_ig$(ig).jld2"; post) + jldsave(postfiles[iorder, ifil, ig]; post) end clean() end # Load learned parameters and training times -post = map(CartesianIndices((size(io_train)..., 2))) do I - ig, ifil, iorder = I.I - name = "$savepath/post_iorder$(iorder)_ifil$(ifil)_ig$(ig).jld2" - load(name)["post"] -end; +post = map(f -> load(f)["post"], postfiles); θ_cnn_post = [copyto!(device(θ₀), p.θ) for p in post]; # Check that parameters are within reasonable bounds @@ -845,7 +858,6 @@ divs = let state[] = state[] # Compute initial divergence dhist end - processors = (; dwriter) if iorder == 1 # Does not depend on projection order d_ref[ig, ifil] = map(data_test.data[ig, ifil].u) do u @@ -856,50 +868,25 @@ divs = let d = sqrt(d) end end - iorder_use = iorder == 3 ? 2 : iorder - d_nomodel[ig, ifil, iorder] = + s(closure_model, θ) = solve_unsteady( - setup, + (; setup..., closure_model), u₀, tlims; method = RKProject(RK44(; T), getorder(iorder)), Δt, - processors, + processors = (; dwriter), psolver, + θ, )[2].dwriter + iorder_use = iorder == 3 ? 2 : iorder + d_nomodel[ig, ifil, iorder] = s(nothing, nothing) d_smag[ig, ifil, iorder] = - solve_unsteady( - (; setup..., closure_model = smagorinsky_closure(setup)), - u₀, - tlims; - method = RKProject(RK44(; T), getorder(iorder)), - Δt, - processors, - psolver, - θ = θ_smag[ifil, iorder_use], - )[2].dwriter + s(smagorinsky_closure(setup), θ_smag[ifil, iorder_use]) d_cnn_prior[ig, ifil, iorder] = - solve_unsteady( - (; setup..., closure_model = wrappedclosure(closure, setup)), - u₀, - tlims; - method = RKProject(RK44(; T), getorder(iorder)), - Δt, - processors, - psolver, - θ = θ_cnn_prior[ig, ifil], - )[2].dwriter + s(wrappedclosure(closure, setup), θ_cnn_prior[ig, ifil]) d_cnn_post[ig, ifil, iorder] = - solve_unsteady( - (; setup..., closure_model = wrappedclosure(closure, setup)), - u₀, - tlims; - method = RKProject(RK44(; T), getorder(iorder)), - Δt, - processors, - psolver, - θ = θ_cnn_post[ig, ifil, iorder_use], - )[2].dwriter + s(wrappedclosure(closure, setup), θ_cnn_post[ig, ifil, iorder_use]) end (; d_ref, d_nomodel, d_smag, d_cnn_prior, d_cnn_post) end; @@ -922,8 +909,9 @@ with_theme(; palette, ) do t = data_test.t - for islog in (true, false) - for iorder = 1:3, ifil = 1:2, igrid = 1:3 + # for islog in (true, false) + for islog in (false,) + for iorder = 1:2, ifil = 1:2, igrid = 1:3 println("iorder = $iorder, ifil = $ifil, igrid = $igrid") lesmodel = if iorder == 1 "DIF" @@ -935,14 +923,16 @@ with_theme(; fil = ifil == 1 ? "FA" : "VA" nles = params_test.nles[igrid] fig = Figure(; size = (500, 400)) - yscale = islog ? log10 : identity ax = Axis( fig[1, 1]; - yscale, + yscale = islog ? log10 : identity, xlabel = "t", title = "Divergence: $lesmodel, $fil, $nles", ) - linestyle = ifil == 1 ? :solid : :dash + lines!(ax, t, divs.d_nomodel[igrid, ifil, iorder]; label = "No closure") + lines!(ax, t, divs.d_smag[igrid, ifil, iorder]; label = "Smagorinsky") + lines!(ax, t, divs.d_cnn_prior[igrid, ifil, iorder]; label = "CNN (prior)") + lines!(ax, t, divs.d_cnn_post[igrid, ifil, iorder]; label = "CNN (post)") lines!( ax, t, @@ -951,34 +941,6 @@ with_theme(; linestyle = :dash, label = "Reference", ) - lines!( - ax, - t, - divs.d_nomodel[igrid, ifil, iorder]; - color = Cycled(1), - label = "No closure", - ) - lines!( - ax, - t, - divs.d_smag[igrid, ifil, iorder]; - color = Cycled(2), - label = "Smagorinsky", - ) - lines!( - ax, - t, - divs.d_cnn_prior[igrid, ifil, iorder]; - color = Cycled(3), - label = "CNN (prior)", - ) - lines!( - ax, - t, - divs.d_cnn_post[igrid, ifil, iorder]; - color = Cycled(4), - label = "CNN (post)", - ) iorder == 2 && ifil == 1 && axislegend(; position = :rt) islog && ylims!(ax, (T(1e-6), T(1e3))) name = "$plotdir/divergence/$mname/$(islog ? "log" : "lin")" @@ -1009,42 +971,27 @@ ufinal = let nupdate = 2 Δt = (t[2] - t[1]) / nupdate T = eltype(u₀[1]) - if iorder == 1 - # Does not depend on projection order - u_ref[igrid, ifil] = data_test.data[igrid, ifil].u[end] - u_nomodel[igrid, ifil] = - solve_unsteady(setup, u₀, tlims; Δt, psolver)[1].u .|> Array - end - u_smag[igrid, ifil, iorder] = + s(closure_model, θ) = solve_unsteady( - (; setup..., closure_model = smagorinsky_closure(setup)), + (; setup..., closure_model), u₀, tlims; method = RKProject(RK44(; T), getorder(iorder)), Δt, psolver, - θ = θ_smag[ifil, iorder], + θ, )[1].u .|> Array + if iorder == 1 + # Does not depend on projection order + u_ref[igrid, ifil] = data_test.data[igrid, ifil].u[end] + u_nomodel[igrid, ifil] = s(nothing, nothing) + end + u_smag[igrid, ifil, iorder] = + s(smagorinsky_closure(setup), θ_smag[ifil, iorder]) u_cnn_prior[igrid, ifil, iorder] = - solve_unsteady( - (; setup..., closure_model = wrappedclosure(closure, setup)), - u₀, - tlims; - method = RKProject(RK44(; T), getorder(iorder)), - Δt, - psolver, - θ = θ_cnn_prior[igrid, ifil], - )[1].u .|> Array + s(wrappedclosure(closure, setup), θ_cnn_prior[igrid, ifil]) u_cnn_post[igrid, ifil, iorder] = - solve_unsteady( - (; setup..., closure_model = wrappedclosure(closure, setup)), - u₀, - tlims; - method = RKProject(RK44(; T), getorder(iorder)), - Δt, - psolver, - θ = θ_cnn_post[igrid, ifil, iorder], - )[1].u .|> Array + s(wrappedclosure(closure, setup), θ_cnn_post[igrid, ifil, iorder]) end (; u_ref, u_nomodel, u_smag, u_cnn_prior, u_cnn_post) end; From 16197cc1e9877e5c452dd13764a0bdfa192159b0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Mon, 29 Apr 2024 14:31:37 +0200 Subject: [PATCH 325/379] docs(NeuralClosure): Update README --- libs/NeuralClosure/README.md | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/libs/NeuralClosure/README.md b/libs/NeuralClosure/README.md index 792d724d6..c3f08b77b 100644 --- a/libs/NeuralClosure/README.md +++ b/libs/NeuralClosure/README.md @@ -2,3 +2,24 @@ Neural closure modeling tools for [IncompressibleNavierStokes.jl](https://github.com/agdestein/IncompressibleNavierStokes.jl). + +## Set up environment + +From this directory, run: + +```sh +julia --project -e ' +using Pkg +Pkg.develop(PackageSpec(; path = "../..")) +Pkg.instantiate() +' +``` + +or interactively from a Julia REPL: + +```julia-repl +julia> ] +(v1.10) pkg> activate . +(NeuralClosure) pkg> dev ../.. +(NeuralClosure) pkg> instantiate +``` From 94c97e8e1539ba8e56a46a8497e1c9a036cdccb2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Thu, 2 May 2024 22:11:29 +0200 Subject: [PATCH 326/379] Fix averaging weights in non-uniform case --- src/grid/grid.jl | 6 ++++-- src/operators.jl | 49 ++++++++++++++++++++++++++++++++++-------------- 2 files changed, 39 insertions(+), 16 deletions(-) diff --git a/src/grid/grid.jl b/src/grid/grid.jl index ee0ef73e1..d0a3ab442 100644 --- a/src/grid/grid.jl +++ b/src/grid/grid.jl @@ -115,8 +115,10 @@ function Grid(x, boundary_conditions; ArrayType = Array) Aαβ2[end] = 1 else # Interpolation from α-face center to left (1) or right (2) α-face β-edge - Aαβ1 = [(x[β][i] - xp[β][i-1]) / Δu[β][i-1] for i = 2:N[β]] - Aαβ2 = 1 .- Aαβ1 + # Aαβ1 = [(x[β][i] - xp[β][i-1]) / Δu[β][i-1] for i = 2:N[β]] + # Aαβ2 = 1 .- Aαβ1 + Aαβ2 = [(x[β][i] - xp[β][i-1]) / Δu[β][i-1] for i = 2:N[β]] + Aαβ1 = 1 .- Aαβ2 pushfirst!(Aαβ1, 1) push!(Aαβ2, 1) end diff --git a/src/operators.jl b/src/operators.jl index 90a225af2..99a8e7d20 100644 --- a/src/operators.jl +++ b/src/operators.jl @@ -179,12 +179,33 @@ function convection!(F, u, setup) # for β = 1:D KernelAbstractions.Extras.LoopInfo.@unroll for β in βrange Δuαβ = α == β ? Δu[β] : Δ[β] - uαβ1 = A[α][β][2][I[β]-1] * u[α][I-δ(β)] + A[α][β][1][I[β]] * u[α][I] + + # Half for u[α], (reverse!) interpolation for u[β] + # Note: + # In matrix version, uses + # 1*u[α][I-δ(β)] + 0*u[α][I] + # instead of 1/2 when u[α][I-δ(β)] is at Dirichlet boundary. + uαβ1 = (u[α][I-δ(β)] + u[α][I]) / 2 + uαβ2 = (u[α][I] + u[α][I+δ(β)]) / 2 uβα1 = A[β][α][2][I[α]-(α==β)] * u[β][I-δ(β)] + A[β][α][1][I[α]+(α!=β)] * u[β][I-δ(β)+δ(α)] - uαβ2 = A[α][β][2][I[β]] * u[α][I] + A[α][β][1][I[β]+1] * u[α][I+δ(β)] uβα2 = A[β][α][2][I[α]] * u[β][I] + A[β][α][1][I[α]+1] * u[β][I+δ(α)] + + # # Half + # uαβ1 = (u[α][I-δ(β)] + u[α][I]) / 2 + # uβα1 = u[β][I-δ(β)] / 2 + u[β][I-δ(β)+δ(α)] / 2 + # uαβ2 = (u[α][I] + u[α][I+δ(β)]) / 2 + # uβα2 = u[β][I] / 2 + u[β][I+δ(α)] / 2 + + # # Interpolation + # uαβ1 = A[α][β][2][I[β]-1] * u[α][I-δ(β)] + A[α][β][1][I[β]] * u[α][I] + # uβα1 = + # A[β][α][2][I[α]-(α==β)] * u[β][I-δ(β)] + + # A[β][α][1][I[α]+(α!=β)] * u[β][I-δ(β)+δ(α)] + # uαβ2 = A[α][β][2][I[β]] * u[α][I] + A[α][β][1][I[β]+1] * u[α][I+δ(β)] + # uβα2 = A[β][α][2][I[α]] * u[β][I] + A[β][α][1][I[α]+1] * u[β][I+δ(α)] + F[α][I] -= (uαβ2 * uβα2 - uαβ1 * uβα1) / Δuαβ[I[β]] end end @@ -201,20 +222,20 @@ function convection_adjoint!(ubar, φbar, u, setup) (; dimension, Δ, Δu, N, Iu, A) = grid D = dimension() δ = Offset{D}() + T = eltype(u[1]) + h = T(1) / 2 @kernel function adj!(ubar, φbar, u, ::Val{γ}, ::Val{looprange}) where {γ,looprange} J = @index(Global, Cartesian) KernelAbstractions.Extras.LoopInfo.@unroll for α in looprange KernelAbstractions.Extras.LoopInfo.@unroll for β in looprange Δuαβ = α == β ? Δu[β] : Δ[β] - Aαβ1 = A[α][β][1] - Aαβ2 = A[α][β][2] Aβα1 = A[β][α][1] Aβα2 = A[β][α][2] # 1 I = J if α == γ && I in Iu[α] - uαβ2 = Aαβ2[I[β]] + uαβ2 = h uβα2 = Aβα2[I[α]] * u[β][I] + Aβα1[I[α]+1] * u[β][I+δ(α)] dφdu = -uαβ2 * uβα2 / Δuαβ[I[β]] ubar[γ][J] += φbar[α][I] * dφdu @@ -223,7 +244,7 @@ function convection_adjoint!(ubar, φbar, u, setup) # 2 I = J - δ(β) if α == γ && I in Iu[α] - uαβ2 = Aαβ1[I[β]+1] + uαβ2 = h uβα2 = Aβα2[I[α]] * u[β][I] + Aβα1[I[α]+1] * u[β][I+δ(α)] dφdu = -uαβ2 * uβα2 / Δuαβ[I[β]] ubar[γ][J] += φbar[α][I] * dφdu @@ -232,7 +253,7 @@ function convection_adjoint!(ubar, φbar, u, setup) # 3 I = J if β == γ && I in Iu[α] - uαβ2 = Aαβ2[I[β]] * u[α][I] + Aαβ1[I[β]+1] * u[α][I+δ(β)] + uαβ2 = h * u[α][I] + h * u[α][I+δ(β)] uβα2 = Aβα2[I[α]] dφdu = -uαβ2 * uβα2 / Δuαβ[I[β]] ubar[γ][J] += φbar[α][I] * dφdu @@ -241,7 +262,7 @@ function convection_adjoint!(ubar, φbar, u, setup) # 4 I = J - δ(α) if β == γ && I in Iu[α] - uαβ2 = Aαβ2[I[β]] * u[α][I] + Aαβ1[I[β]+1] * u[α][I+δ(β)] + uαβ2 = h * u[α][I] + h * u[α][I+δ(β)] uβα2 = Aβα1[I[α]+1] dφdu = -uαβ2 * uβα2 / Δuαβ[I[β]] ubar[γ][J] += φbar[α][I] * dφdu @@ -250,7 +271,7 @@ function convection_adjoint!(ubar, φbar, u, setup) # 5 I = J + δ(β) if α == γ && I in Iu[α] - uαβ1 = Aαβ2[I[β]-1] + uαβ1 = h uβα1 = Aβα2[I[α]-(α==β)] * u[β][I-δ(β)] + Aβα1[I[α]+(α!=β)] * u[β][I-δ(β)+δ(α)] @@ -261,7 +282,7 @@ function convection_adjoint!(ubar, φbar, u, setup) # 6 I = J if α == γ && I in Iu[α] - uαβ1 = Aαβ1[I[β]] + uαβ1 = h uβα1 = Aβα2[I[α]-(α==β)] * u[β][I-δ(β)] + Aβα1[I[α]+(α!=β)] * u[β][I-δ(β)+δ(α)] @@ -272,7 +293,7 @@ function convection_adjoint!(ubar, φbar, u, setup) # 7 I = J + δ(β) if β == γ && I in Iu[α] - uαβ1 = Aαβ2[I[β]-1] * u[α][I-δ(β)] + Aαβ1[I[β]] * u[α][I] + uαβ1 = h * u[α][I-δ(β)] + h * u[α][I] uβα1 = Aβα2[I[α]-(α==β)] dφdu = uαβ1 * uβα1 / Δuαβ[I[β]] ubar[γ][J] += φbar[α][I] * dφdu @@ -281,7 +302,7 @@ function convection_adjoint!(ubar, φbar, u, setup) # 8 I = J + δ(β) - δ(α) if β == γ && I in Iu[α] - uαβ1 = Aαβ2[I[β]-1] * u[α][I-δ(β)] + Aαβ1[I[β]] * u[α][I] + uαβ1 = h * u[α][I-δ(β)] + h * u[α][I] uβα1 = Aβα1[I[α]+(α!=β)] dφdu = uαβ1 * uβα1 / Δuαβ[I[β]] ubar[γ][J] += φbar[α][I] * dφdu @@ -392,11 +413,11 @@ function convectiondiffusion!(F, u, setup) # for β = 1:D KernelAbstractions.Extras.LoopInfo.@unroll for β in βrange Δuαβ = α == β ? Δu[β] : Δ[β] - uαβ1 = A[α][β][2][I[β]-1] * u[α][I-δ(β)] + A[α][β][1][I[β]] * u[α][I] + uαβ1 = (u[α][I-δ(β)] + u[α][I]) / 2 + uαβ2 = (u[α][I] + u[α][I+δ(β)]) / 2 uβα1 = A[β][α][2][I[α]-(α==β)] * u[β][I-δ(β)] + A[β][α][1][I[α]+(α!=β)] * u[β][I-δ(β)+δ(α)] - uαβ2 = A[α][β][2][I[β]] * u[α][I] + A[α][β][1][I[β]+1] * u[α][I+δ(β)] uβα2 = A[β][α][2][I[α]] * u[β][I] + A[β][α][1][I[α]+1] * u[β][I+δ(α)] uαuβ1 = uαβ1 * uβα1 uαuβ2 = uαβ2 * uβα2 From d911d04f66d2a23a3bbcefbecf33d0929fc381c3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Thu, 2 May 2024 22:12:08 +0200 Subject: [PATCH 327/379] Fill with zero --- src/create_initial_conditions.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/create_initial_conditions.jl b/src/create_initial_conditions.jl index 894bc38a6..2c770d995 100644 --- a/src/create_initial_conditions.jl +++ b/src/create_initial_conditions.jl @@ -25,7 +25,7 @@ function create_initial_conditions( D = dimension() # Allocate velocity - u = ntuple(d -> similar(x[1], N), D) + u = ntuple(d -> fill!(similar(x[1], N), 0), D) # Initial velocities for α = 1:D From 9684ef6c00769bde3c2843a51e1e0a76c7a7aa42 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Fri, 3 May 2024 09:26:54 +0200 Subject: [PATCH 328/379] Update test --- test/pressure_solvers.jl | 38 ++++++++++++++------------------------ 1 file changed, 14 insertions(+), 24 deletions(-) diff --git a/test/pressure_solvers.jl b/test/pressure_solvers.jl index 3bb7caa2c..8fb836497 100644 --- a/test/pressure_solvers.jl +++ b/test/pressure_solvers.jl @@ -12,6 +12,7 @@ direct = DirectPressureSolver(setup) cg = CGPressureSolver(setup) spectral = SpectralPressureSolver(setup) + lmspectral = LowerMemorySpectralPressureSolver(setup) initial_pressure(x, y) = 1 / 4 * (cos(2x) + cos(2y)) p_exact = @@ -21,41 +22,30 @@ IncompressibleNavierStokes.apply_bc_p!(p_exact, 0.0, setup) lap = IncompressibleNavierStokes.laplacian(p_exact, setup) - p_direct = IncompressibleNavierStokes.apply_bc_p!( - IncompressibleNavierStokes.poisson(direct, lap), - 0.0, - setup, - ) - p_cg = IncompressibleNavierStokes.apply_bc_p!( - IncompressibleNavierStokes.poisson(cg, lap), - 0.0, - setup, - ) - p_spectral = IncompressibleNavierStokes.apply_bc_p!( - IncompressibleNavierStokes.poisson(spectral, lap), + get_p(psolver) = IncompressibleNavierStokes.apply_bc_p( + IncompressibleNavierStokes.poisson(psolver, lap), 0.0, setup, ) + p_direct = get_p(direct) + p_cg = get_p(cg) + p_spectral = get_p(spectral) + p_lmspectral = get_p(lmspectral) # Test that in-place and out-of-place versions give same result - @test p_direct ≈ IncompressibleNavierStokes.apply_bc_p!( - IncompressibleNavierStokes.poisson!(direct, zero(p_exact), lap), - 0.0, - setup, - ) - @test p_cg ≈ IncompressibleNavierStokes.apply_bc_p!( - IncompressibleNavierStokes.poisson!(cg, zero(p_exact), lap), - 0.0, - setup, - ) - @test p_spectral ≈ IncompressibleNavierStokes.apply_bc_p!( - IncompressibleNavierStokes.poisson!(spectral, zero(p_exact), lap), + get_p_inplace(psolver) = IncompressibleNavierStokes.apply_bc_p!( + IncompressibleNavierStokes.poisson!(psolver, zero(p_exact), lap), 0.0, setup, ) + @test p_direct ≈ get_p_inplace(direct) + @test p_cg ≈ get_p_inplace(cg) + @test p_spectral ≈ get_p_inplace(spectral) + @test p_lmspectral ≈ get_p_inplace(lmspectral) # Test that solvers compute the exact pressure @test p_direct ≈ p_exact @test p_cg ≈ p_exact @test p_spectral ≈ p_exact + @test p_lmspectral ≈ p_exact end From 0cfc799abba63ef2d90cbcccf6476d2312e5866e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Fri, 3 May 2024 09:35:06 +0200 Subject: [PATCH 329/379] Move functions --- src/operators.jl | 957 ++++++++++++++++++++++++----------------------- 1 file changed, 479 insertions(+), 478 deletions(-) diff --git a/src/operators.jl b/src/operators.jl index 99a8e7d20..a3b165a2a 100644 --- a/src/operators.jl +++ b/src/operators.jl @@ -105,259 +105,485 @@ ChainRulesCore.rrule(::typeof(divergence), u, setup) = ( ) """ - vorticity(u, setup) - -Compute vorticity field. -""" -vorticity(u, setup) = vorticity!( - length(u) == 2 ? similar(u[1], setup.grid.N) : - ntuple(α -> similar(u[1], setup.grid.N), length(u)), - u, - setup, -) - -""" - vorticity!(ω, u, setup) + pressuregradient!(G, p, setup) -Compute vorticity field. +Compute pressure gradient (in-place). """ -vorticity!(ω, u, setup) = vorticity!(setup.grid.dimension, ω, u, setup) - -function vorticity!(::Dimension{2}, ω, u, setup) - (; grid, workgroupsize) = setup - (; dimension, Δu, N) = grid - D = dimension() - δ = Offset{D}() - @kernel function ω!(ω, u, I0) - I = @index(Global, Cartesian) - I = I + I0 - ω[I] = - (u[2][I+δ(1)] - u[2][I]) / Δu[1][I[1]] - (u[1][I+δ(2)] - u[1][I]) / Δu[2][I[2]] - end - I0 = CartesianIndex(ntuple(Returns(1), D)) - I0 -= oneunit(I0) - ω!(get_backend(ω), workgroupsize)(ω, u, I0; ndrange = N .- 1) - ω -end - -function vorticity!(::Dimension{3}, ω, u, setup) +function pressuregradient!(G, p, setup) (; grid, workgroupsize) = setup - (; dimension, Δu, N) = grid + (; dimension, Δu, Nu, Iu) = grid D = dimension() δ = Offset{D}() - @kernel function ω!(ω, u, I0) - T = eltype(ω) + @kernel function G!(G, p, ::Val{α}, I0) where {α} I = @index(Global, Cartesian) - I = I + I0 - for (α, α₊, α₋) in ((1, 2, 3), (2, 3, 1), (3, 1, 2)) - # α₊ = mod1(α + 1, D) - # α₋ = mod1(α - 1, D) - ω[α][I] = - (u[α₋][I+δ(α₊)] - u[α₋][I]) / Δu[α₊][I[α₊]] - - (u[α₊][I+δ(α₋)] - u[α₊][I]) / Δu[α₋][I[α₋]] - end + I = I0 + I + G[α][I] = (p[I+δ(α)] - p[I]) / Δu[α][I[α]] end - I0 = CartesianIndex(ntuple(Returns(1), D)) - I0 -= oneunit(I0) - ω!(get_backend(ω[1]), workgroupsize)(ω, u, I0; ndrange = N .- 1) - ω -end - -""" - convection!(F, u, setup) - -Compute convective term. -""" -function convection!(F, u, setup) - (; grid, workgroupsize) = setup - (; dimension, Δ, Δu, Nu, Iu, A) = grid D = dimension() - δ = Offset{D}() - @kernel function conv!(F, u, ::Val{α}, ::Val{βrange}, I0) where {α,βrange} - I = @index(Global, Cartesian) - I = I + I0 - # for β = 1:D - KernelAbstractions.Extras.LoopInfo.@unroll for β in βrange - Δuαβ = α == β ? Δu[β] : Δ[β] - - # Half for u[α], (reverse!) interpolation for u[β] - # Note: - # In matrix version, uses - # 1*u[α][I-δ(β)] + 0*u[α][I] - # instead of 1/2 when u[α][I-δ(β)] is at Dirichlet boundary. - uαβ1 = (u[α][I-δ(β)] + u[α][I]) / 2 - uαβ2 = (u[α][I] + u[α][I+δ(β)]) / 2 - uβα1 = - A[β][α][2][I[α]-(α==β)] * u[β][I-δ(β)] + - A[β][α][1][I[α]+(α!=β)] * u[β][I-δ(β)+δ(α)] - uβα2 = A[β][α][2][I[α]] * u[β][I] + A[β][α][1][I[α]+1] * u[β][I+δ(α)] - - # # Half - # uαβ1 = (u[α][I-δ(β)] + u[α][I]) / 2 - # uβα1 = u[β][I-δ(β)] / 2 + u[β][I-δ(β)+δ(α)] / 2 - # uαβ2 = (u[α][I] + u[α][I+δ(β)]) / 2 - # uβα2 = u[β][I] / 2 + u[β][I+δ(α)] / 2 - - # # Interpolation - # uαβ1 = A[α][β][2][I[β]-1] * u[α][I-δ(β)] + A[α][β][1][I[β]] * u[α][I] - # uβα1 = - # A[β][α][2][I[α]-(α==β)] * u[β][I-δ(β)] + - # A[β][α][1][I[α]+(α!=β)] * u[β][I-δ(β)+δ(α)] - # uαβ2 = A[α][β][2][I[β]] * u[α][I] + A[α][β][1][I[β]+1] * u[α][I+δ(β)] - # uβα2 = A[β][α][2][I[α]] * u[β][I] + A[β][α][1][I[α]+1] * u[β][I+δ(α)] - - F[α][I] -= (uαβ2 * uβα2 - uαβ1 * uβα1) / Δuαβ[I[β]] - end - end for α = 1:D I0 = first(Iu[α]) I0 -= oneunit(I0) - conv!(get_backend(F[1]), workgroupsize)(F, u, Val(α), Val(1:D), I0; ndrange = Nu[α]) + G!(get_backend(G[1]), workgroupsize)(G, p, Val(α), I0; ndrange = Nu[α]) end - F + G end -function convection_adjoint!(ubar, φbar, u, setup) +function pressuregradient_adjoint!(pbar, φ, setup) (; grid, workgroupsize) = setup - (; dimension, Δ, Δu, N, Iu, A) = grid + (; dimension, Δu, N, Iu) = grid D = dimension() δ = Offset{D}() - T = eltype(u[1]) - h = T(1) / 2 - @kernel function adj!(ubar, φbar, u, ::Val{γ}, ::Val{looprange}) where {γ,looprange} - J = @index(Global, Cartesian) - KernelAbstractions.Extras.LoopInfo.@unroll for α in looprange - KernelAbstractions.Extras.LoopInfo.@unroll for β in looprange - Δuαβ = α == β ? Δu[β] : Δ[β] - Aβα1 = A[β][α][1] - Aβα2 = A[β][α][2] - - # 1 - I = J - if α == γ && I in Iu[α] - uαβ2 = h - uβα2 = Aβα2[I[α]] * u[β][I] + Aβα1[I[α]+1] * u[β][I+δ(α)] - dφdu = -uαβ2 * uβα2 / Δuαβ[I[β]] - ubar[γ][J] += φbar[α][I] * dφdu - end - - # 2 - I = J - δ(β) - if α == γ && I in Iu[α] - uαβ2 = h - uβα2 = Aβα2[I[α]] * u[β][I] + Aβα1[I[α]+1] * u[β][I+δ(α)] - dφdu = -uαβ2 * uβα2 / Δuαβ[I[β]] - ubar[γ][J] += φbar[α][I] * dφdu - end - - # 3 - I = J - if β == γ && I in Iu[α] - uαβ2 = h * u[α][I] + h * u[α][I+δ(β)] - uβα2 = Aβα2[I[α]] - dφdu = -uαβ2 * uβα2 / Δuαβ[I[β]] - ubar[γ][J] += φbar[α][I] * dφdu - end - - # 4 - I = J - δ(α) - if β == γ && I in Iu[α] - uαβ2 = h * u[α][I] + h * u[α][I+δ(β)] - uβα2 = Aβα1[I[α]+1] - dφdu = -uαβ2 * uβα2 / Δuαβ[I[β]] - ubar[γ][J] += φbar[α][I] * dφdu - end - - # 5 - I = J + δ(β) - if α == γ && I in Iu[α] - uαβ1 = h - uβα1 = - Aβα2[I[α]-(α==β)] * u[β][I-δ(β)] + - Aβα1[I[α]+(α!=β)] * u[β][I-δ(β)+δ(α)] - dφdu = uαβ1 * uβα1 / Δuαβ[I[β]] - ubar[γ][J] += φbar[α][I] * dφdu - end - - # 6 - I = J - if α == γ && I in Iu[α] - uαβ1 = h - uβα1 = - Aβα2[I[α]-(α==β)] * u[β][I-δ(β)] + - Aβα1[I[α]+(α!=β)] * u[β][I-δ(β)+δ(α)] - dφdu = uαβ1 * uβα1 / Δuαβ[I[β]] - ubar[γ][J] += φbar[α][I] * dφdu - end - - # 7 - I = J + δ(β) - if β == γ && I in Iu[α] - uαβ1 = h * u[α][I-δ(β)] + h * u[α][I] - uβα1 = Aβα2[I[α]-(α==β)] - dφdu = uαβ1 * uβα1 / Δuαβ[I[β]] - ubar[γ][J] += φbar[α][I] * dφdu - end - - # 8 - I = J + δ(β) - δ(α) - if β == γ && I in Iu[α] - uαβ1 = h * u[α][I-δ(β)] + h * u[α][I] - uβα1 = Aβα1[I[α]+(α!=β)] - dφdu = uαβ1 * uβα1 / Δuαβ[I[β]] - ubar[γ][J] += φbar[α][I] * dφdu - end - end + @kernel function adj!(p, φ) + I = @index(Global, Cartesian) + p[I] = zero(eltype(p)) + for α = 1:D + I - δ(α) ∈ Iu[α] && (p[I] += φ[α][I-δ(α)] / Δu[α][I[α]-1]) + I ∈ Iu[α] && (p[I] -= φ[α][I] / Δu[α][I[α]]) end end - for γ = 1:D - adj!(get_backend(u[1]), workgroupsize)(ubar, φbar, u, Val(γ), Val(1:D); ndrange = N) - end - ubar + adj!(get_backend(pbar), workgroupsize)(pbar, φ; ndrange = N) + pbar end -convection(u, setup) = convection!(zero.(u), u, setup) +""" + pressuregradient(p, setup) -ChainRulesCore.rrule(::typeof(convection), u, setup) = ( - convection(u, setup), - φ -> (NoTangent(), convection_adjoint!(zero.(u), (φ...,), u, setup), NoTangent()), +Compute pressure gradient. +""" +pressuregradient(p, setup) = + pressuregradient!(ntuple(α -> zero(p), setup.grid.dimension()), p, setup) + +ChainRulesCore.rrule(::typeof(pressuregradient), p, setup) = ( + pressuregradient(p, setup), + φ -> (NoTangent(), pressuregradient_adjoint!(similar(p), (φ...,), setup), NoTangent()), ) """ - diffusion!(F, u, setup) + applypressure!(u, p, setup) -Compute diffusive term. +Subtract pressure gradient (in-place). """ -function diffusion!(F, u, setup) - (; grid, workgroupsize, Re) = setup - (; dimension, Δ, Δu, Nu, Iu) = grid +function applypressure!(u, p, setup) + (; grid, workgroupsize) = setup + (; dimension, Δu, Nu, Iu) = grid D = dimension() δ = Offset{D}() - ν = 1 / Re - @kernel function diff!(F, u, ::Val{α}, ::Val{βrange}, I0) where {α,βrange} + @kernel function apply!(u, p, ::Val{α}, I0) where {α} I = @index(Global, Cartesian) - I = I + I0 - # for β = 1:D - KernelAbstractions.Extras.LoopInfo.@unroll for β in βrange - Δuαβ = (α == β ? Δu[β] : Δ[β]) - F[α][I] += - ν * ( - (u[α][I+δ(β)] - u[α][I]) / (β == α ? Δ[β][I[β]+1] : Δu[β][I[β]]) - - (u[α][I] - u[α][I-δ(β)]) / (β == α ? Δ[β][I[β]] : Δu[β][I[β]-1]) - ) / Δuαβ[I[β]] - end + I = I0 + I + u[α][I] -= (p[I+δ(α)] - p[I]) / Δu[α][I[α]] end + D = dimension() for α = 1:D I0 = first(Iu[α]) I0 -= oneunit(I0) - diff!(get_backend(F[1]), workgroupsize)(F, u, Val(α), Val(1:D), I0; ndrange = Nu[α]) + apply!(get_backend(u[1]), workgroupsize)(u, p, Val(α), I0; ndrange = Nu[α]) end - F + u end -function diffusion_adjoint!(u, φ, setup) - (; grid, workgroupsize, Re) = setup - (; dimension, N, Δ, Δu, Iu) = grid +# function applypressure_adjoint!(pbar, φ, u, setup) +# (; grid, workgroupsize) = setup +# (; dimension, Δu, N, Iu) = grid +# D = dimension() +# δ = Offset{D}() +# @kernel function adj!(p, φ) +# I = @index(Global, Cartesian) +# p[I] = zero(eltype(p)) +# for α = 1:D +# I - δ(α) ∈ Iu[α] && (p[I] += φ[α][I-δ(α)] / Δu[α][I[α]-1]) +# I ∈ Iu[α] && (p[I] -= φ[α][I] / Δu[α][I[α]]) +# end +# end +# adj!(get_backend(pbar), workgroupsize)(pbar, φ; ndrange = N) +# pbar +# end +# +# """ +# applypressure(p, setup) +# +# Compute pressure gradient. +# """ +# applypressure(u, p, setup) = +# applypressure!(copy.(u), p, setup) +# +# ChainRulesCore.rrule(::typeof(applypressure), p, setup) = ( +# applypressure(u, p, setup), +# φ -> (NoTangent(), applypressure_adjoint!(similar(p), (φ...,), setup), NoTangent()), +# ) + +""" + laplacian!(L, p, setup) + +Compute Laplacian of pressure field (in-place version). +""" +function laplacian!(L, p, setup) + (; grid, workgroupsize, boundary_conditions) = setup + (; dimension, Δ, Δu, N, Np, Ip, Ω) = grid + D = dimension() + δ = Offset{D}() + # @kernel function lap!(L, p, I0) + # I = @index(Global, Cartesian) + # I = I + I0 + # lap = zero(eltype(p)) + # for α = 1:D + # # bc = boundary_conditions[α] + # if bc[1] isa PressureBC && I[α] == I0[α] + 1 + # lap += + # Ω[I] / Δ[α][I[α]] * + # ((p[I+δ(α)] - p[I]) / Δu[α][I[α]] - (p[I]) / Δu[α][I[α]-1]) + # elseif bc[2] isa PressureBC && I[α] == I0[α] + Np[α] + # lap += + # Ω[I] / Δ[α][I[α]] * + # ((-p[I]) / Δu[α][I[α]] - (p[I] - p[I-δ(α)]) / Δu[α][I[α]-1]) + # elseif bc[1] isa DirichletBC && I[α] == I0[α] + 1 + # lap += Ω[I] / Δ[α][I[α]] * ((p[I+δ(α)] - p[I]) / Δu[α][I[α]]) + # elseif bc[2] isa DirichletBC && I[α] == I0[α] + Np[α] + # lap += Ω[I] / Δ[α][I[α]] * (-(p[I] - p[I-δ(α)]) / Δu[α][I[α]-1]) + # else + # lap += + # Ω[I] / Δ[α][I[α]] * + # ((p[I+δ(α)] - p[I]) / Δu[α][I[α]] - (p[I] - p[I-δ(α)]) / Δu[α][I[α]-1]) + # end + # end + # L[I] = lap + # end + @kernel function lapα!(L, p, I0, ::Val{α}, bc) where {α} + I = @index(Global, Cartesian) + I = I + I0 + # bc = boundary_conditions[α] + if bc[1] isa PressureBC && I[α] == I0[α] + 1 + L[I] += + Ω[I] / Δ[α][I[α]] * + ((p[I+δ(α)] - p[I]) / Δu[α][I[α]] - (p[I]) / Δu[α][I[α]-1]) + elseif bc[2] isa PressureBC && I[α] == I0[α] + Np[α] + L[I] += + Ω[I] / Δ[α][I[α]] * + ((-p[I]) / Δu[α][I[α]] - (p[I] - p[I-δ(α)]) / Δu[α][I[α]-1]) + elseif bc[1] isa DirichletBC && I[α] == I0[α] + 1 + L[I] += Ω[I] / Δ[α][I[α]] * ((p[I+δ(α)] - p[I]) / Δu[α][I[α]]) + elseif bc[2] isa DirichletBC && I[α] == I0[α] + Np[α] + L[I] += Ω[I] / Δ[α][I[α]] * (-(p[I] - p[I-δ(α)]) / Δu[α][I[α]-1]) + else + L[I] += + Ω[I] / Δ[α][I[α]] * + ((p[I+δ(α)] - p[I]) / Δu[α][I[α]] - (p[I] - p[I-δ(α)]) / Δu[α][I[α]-1]) + end + # L[I] = lap + end + # All volumes have a right velocity + # All volumes have a left velocity except the first one + # Start at second volume + ndrange = Np + I0 = first(Ip) + I0 -= oneunit(I0) + # lap!(get_backend(L), workgroupsize)(L, p, I0; ndrange) + L .= 0 + for α = 1:D + lapα!(get_backend(L), workgroupsize)( + L, + p, + I0, + Val(α), + boundary_conditions[α]; + ndrange, + ) + end + L +end + +""" + laplacian(p, setup) + +Compute Laplacian of pressure field. +""" +laplacian(p, setup) = laplacian!(similar(p), p, setup) + +function laplacian_mat(setup) + (; grid, boundary_conditions) = setup + (; dimension, x, N, Np, Ip, Δ, Δu, Ω) = grid + backend = get_backend(x[1]) + T = eltype(x[1]) + D = dimension() + δ = Offset{D}() + Ia = first(Ip) + Ib = last(Ip) + I = similar(x[1], CartesianIndex{D}, 0) + J = similar(x[1], CartesianIndex{D}, 0) + val = similar(x[1], 0) + I0 = Ia - oneunit(Ia) + for α = 1:D + a, b = boundary_conditions[α] + i = Ip[ntuple(β -> α == β ? (2:Np[α]-1) : (:), D)...][:] + ia = Ip[ntuple(β -> α == β ? (1:1) : (:), D)...][:] + ib = Ip[ntuple(β -> α == β ? (Np[α]:Np[α]) : (:), D)...][:] + for (aa, bb, j) in [(a, nothing, ia), (nothing, nothing, i), (nothing, b, ib)] + vala = @.(Ω[j] / Δ[α][getindex.(j, α)] / Δu[α][getindex.(j, α)-1]) + if isnothing(aa) + J = [J; j .- [δ(α)]; j] + I = [I; j; j] + val = [val; vala; -vala] + elseif aa isa PressureBC + J = [J; j] + I = [I; j] + val = [val; -vala] + elseif aa isa PeriodicBC + J = [J; ib; j] + I = [I; j; j] + val = [val; vala; -vala] + elseif aa isa SymmetricBC + J = [J; ia; j] + I = [I; j; j] + val = [val; vala; -vala] + elseif aa isa DirichletBC + end + + valb = @.(Ω[j] / Δ[α][getindex.(j, α)] / Δu[α][getindex.(j, α)]) + if isnothing(bb) + J = [J; j; j .+ [δ(α)]] + I = [I; j; j] + val = [val; -valb; valb] + elseif bb isa PressureBC + # The weight of the "right" BC is zero, but still needs a J inside Ip, so + # just set it to ib + J = [J; j] + I = [I; j] + val = [val; -valb] + elseif bb isa PeriodicBC + J = [J; j; ia] + I = [I; j; j] + val = [val; -valb; valb] + elseif bb isa SymmetricBC + J = [J; j; ib] + I = [I; j; j] + val = [val; -valb; valb] + elseif bb isa DirichletBC + end + # val = vcat( + # val, + # map(I -> Ω[I] / Δ[α][I[α]] / Δu[α][I[α]-1], j), + # map(I -> -Ω[I] / Δ[α][I[α]] * (1 / Δu[α][I[α]] + 1 / Δu[α][I[α]-1]), j), + # map(I -> Ω[I] / Δ[α][I[α]] / Δu[α][I[α]], j), + end + end + # Go back to CPU, otherwise get following error: + # ERROR: CUDA error: an illegal memory access was encountered (code 700, ERROR_ILLEGAL_ADDRESS) + I = Array(I) + J = Array(J) + # I = I .- I0 + # J = J .- I0 + I = I .- [I0] + J = J .- [I0] + # linear = copyto!(similar(x[1], Int, Np), collect(LinearIndices(Ip))) + linear = LinearIndices(Ip) + I = linear[I] + J = linear[J] + + # Assemble on CPU, since CUDA overwrites instead of adding + L = sparse(I, J, Array(val)) + # II = copyto!(similar(x[1], Int, length(I)), I) + # JJ = copyto!(similar(x[1], Int, length(J)), J) + # sparse(II, JJ, val) + + L + # Ω isa CuArray ? cu(L) : L +end + +""" + convection!(F, u, setup) + +Compute convective term. +""" +function convection!(F, u, setup) + (; grid, workgroupsize) = setup + (; dimension, Δ, Δu, Nu, Iu, A) = grid + D = dimension() + δ = Offset{D}() + @kernel function conv!(F, u, ::Val{α}, ::Val{βrange}, I0) where {α,βrange} + I = @index(Global, Cartesian) + I = I + I0 + # for β = 1:D + KernelAbstractions.Extras.LoopInfo.@unroll for β in βrange + Δuαβ = α == β ? Δu[β] : Δ[β] + + # Half for u[α], (reverse!) interpolation for u[β] + # Note: + # In matrix version, uses + # 1*u[α][I-δ(β)] + 0*u[α][I] + # instead of 1/2 when u[α][I-δ(β)] is at Dirichlet boundary. + uαβ1 = (u[α][I-δ(β)] + u[α][I]) / 2 + uαβ2 = (u[α][I] + u[α][I+δ(β)]) / 2 + uβα1 = + A[β][α][2][I[α]-(α==β)] * u[β][I-δ(β)] + + A[β][α][1][I[α]+(α!=β)] * u[β][I-δ(β)+δ(α)] + uβα2 = A[β][α][2][I[α]] * u[β][I] + A[β][α][1][I[α]+1] * u[β][I+δ(α)] + + # # Half + # uαβ1 = (u[α][I-δ(β)] + u[α][I]) / 2 + # uβα1 = u[β][I-δ(β)] / 2 + u[β][I-δ(β)+δ(α)] / 2 + # uαβ2 = (u[α][I] + u[α][I+δ(β)]) / 2 + # uβα2 = u[β][I] / 2 + u[β][I+δ(α)] / 2 + + # # Interpolation + # uαβ1 = A[α][β][2][I[β]-1] * u[α][I-δ(β)] + A[α][β][1][I[β]] * u[α][I] + # uβα1 = + # A[β][α][2][I[α]-(α==β)] * u[β][I-δ(β)] + + # A[β][α][1][I[α]+(α!=β)] * u[β][I-δ(β)+δ(α)] + # uαβ2 = A[α][β][2][I[β]] * u[α][I] + A[α][β][1][I[β]+1] * u[α][I+δ(β)] + # uβα2 = A[β][α][2][I[α]] * u[β][I] + A[β][α][1][I[α]+1] * u[β][I+δ(α)] + + F[α][I] -= (uαβ2 * uβα2 - uαβ1 * uβα1) / Δuαβ[I[β]] + end + end + for α = 1:D + I0 = first(Iu[α]) + I0 -= oneunit(I0) + conv!(get_backend(F[1]), workgroupsize)(F, u, Val(α), Val(1:D), I0; ndrange = Nu[α]) + end + F +end + +function convection_adjoint!(ubar, φbar, u, setup) + (; grid, workgroupsize) = setup + (; dimension, Δ, Δu, N, Iu, A) = grid + D = dimension() + δ = Offset{D}() + T = eltype(u[1]) + h = T(1) / 2 + @kernel function adj!(ubar, φbar, u, ::Val{γ}, ::Val{looprange}) where {γ,looprange} + J = @index(Global, Cartesian) + KernelAbstractions.Extras.LoopInfo.@unroll for α in looprange + KernelAbstractions.Extras.LoopInfo.@unroll for β in looprange + Δuαβ = α == β ? Δu[β] : Δ[β] + Aβα1 = A[β][α][1] + Aβα2 = A[β][α][2] + + # 1 + I = J + if α == γ && I in Iu[α] + uαβ2 = h + uβα2 = Aβα2[I[α]] * u[β][I] + Aβα1[I[α]+1] * u[β][I+δ(α)] + dφdu = -uαβ2 * uβα2 / Δuαβ[I[β]] + ubar[γ][J] += φbar[α][I] * dφdu + end + + # 2 + I = J - δ(β) + if α == γ && I in Iu[α] + uαβ2 = h + uβα2 = Aβα2[I[α]] * u[β][I] + Aβα1[I[α]+1] * u[β][I+δ(α)] + dφdu = -uαβ2 * uβα2 / Δuαβ[I[β]] + ubar[γ][J] += φbar[α][I] * dφdu + end + + # 3 + I = J + if β == γ && I in Iu[α] + uαβ2 = h * u[α][I] + h * u[α][I+δ(β)] + uβα2 = Aβα2[I[α]] + dφdu = -uαβ2 * uβα2 / Δuαβ[I[β]] + ubar[γ][J] += φbar[α][I] * dφdu + end + + # 4 + I = J - δ(α) + if β == γ && I in Iu[α] + uαβ2 = h * u[α][I] + h * u[α][I+δ(β)] + uβα2 = Aβα1[I[α]+1] + dφdu = -uαβ2 * uβα2 / Δuαβ[I[β]] + ubar[γ][J] += φbar[α][I] * dφdu + end + + # 5 + I = J + δ(β) + if α == γ && I in Iu[α] + uαβ1 = h + uβα1 = + Aβα2[I[α]-(α==β)] * u[β][I-δ(β)] + + Aβα1[I[α]+(α!=β)] * u[β][I-δ(β)+δ(α)] + dφdu = uαβ1 * uβα1 / Δuαβ[I[β]] + ubar[γ][J] += φbar[α][I] * dφdu + end + + # 6 + I = J + if α == γ && I in Iu[α] + uαβ1 = h + uβα1 = + Aβα2[I[α]-(α==β)] * u[β][I-δ(β)] + + Aβα1[I[α]+(α!=β)] * u[β][I-δ(β)+δ(α)] + dφdu = uαβ1 * uβα1 / Δuαβ[I[β]] + ubar[γ][J] += φbar[α][I] * dφdu + end + + # 7 + I = J + δ(β) + if β == γ && I in Iu[α] + uαβ1 = h * u[α][I-δ(β)] + h * u[α][I] + uβα1 = Aβα2[I[α]-(α==β)] + dφdu = uαβ1 * uβα1 / Δuαβ[I[β]] + ubar[γ][J] += φbar[α][I] * dφdu + end + + # 8 + I = J + δ(β) - δ(α) + if β == γ && I in Iu[α] + uαβ1 = h * u[α][I-δ(β)] + h * u[α][I] + uβα1 = Aβα1[I[α]+(α!=β)] + dφdu = uαβ1 * uβα1 / Δuαβ[I[β]] + ubar[γ][J] += φbar[α][I] * dφdu + end + end + end + end + for γ = 1:D + adj!(get_backend(u[1]), workgroupsize)(ubar, φbar, u, Val(γ), Val(1:D); ndrange = N) + end + ubar +end + +convection(u, setup) = convection!(zero.(u), u, setup) + +ChainRulesCore.rrule(::typeof(convection), u, setup) = ( + convection(u, setup), + φ -> (NoTangent(), convection_adjoint!(zero.(u), (φ...,), u, setup), NoTangent()), +) + +""" + diffusion!(F, u, setup) + +Compute diffusive term. +""" +function diffusion!(F, u, setup) + (; grid, workgroupsize, Re) = setup + (; dimension, Δ, Δu, Nu, Iu) = grid + D = dimension() + δ = Offset{D}() + ν = 1 / Re + @kernel function diff!(F, u, ::Val{α}, ::Val{βrange}, I0) where {α,βrange} + I = @index(Global, Cartesian) + I = I + I0 + # for β = 1:D + KernelAbstractions.Extras.LoopInfo.@unroll for β in βrange + Δuαβ = (α == β ? Δu[β] : Δ[β]) + F[α][I] += + ν * ( + (u[α][I+δ(β)] - u[α][I]) / (β == α ? Δ[β][I[β]+1] : Δu[β][I[β]]) - + (u[α][I] - u[α][I-δ(β)]) / (β == α ? Δ[β][I[β]] : Δu[β][I[β]-1]) + ) / Δuαβ[I[β]] + end + end + for α = 1:D + I0 = first(Iu[α]) + I0 -= oneunit(I0) + diff!(get_backend(F[1]), workgroupsize)(F, u, Val(α), Val(1:D), I0; ndrange = Nu[α]) + end + F +end + +function diffusion_adjoint!(u, φ, setup) + (; grid, workgroupsize, Re) = setup + (; dimension, N, Δ, Δu, Iu) = grid D = dimension() δ = Offset{D}() ν = 1 / Re @@ -536,293 +762,68 @@ end # momentum_pullback!(zero.(φ), φ, u, t, setup), # NoTangent(), # NoTangent(), -# ), -# ) - -""" - pressuregradient!(G, p, setup) - -Compute pressure gradient (in-place). -""" -function pressuregradient!(G, p, setup) - (; grid, workgroupsize) = setup - (; dimension, Δu, Nu, Iu) = grid - D = dimension() - δ = Offset{D}() - @kernel function G!(G, p, ::Val{α}, I0) where {α} - I = @index(Global, Cartesian) - I = I0 + I - G[α][I] = (p[I+δ(α)] - p[I]) / Δu[α][I[α]] - end - D = dimension() - for α = 1:D - I0 = first(Iu[α]) - I0 -= oneunit(I0) - G!(get_backend(G[1]), workgroupsize)(G, p, Val(α), I0; ndrange = Nu[α]) - end - G -end - -function pressuregradient_adjoint!(pbar, φ, setup) - (; grid, workgroupsize) = setup - (; dimension, Δu, N, Iu) = grid - D = dimension() - δ = Offset{D}() - @kernel function adj!(p, φ) - I = @index(Global, Cartesian) - p[I] = zero(eltype(p)) - for α = 1:D - I - δ(α) ∈ Iu[α] && (p[I] += φ[α][I-δ(α)] / Δu[α][I[α]-1]) - I ∈ Iu[α] && (p[I] -= φ[α][I] / Δu[α][I[α]]) - end - end - adj!(get_backend(pbar), workgroupsize)(pbar, φ; ndrange = N) - pbar -end +# ), +# ) """ - pressuregradient(p, setup) + vorticity(u, setup) -Compute pressure gradient. +Compute vorticity field. """ -pressuregradient(p, setup) = - pressuregradient!(ntuple(α -> zero(p), setup.grid.dimension()), p, setup) - -ChainRulesCore.rrule(::typeof(pressuregradient), p, setup) = ( - pressuregradient(p, setup), - φ -> (NoTangent(), pressuregradient_adjoint!(similar(p), (φ...,), setup), NoTangent()), +vorticity(u, setup) = vorticity!( + length(u) == 2 ? similar(u[1], setup.grid.N) : + ntuple(α -> similar(u[1], setup.grid.N), length(u)), + u, + setup, ) """ - applypressure!(u, p, setup) + vorticity!(ω, u, setup) -Subtract pressure gradient (in-place). +Compute vorticity field. """ -function applypressure!(u, p, setup) +vorticity!(ω, u, setup) = vorticity!(setup.grid.dimension, ω, u, setup) + +function vorticity!(::Dimension{2}, ω, u, setup) (; grid, workgroupsize) = setup - (; dimension, Δu, Nu, Iu) = grid + (; dimension, Δu, N) = grid D = dimension() δ = Offset{D}() - @kernel function apply!(u, p, ::Val{α}, I0) where {α} + @kernel function ω!(ω, u, I0) I = @index(Global, Cartesian) - I = I0 + I - u[α][I] -= (p[I+δ(α)] - p[I]) / Δu[α][I[α]] - end - D = dimension() - for α = 1:D - I0 = first(Iu[α]) - I0 -= oneunit(I0) - apply!(get_backend(u[1]), workgroupsize)(u, p, Val(α), I0; ndrange = Nu[α]) + I = I + I0 + ω[I] = + (u[2][I+δ(1)] - u[2][I]) / Δu[1][I[1]] - (u[1][I+δ(2)] - u[1][I]) / Δu[2][I[2]] end - u + I0 = CartesianIndex(ntuple(Returns(1), D)) + I0 -= oneunit(I0) + ω!(get_backend(ω), workgroupsize)(ω, u, I0; ndrange = N .- 1) + ω end -# function applypressure_adjoint!(pbar, φ, u, setup) -# (; grid, workgroupsize) = setup -# (; dimension, Δu, N, Iu) = grid -# D = dimension() -# δ = Offset{D}() -# @kernel function adj!(p, φ) -# I = @index(Global, Cartesian) -# p[I] = zero(eltype(p)) -# for α = 1:D -# I - δ(α) ∈ Iu[α] && (p[I] += φ[α][I-δ(α)] / Δu[α][I[α]-1]) -# I ∈ Iu[α] && (p[I] -= φ[α][I] / Δu[α][I[α]]) -# end -# end -# adj!(get_backend(pbar), workgroupsize)(pbar, φ; ndrange = N) -# pbar -# end -# -# """ -# applypressure(p, setup) -# -# Compute pressure gradient. -# """ -# applypressure(u, p, setup) = -# applypressure!(copy.(u), p, setup) -# -# ChainRulesCore.rrule(::typeof(applypressure), p, setup) = ( -# applypressure(u, p, setup), -# φ -> (NoTangent(), applypressure_adjoint!(similar(p), (φ...,), setup), NoTangent()), -# ) - -""" - laplacian!(L, p, setup) - -Compute Laplacian of pressure field (in-place version). -""" -function laplacian!(L, p, setup) - (; grid, workgroupsize, boundary_conditions) = setup - (; dimension, Δ, Δu, N, Np, Ip, Ω) = grid +function vorticity!(::Dimension{3}, ω, u, setup) + (; grid, workgroupsize) = setup + (; dimension, Δu, N) = grid D = dimension() δ = Offset{D}() - # @kernel function lap!(L, p, I0) - # I = @index(Global, Cartesian) - # I = I + I0 - # lap = zero(eltype(p)) - # for α = 1:D - # # bc = boundary_conditions[α] - # if bc[1] isa PressureBC && I[α] == I0[α] + 1 - # lap += - # Ω[I] / Δ[α][I[α]] * - # ((p[I+δ(α)] - p[I]) / Δu[α][I[α]] - (p[I]) / Δu[α][I[α]-1]) - # elseif bc[2] isa PressureBC && I[α] == I0[α] + Np[α] - # lap += - # Ω[I] / Δ[α][I[α]] * - # ((-p[I]) / Δu[α][I[α]] - (p[I] - p[I-δ(α)]) / Δu[α][I[α]-1]) - # elseif bc[1] isa DirichletBC && I[α] == I0[α] + 1 - # lap += Ω[I] / Δ[α][I[α]] * ((p[I+δ(α)] - p[I]) / Δu[α][I[α]]) - # elseif bc[2] isa DirichletBC && I[α] == I0[α] + Np[α] - # lap += Ω[I] / Δ[α][I[α]] * (-(p[I] - p[I-δ(α)]) / Δu[α][I[α]-1]) - # else - # lap += - # Ω[I] / Δ[α][I[α]] * - # ((p[I+δ(α)] - p[I]) / Δu[α][I[α]] - (p[I] - p[I-δ(α)]) / Δu[α][I[α]-1]) - # end - # end - # L[I] = lap - # end - @kernel function lapα!(L, p, I0, ::Val{α}, bc) where {α} + @kernel function ω!(ω, u, I0) + T = eltype(ω) I = @index(Global, Cartesian) I = I + I0 - # bc = boundary_conditions[α] - if bc[1] isa PressureBC && I[α] == I0[α] + 1 - L[I] += - Ω[I] / Δ[α][I[α]] * - ((p[I+δ(α)] - p[I]) / Δu[α][I[α]] - (p[I]) / Δu[α][I[α]-1]) - elseif bc[2] isa PressureBC && I[α] == I0[α] + Np[α] - L[I] += - Ω[I] / Δ[α][I[α]] * - ((-p[I]) / Δu[α][I[α]] - (p[I] - p[I-δ(α)]) / Δu[α][I[α]-1]) - elseif bc[1] isa DirichletBC && I[α] == I0[α] + 1 - L[I] += Ω[I] / Δ[α][I[α]] * ((p[I+δ(α)] - p[I]) / Δu[α][I[α]]) - elseif bc[2] isa DirichletBC && I[α] == I0[α] + Np[α] - L[I] += Ω[I] / Δ[α][I[α]] * (-(p[I] - p[I-δ(α)]) / Δu[α][I[α]-1]) - else - L[I] += - Ω[I] / Δ[α][I[α]] * - ((p[I+δ(α)] - p[I]) / Δu[α][I[α]] - (p[I] - p[I-δ(α)]) / Δu[α][I[α]-1]) + for (α, α₊, α₋) in ((1, 2, 3), (2, 3, 1), (3, 1, 2)) + # α₊ = mod1(α + 1, D) + # α₋ = mod1(α - 1, D) + ω[α][I] = + (u[α₋][I+δ(α₊)] - u[α₋][I]) / Δu[α₊][I[α₊]] - + (u[α₊][I+δ(α₋)] - u[α₊][I]) / Δu[α₋][I[α₋]] end - # L[I] = lap end - # All volumes have a right velocity - # All volumes have a left velocity except the first one - # Start at second volume - ndrange = Np - I0 = first(Ip) + I0 = CartesianIndex(ntuple(Returns(1), D)) I0 -= oneunit(I0) - # lap!(get_backend(L), workgroupsize)(L, p, I0; ndrange) - L .= 0 - for α = 1:D - lapα!(get_backend(L), workgroupsize)( - L, - p, - I0, - Val(α), - boundary_conditions[α]; - ndrange, - ) - end - L + ω!(get_backend(ω[1]), workgroupsize)(ω, u, I0; ndrange = N .- 1) + ω end -""" - laplacian(p, setup) - -Compute Laplacian of pressure field. -""" -laplacian(p, setup) = laplacian!(similar(p), p, setup) - -function laplacian_mat(setup) - (; grid, boundary_conditions) = setup - (; dimension, x, N, Np, Ip, Δ, Δu, Ω) = grid - backend = get_backend(x[1]) - T = eltype(x[1]) - D = dimension() - δ = Offset{D}() - Ia = first(Ip) - Ib = last(Ip) - I = similar(x[1], CartesianIndex{D}, 0) - J = similar(x[1], CartesianIndex{D}, 0) - val = similar(x[1], 0) - I0 = Ia - oneunit(Ia) - for α = 1:D - a, b = boundary_conditions[α] - i = Ip[ntuple(β -> α == β ? (2:Np[α]-1) : (:), D)...][:] - ia = Ip[ntuple(β -> α == β ? (1:1) : (:), D)...][:] - ib = Ip[ntuple(β -> α == β ? (Np[α]:Np[α]) : (:), D)...][:] - for (aa, bb, j) in [(a, nothing, ia), (nothing, nothing, i), (nothing, b, ib)] - vala = @.(Ω[j] / Δ[α][getindex.(j, α)] / Δu[α][getindex.(j, α)-1]) - if isnothing(aa) - J = [J; j .- [δ(α)]; j] - I = [I; j; j] - val = [val; vala; -vala] - elseif aa isa PressureBC - J = [J; j] - I = [I; j] - val = [val; -vala] - elseif aa isa PeriodicBC - J = [J; ib; j] - I = [I; j; j] - val = [val; vala; -vala] - elseif aa isa SymmetricBC - J = [J; ia; j] - I = [I; j; j] - val = [val; vala; -vala] - elseif aa isa DirichletBC - end - - valb = @.(Ω[j] / Δ[α][getindex.(j, α)] / Δu[α][getindex.(j, α)]) - if isnothing(bb) - J = [J; j; j .+ [δ(α)]] - I = [I; j; j] - val = [val; -valb; valb] - elseif bb isa PressureBC - # The weight of the "right" BC is zero, but still needs a J inside Ip, so - # just set it to ib - J = [J; j] - I = [I; j] - val = [val; -valb] - elseif bb isa PeriodicBC - J = [J; j; ia] - I = [I; j; j] - val = [val; -valb; valb] - elseif bb isa SymmetricBC - J = [J; j; ib] - I = [I; j; j] - val = [val; -valb; valb] - elseif bb isa DirichletBC - end - # val = vcat( - # val, - # map(I -> Ω[I] / Δ[α][I[α]] / Δu[α][I[α]-1], j), - # map(I -> -Ω[I] / Δ[α][I[α]] * (1 / Δu[α][I[α]] + 1 / Δu[α][I[α]-1]), j), - # map(I -> Ω[I] / Δ[α][I[α]] / Δu[α][I[α]], j), - end - end - # Go back to CPU, otherwise get following error: - # ERROR: CUDA error: an illegal memory access was encountered (code 700, ERROR_ILLEGAL_ADDRESS) - I = Array(I) - J = Array(J) - # I = I .- I0 - # J = J .- I0 - I = I .- [I0] - J = J .- [I0] - # linear = copyto!(similar(x[1], Int, Np), collect(LinearIndices(Ip))) - linear = LinearIndices(Ip) - I = linear[I] - J = linear[J] - - # Assemble on CPU, since CUDA overwrites instead of adding - L = sparse(I, J, Array(val)) - # II = copyto!(similar(x[1], Int, length(I)), I) - # JJ = copyto!(similar(x[1], Int, length(J)), J) - # sparse(II, JJ, val) - - L - # Ω isa CuArray ? cu(L) : L -end @inline ∂x(uα, I::CartesianIndex{D}, α, β, Δβ, Δuβ; δ = Offset{D}()) where {D} = α == β ? (uα[I] - uα[I-δ(β)]) / Δβ[I[β]] : From e19f8badfb616be881bca13993bae6636ff0e36a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Fri, 3 May 2024 09:41:24 +0200 Subject: [PATCH 330/379] Do not assemble unused fields --- src/grid/grid.jl | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/src/grid/grid.jl b/src/grid/grid.jl index d0a3ab442..13bd41597 100644 --- a/src/grid/grid.jl +++ b/src/grid/grid.jl @@ -81,24 +81,24 @@ function Grid(x, boundary_conditions; ArrayType = Array) Ω .*= reshape(Δ[d], ntuple(Returns(1), d - 1)..., :) end - # Velocity volume sizes - Ωu = ntuple(α -> ones(T, N), D) - for α = 1:D, β = 1:D - Ωu[α] .*= reshape((α == β ? Δu : Δ)[β], ntuple(Returns(1), β - 1)..., :) - end - - # Vorticity volume sizes - Ωω = ones(T, N) - for α = 1:D - Ωω .*= reshape(Δu[α], ntuple(Returns(1), α - 1)..., :) - end - - # Velocity volume mid-sections - Γu = ntuple(α -> ntuple(β -> ones(T, N), D), D) - for α = 1:D, β = 1:D, γ in ((1:β-1)..., (β+1:D)...) - Γu[α][β] .*= - reshape(γ == β ? 1 : γ == α ? Δu[γ] : Δ[γ], ntuple(Returns(1), γ - 1)..., :) - end + # # Velocity volume sizes + # Ωu = ntuple(α -> ones(T, N), D) + # for α = 1:D, β = 1:D + # Ωu[α] .*= reshape((α == β ? Δu : Δ)[β], ntuple(Returns(1), β - 1)..., :) + # end + + # # Vorticity volume sizes + # Ωω = ones(T, N) + # for α = 1:D + # Ωω .*= reshape(Δu[α], ntuple(Returns(1), α - 1)..., :) + # end + + # # Velocity volume mid-sections + # Γu = ntuple(α -> ntuple(β -> ones(T, N), D), D) + # for α = 1:D, β = 1:D, γ in ((1:β-1)..., (β+1:D)...) + # Γu[α][β] .*= + # reshape(γ == β ? 1 : γ == α ? Δu[γ] : Δ[γ], ntuple(Returns(1), γ - 1)..., :) + # end # # Velocity points # Xu = ntuple(α -> ones(T, N)) From d5ac8fac5be161d56ac99e63c428f4821e9d241a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Fri, 3 May 2024 10:06:15 +0200 Subject: [PATCH 331/379] Add operator tests --- test/operators.jl | 62 +++++++++++++++++++++++++++++++++++++++++++++++ test/runtests.jl | 1 + 2 files changed, 63 insertions(+) create mode 100644 test/operators.jl diff --git a/test/operators.jl b/test/operators.jl new file mode 100644 index 000000000..8b87964de --- /dev/null +++ b/test/operators.jl @@ -0,0 +1,62 @@ +@testset "Operators" begin + # Setup + T = Float64 + Re = T(1_000) + n = 16 + lims = T(0), T(1) + x = stretched_grid(lims..., n, 1.2), cosine_grid(lims..., n) + setup = Setup(x...; Re) + psolver = DirectPressureSolver(setup) + u = random_field(setup, T(0); psolver) + (; Iu, Ip) = setup.grid + + @testset "Divergence" begin + div = divergence(u, setup) + @test div isa Array{T} + @test all(!isnan, div) + end + + @testset "Pressure gradient" begin + v = randn!.(similar.(u)) + p = randn!(similar(u[1])) + apply_bc_u!(v, T(0), setup) + apply_bc_p!(p, T(0), setup) + Dv = divergence(v, setup) + Gp = pressuregradient(p, setup) + pDv = if length(v) == 2 + sum((p .* Ω .* Dv)[Ip]) + end + vGp = if length(v) == 2 + vGpx = v[1] .* setup.grid.Δu[1] .* setup.grid.Δ[2]' .* Gp[1] + vGpx = v[2] .* setup.grid.Δ[1] .* setup.grid.Δu[2]' .* Gp[2] + sum(vGpx[Iu[1]]) + sum(vGpx[Iu[2]]) + end + @test G isa Tuple + @test G[1] isa Array{T} + @test pDv ≈ -vGp # Check that D = -G' + end + + @testset "Convection" begin + c = convection(u, setup) + uCu = if length(u) == 2 + uCux = u[1] .* setup.grid.Δu[1] .* setup.grid.Δ[2]' .* c[1] + uCuy = u[2] .* setup.grid.Δ[1] .* setup.grid.Δu[2]' .* c[2] + sum(uCux[Iu[1]]) + sum(uCuy[Iu[2]]) + end + @test c isa Tuple + @test c[1] isa Array{T} + @test uCu ≈ 0 atol = 1e-12 # Check skew-symmetry + end + + @testset "Diffusion" begin + d = diffusion(u, setup) + uDu = if length(u) == 2 + uDux = u[1] .* setup.grid.Δu[1] .* setup.grid.Δ[2]' .* d[1] + uDuy = u[2] .* setup.grid.Δ[1] .* setup.grid.Δu[2]' .* d[2] + sum(uDux[Iu[1]]) + sum(uDuy[Iu[2]]) + end + @test d isa Tuple + @test d[1] isa Array{T} + @test uDu ≥ 0 # Check positivity + end +end diff --git a/test/runtests.jl b/test/runtests.jl index f878eb785..21ba6f6fc 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -8,6 +8,7 @@ using Test @testset "IncompressibleNavierStokes" begin include("grid.jl") include("pressure_solvers.jl") + include("operators.jl") # include("models.jl") # include("solvers.jl") # include("simulation2D.jl") From dce6ea770bab29ec04d781eb6694ea7ef8caf63c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Fri, 3 May 2024 10:10:36 +0200 Subject: [PATCH 332/379] Add rule test outline --- test/Project.toml | 1 + test/chainrules.jl | 2 ++ test/runtests.jl | 2 ++ 3 files changed, 5 insertions(+) create mode 100644 test/chainrules.jl diff --git a/test/Project.toml b/test/Project.toml index 4533703fc..7171c3a49 100644 --- a/test/Project.toml +++ b/test/Project.toml @@ -1,6 +1,7 @@ [deps] Aqua = "4c88cf16-eb10-579e-8560-4a9242c79595" CairoMakie = "13f3f980-e62b-5c42-98c6-ff1f3baf88f0" +ChainRulesTestUtils = "cdddcdb0-9152-4a09-a978-84456f9df70a" LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2" Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" diff --git a/test/chainrules.jl b/test/chainrules.jl new file mode 100644 index 000000000..bf84661d6 --- /dev/null +++ b/test/chainrules.jl @@ -0,0 +1,2 @@ +@testset "Chain rules" begin +end diff --git a/test/runtests.jl b/test/runtests.jl index 21ba6f6fc..14220eff4 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -1,5 +1,6 @@ using Aqua using CairoMakie +using ChainRulesTestUtils using IncompressibleNavierStokes using LinearAlgebra using Statistics @@ -9,6 +10,7 @@ using Test include("grid.jl") include("pressure_solvers.jl") include("operators.jl") + include("chainrules.jl") # include("models.jl") # include("solvers.jl") # include("simulation2D.jl") From f3c64e4f146d6a2c1144b2be142fbef1c0ef55e3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Fri, 3 May 2024 10:14:25 +0200 Subject: [PATCH 333/379] Remove unused code --- src/IncompressibleNavierStokes.jl | 6 ----- src/models/viscosity_models.jl | 40 ------------------------------- 2 files changed, 46 deletions(-) delete mode 100644 src/models/viscosity_models.jl diff --git a/src/IncompressibleNavierStokes.jl b/src/IncompressibleNavierStokes.jl index cdb176749..a4589ee0b 100644 --- a/src/IncompressibleNavierStokes.jl +++ b/src/IncompressibleNavierStokes.jl @@ -39,9 +39,6 @@ include("grid/stretched_grid.jl") include("grid/cosine_grid.jl") include("grid/max_size.jl") -# Models -include("models/viscosity_models.jl") - # Setup include("setup.jl") @@ -87,9 +84,6 @@ include("utils/spectral_stuff.jl") # Boundary conditions export PeriodicBC, DirichletBC, SymmetricBC, PressureBC -# Models -export LaminarModel, MixingLengthModel, SmagorinskyModel, QRModel - # Processors export processor, timelogger, vtk_writer, fieldsaver, realtimeplotter export fieldplot, energy_history_plot, energy_spectrum_plot diff --git a/src/models/viscosity_models.jl b/src/models/viscosity_models.jl deleted file mode 100644 index fbe67fa18..000000000 --- a/src/models/viscosity_models.jl +++ /dev/null @@ -1,40 +0,0 @@ -""" - AbstractViscosityModel - -Abstract viscosity model. -""" -abstract type AbstractViscosityModel end - -""" - LaminarModel() - -Laminar model. This model assumes that there are no -sub-grid stresses. It can be used if the grid is sufficiently refined for the -given flow. It has the advantage of having a constant diffusion operator. -""" -struct LaminarModel <: AbstractViscosityModel end - -""" - MixingLengthModel() - -Mixing-length model with mixing length `lm`. -""" -@kwdef struct MixingLengthModel{T} <: AbstractViscosityModel - lm::T = 1 # Mixing length -end - -""" - SmagorinskyModel(C_s = 0.17) - -Smagorinsky-Lilly model with constant `C_s`. -""" -@kwdef struct SmagorinskyModel{T} <: AbstractViscosityModel - C_s::T = 0.17 # Smagorinsky constant -end - -""" - QR(Re) - -QR-model. -""" -struct QRModel{T} <: AbstractViscosityModel end From 246c91d33da8972cfa7e2ea7752fdc9c7d7de79c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Fri, 3 May 2024 10:27:48 +0200 Subject: [PATCH 334/379] Put RK methods in module --- README.md | 2 +- docs/src/api/tableaux.md | 103 +++++++++++++++-------------- examples/Actuator2D.jl | 2 +- examples/Actuator3D.jl | 2 +- examples/MultiActuator.jl | 2 +- examples/PlanarMixing2D.jl | 2 +- examples/PlaneJets2D.jl | 2 +- libs/NeuralClosure/src/training.jl | 8 +-- libs/PaperDC/postanalysis.jl | 1 + libs/PaperDC/src/rk.jl | 2 +- src/IncompressibleNavierStokes.jl | 30 ++------- src/solvers/solve_unsteady.jl | 4 +- src/time_steppers/methods.jl | 8 +-- src/time_steppers/tableaux.jl | 29 ++++++++ 14 files changed, 104 insertions(+), 93 deletions(-) diff --git a/README.md b/README.md index 58b71e1dd..b41a3bb3b 100644 --- a/README.md +++ b/README.md @@ -101,7 +101,7 @@ solve_unsteady( setup, u₀, (0.0, 12.0); Δt = 0.05, processors = ( - anim = animator(; setup, path ="vorticity.mp4", nupdate = 4), + anim = animator(; setup, path = "vorticity.mp4", nupdate = 4), log = timelogger(), ), ) diff --git a/docs/src/api/tableaux.md b/docs/src/api/tableaux.md index f55b15640..9c0c29b9f 100644 --- a/docs/src/api/tableaux.md +++ b/docs/src/api/tableaux.md @@ -4,93 +4,96 @@ CurrentModule = IncompressibleNavierStokes # API Reference -- Runge-Kutta methods +```@docs +RKMethods +``` ## Explicit Methods ```@docs -FE11 -SSP22 -SSP42 -SSP33 -SSP43 -SSP104 -rSSPs2 -rSSPs3 -Wray3 -RK56 -DOPRI6 +RKMethods.FE11 +RKMethods.SSP22 +RKMethods.SSP42 +RKMethods.SSP33 +RKMethods.SSP43 +RKMethods.SSP104 +RKMethods.rSSPs2 +RKMethods.rSSPs3 +RKMethods.Wray3 +RKMethods.RK56 +RKMethods.DOPRI6 ``` ## Implicit Methods ```@docs -BE11 -SDIRK34 -ISSPm2 -ISSPs3 +RKMethods.BE11 +RKMethods.SDIRK34 +RKMethods.ISSPm2 +RKMethods.ISSPs3 ``` ## Half explicit methods ```@docs -HEM3 -HEM3BS -HEM5 +RKMethods.HEM3 +RKMethods.HEM3BS +RKMethods.HEM5 ``` ## Classical Methods ```@docs -GL1 -GL2 -GL3 -RIA1 -RIA2 -RIA3 -RIIA1 -RIIA2 -RIIA3 -LIIIA2 -LIIIA3 +RKMethods.GL1 +RKMethods.GL2 +RKMethods.GL3 +RKMethods.RIA1 +RKMethods.RIA2 +RKMethods.RIA3 +RKMethods.RIIA1 +RKMethods.RIIA2 +RKMethods.RIIA3 +RKMethods.LIIIA2 +RKMethods.LIIIA3 ``` ## Chebyshev methods ```@docs -CHDIRK3 -CHCONS3 -CHC3 -CHC5 +RKMethods.CHDIRK3 +RKMethods.CHCONS3 +RKMethods.CHC3 +RKMethods.CHC5 ``` ## Miscellaneous Methods ```@docs -Mid22 -MTE22 -CN22 -Heun33 -RK33C2 -RK33P2 -RK44 -RK44C2 -RK44C23 -RK44P2 +RKMethods.Mid22 +RKMethods.MTE22 +RKMethods.CN22 +RKMethods.Heun33 +RKMethods.RK33C2 +RKMethods.RK33P2 +RKMethods.RK44 +RKMethods.RK44C2 +RKMethods.RK44C23 +RKMethods.RK44P2 ``` ## DSRK Methods ```@docs -DSso2 -DSRK2 -DSRK3 +RKMethods.DSso2 +RKMethods.DSRK2 +RKMethods.DSRK3 ``` ## "Non-SSP" Methods of Wong & Spiteri ```@docs -NSSP21 -NSSP32 -NSSP33 -NSSP53 +RKMethods.NSSP21 +RKMethods.NSSP32 +RKMethods.NSSP33 +RKMethods.NSSP53 ``` diff --git a/examples/Actuator2D.jl b/examples/Actuator2D.jl index 71435382e..66dc83881 100644 --- a/examples/Actuator2D.jl +++ b/examples/Actuator2D.jl @@ -65,7 +65,7 @@ state, outputs = solve_unsteady( u₀, (0.0, 12.0); psolver, - method = RK44P2(), + method = RKMethods.RK44P2(), Δt = 0.05, processors = ( rtp = realtimeplotter(; setup, plot = fieldplot, nupdate = 1), diff --git a/examples/Actuator3D.jl b/examples/Actuator3D.jl index fb0dfd508..941259c34 100644 --- a/examples/Actuator3D.jl +++ b/examples/Actuator3D.jl @@ -79,7 +79,7 @@ u₀ = create_initial_conditions(setup, (dim, x, y, z) -> dim() == 1 ? one(x) : setup, u₀, (T(0), T(3)); - method = RK44P2(), + method = RKMethods.RK44P2(), Δt = T(0.05), processors = ( rtp = realtimeplotter(; diff --git a/examples/MultiActuator.jl b/examples/MultiActuator.jl index 13d01492e..cb437830c 100644 --- a/examples/MultiActuator.jl +++ b/examples/MultiActuator.jl @@ -113,7 +113,7 @@ state, outputs = solve_unsteady( u₀, (T(0), 4 * T(12)); # (T(0), T(1)); - method = RK44P2(), + method = RKMethods.RK44P2(), Δt = T(0.01), psolver, processors = ( diff --git a/examples/PlanarMixing2D.jl b/examples/PlanarMixing2D.jl index ffd0ca80b..0098dfdf9 100644 --- a/examples/PlanarMixing2D.jl +++ b/examples/PlanarMixing2D.jl @@ -58,7 +58,7 @@ state, outputs = solve_unsteady( u₀, (0.0, 100.0); psolver, - method = RK44P2(), + method = RKMethods.RK44P2(), Δt = 0.1, processors = ( rtp = realtimeplotter(; diff --git a/examples/PlaneJets2D.jl b/examples/PlaneJets2D.jl index 111856413..fe2c558c0 100644 --- a/examples/PlaneJets2D.jl +++ b/examples/PlaneJets2D.jl @@ -154,7 +154,7 @@ state, outputs = solve_unsteady( setup, u₀, (T(0), T(1)); - method = RK44P2(), + method = RKMethods.RK44P2(), Δt = 0.001, psolver, processors = ( diff --git a/libs/NeuralClosure/src/training.jl b/libs/NeuralClosure/src/training.jl index 63aada1e0..a663bf0df 100644 --- a/libs/NeuralClosure/src/training.jl +++ b/libs/NeuralClosure/src/training.jl @@ -100,7 +100,7 @@ mean_squared_error(f, x, y, θ; normalize = y -> sum(abs2, y), λ = sqrt(eltype( """ create_loss_post(; setup, - method = RK44(; T = eltype(setup.grid.x[1])), + method = RKMethods.RK44(; T = eltype(setup.grid.x[1])), psolver, closure, nupdate = 1, @@ -111,7 +111,7 @@ Create a-posteriori loss function. """ function create_loss_post(; setup, - method = RK44(; T = eltype(setup.grid.x[1])), + method = RKMethods.RK44(; T = eltype(setup.grid.x[1])), psolver, closure, nupdate = 1, @@ -147,7 +147,7 @@ end create_relerr_post(; data, setup, - method = RK44(; T = eltype(setup.grid.x[1])), + method = RKMethods.RK44(; T = eltype(setup.grid.x[1])), psolver, closure_model, nupdate = 1, @@ -158,7 +158,7 @@ Create a-posteriori relative error. function create_relerr_post(; data, setup, - method = RK44(; T = eltype(setup.grid.x[1])), + method = RKMethods.RK44(; T = eltype(setup.grid.x[1])), psolver, closure_model, nupdate = 1, diff --git a/libs/PaperDC/postanalysis.jl b/libs/PaperDC/postanalysis.jl index c95bb527f..1398c6f4f 100644 --- a/libs/PaperDC/postanalysis.jl +++ b/libs/PaperDC/postanalysis.jl @@ -10,6 +10,7 @@ using Adapt using GLMakie using CairoMakie using IncompressibleNavierStokes +using IncompressibleNavierStokes.RKMethods using JLD2 using LaTeXStrings using LinearAlgebra diff --git a/libs/PaperDC/src/rk.jl b/libs/PaperDC/src/rk.jl index 42caf3e06..72e9059b4 100644 --- a/libs/PaperDC/src/rk.jl +++ b/libs/PaperDC/src/rk.jl @@ -2,7 +2,7 @@ RKProject(rk, projectorder) Runge-Kutta method with different projection order. -The Runge-Kutta method `rk` can be for example `RK44()`. +The Runge-Kutta method `rk` can be for example `RKMethods.RK44()`. - `projetorder = :first`: Project RHS before applying closure term. - `projetorder = :second`: Project RHS after applying closure term. diff --git a/src/IncompressibleNavierStokes.jl b/src/IncompressibleNavierStokes.jl index a4589ee0b..4ac3719d1 100644 --- a/src/IncompressibleNavierStokes.jl +++ b/src/IncompressibleNavierStokes.jl @@ -105,39 +105,17 @@ export DirectPressureSolver, # Solvers export solve_unsteady, solve_steady_state +# Field generation export create_initial_conditions, random_field +# Utils export plotgrid, save_vtk export plotmat +# Filter export FaceAverage, VolumeAverage, reconstruct, reconstruct! # ODE methods - -export AdamsBashforthCrankNicolsonMethod, OneLegMethod - -# Explicit Methods -export FE11, SSP22, SSP42, SSP33, SSP43, SSP104, rSSPs2, rSSPs3, Wray3, RK56, DOPRI6 - -# Implicit Methods -export BE11, SDIRK34, ISSPm2, ISSPs3 - -# Half explicit methods -export HEM3, HEM3BS, HEM5 - -# Classical Methods -export GL1, GL2, GL3, RIA1, RIA2, RIA3, RIIA1, RIIA2, RIIA3, LIIIA2, LIIIA3 - -# Chebyshev methods -export CHDIRK3, CHCONS3, CHC3, CHC5 - -# Miscellaneous Methods -export Mid22, MTE22, CN22, Heun33, RK33C2, RK33P2, RK44, RK44C2, RK44C23, RK44P2 - -# DSRK Methods -export DSso2, DSRK2, DSRK3 - -# "Non-SSP" Methods of Wong & Spiteri -export NSSP21, NSSP32, NSSP33, NSSP53 +export AdamsBashforthCrankNicolsonMethod, OneLegMethod, RKMethods end diff --git a/src/solvers/solve_unsteady.jl b/src/solvers/solve_unsteady.jl index a97e60058..88781454d 100644 --- a/src/solvers/solve_unsteady.jl +++ b/src/solvers/solve_unsteady.jl @@ -3,7 +3,7 @@ setup, u₀, tlims; - method = RK44(; T = eltype(u₀[1])), + method = RKMethods.RK44(; T = eltype(u₀[1])), psolver = DirectPressureSolver(setup), Δt = zero(eltype(u₀[1])), cfl = 1, @@ -32,7 +32,7 @@ function solve_unsteady( setup, u₀, tlims; - method = RK44(; T = eltype(u₀[1])), + method = RKMethods.RK44(; T = eltype(u₀[1])), psolver = DirectPressureSolver(setup), Δt = zero(eltype(u₀[1])), cfl = 1, diff --git a/src/time_steppers/methods.jl b/src/time_steppers/methods.jl index 45cca744d..56a432b11 100644 --- a/src/time_steppers/methods.jl +++ b/src/time_steppers/methods.jl @@ -12,7 +12,7 @@ abstract type AbstractODEMethod{T} end α₂ = T(-1 // 2), θ = T(1 // 2), p_add_solve = true, - method_startup = RK44(; T), + method_startup, ) IMEX AB-CN: Adams-Bashforth for explicit convection (parameters `α₁` and `α₂`) @@ -37,7 +37,7 @@ struct AdamsBashforthCrankNicolsonMethod{T,M} <: AbstractODEMethod{T} α₂ = T(-1 // 2), θ = T(1 // 2), p_add_solve = true, - method_startup = RK44(; T), + method_startup, ) = new{T,typeof(method_startup)}(α₁, α₂, θ, p_add_solve, method_startup) end @@ -46,7 +46,7 @@ end T = Float64; β = T(1 // 2), p_add_solve = true, - method_startup = RK44(; T), + method_startup, ) Explicit one-leg β-method following symmetry-preserving discretization of @@ -61,7 +61,7 @@ struct OneLegMethod{T,M} <: AbstractODEMethod{T} T = Float64; β = T(1 // 2), p_add_solve = true, - method_startup = RK44(; T), + method_startup, ) = new{T,typeof(method_startup)}(β, p_add_solve, method_startup) end diff --git a/src/time_steppers/tableaux.jl b/src/time_steppers/tableaux.jl index e9df37667..c4dc7e8b8 100644 --- a/src/time_steppers/tableaux.jl +++ b/src/time_steppers/tableaux.jl @@ -1,9 +1,36 @@ """ + RKMethods + Set up Butcher arrays `A`, `b`, and `c`, as well as and SSP coefficient `r`. For families of methods, optional input `s` is the number of stages. Original (MATLAB) by David Ketcheson, extended by Benjamin Sanderse. """ +module RKMethods + +# Explicit Methods +export FE11, SSP22, SSP42, SSP33, SSP43, SSP104, rSSPs2, rSSPs3, Wray3, RK56, DOPRI6 + +# Implicit Methods +export BE11, SDIRK34, ISSPm2, ISSPs3 + +# Half explicit methods +export HEM3, HEM3BS, HEM5 + +# Classical Methods +export GL1, GL2, GL3, RIA1, RIA2, RIA3, RIIA1, RIIA2, RIIA3, LIIIA2, LIIIA3 + +# Chebyshev methods +export CHDIRK3, CHCONS3, CHC3, CHC5 + +# Miscellaneous Methods +export Mid22, MTE22, CN22, Heun33, RK33C2, RK33P2, RK44, RK44C2, RK44C23, RK44P2 + +# DSRK Methods +export DSso2, DSRK2, DSRK3 + +# "Non-SSP" Methods of Wong & Spiteri +export NSSP21, NSSP32, NSSP33, NSSP53 # Default SSP coefficient # r = 0 @@ -807,3 +834,5 @@ function NSSP53(; kwargs...) c = [0, 1 // 7, 3 // 16, 1 // 3, 2 // 3] runge_kutta_method(A, b, c, r; kwargs...) end + +end From 54757c9c3d1b18c914d0dcebd274c3fa5d023368 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Fri, 3 May 2024 10:30:29 +0200 Subject: [PATCH 335/379] Fix setup --- src/setup.jl | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/setup.jl b/src/setup.jl index 6a28f2197..2f9c87f11 100644 --- a/src/setup.jl +++ b/src/setup.jl @@ -3,7 +3,6 @@ x...; boundary_conditions = ntuple(d -> (PeriodicBC(), PeriodicBC()), length(x)), Re = convert(eltype(x[1]), 1_000), - viscosity_model = LaminarModel(), bodyforce = nothing, issteadybodyforce = true, closure_model = nothing, @@ -17,7 +16,6 @@ function Setup( x...; boundary_conditions = ntuple(d -> (PeriodicBC(), PeriodicBC()), length(x)), Re = convert(eltype(x[1]), 1_000), - viscosity_model = LaminarModel(), bodyforce = nothing, issteadybodyforce = true, closure_model = nothing, @@ -29,7 +27,6 @@ function Setup( grid = Grid(x, boundary_conditions; ArrayType), boundary_conditions, Re, - viscosity_model, bodyforce, issteadybodyforce = false, closure_model, From 547b61e1cd22da39a91a042ffbd28041d2158b5a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Fri, 3 May 2024 10:36:56 +0200 Subject: [PATCH 336/379] Fix tests --- test/Project.toml | 1 + test/operators.jl | 15 ++++++++------- test/pressure_solvers.jl | 2 +- test/runtests.jl | 2 ++ 4 files changed, 12 insertions(+), 8 deletions(-) diff --git a/test/Project.toml b/test/Project.toml index 7171c3a49..7a0313284 100644 --- a/test/Project.toml +++ b/test/Project.toml @@ -3,5 +3,6 @@ Aqua = "4c88cf16-eb10-579e-8560-4a9242c79595" CairoMakie = "13f3f980-e62b-5c42-98c6-ff1f3baf88f0" ChainRulesTestUtils = "cdddcdb0-9152-4a09-a978-84456f9df70a" LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" +Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2" Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" diff --git a/test/operators.jl b/test/operators.jl index 8b87964de..258e183e3 100644 --- a/test/operators.jl +++ b/test/operators.jl @@ -8,7 +8,7 @@ setup = Setup(x...; Re) psolver = DirectPressureSolver(setup) u = random_field(setup, T(0); psolver) - (; Iu, Ip) = setup.grid + (; Iu, Ip, Ω) = setup.grid @testset "Divergence" begin div = divergence(u, setup) @@ -19,8 +19,8 @@ @testset "Pressure gradient" begin v = randn!.(similar.(u)) p = randn!(similar(u[1])) - apply_bc_u!(v, T(0), setup) - apply_bc_p!(p, T(0), setup) + v = apply_bc_u(v, T(0), setup) + p = apply_bc_p(p, T(0), setup) Dv = divergence(v, setup) Gp = pressuregradient(p, setup) pDv = if length(v) == 2 @@ -31,9 +31,10 @@ vGpx = v[2] .* setup.grid.Δ[1] .* setup.grid.Δu[2]' .* Gp[2] sum(vGpx[Iu[1]]) + sum(vGpx[Iu[2]]) end - @test G isa Tuple - @test G[1] isa Array{T} - @test pDv ≈ -vGp # Check that D = -G' + @test Gp isa Tuple + @test Gp[1] isa Array{T} + # FIXME: Find how to put Ω + @test_broken pDv ≈ -vGp # Check that D = -G' end @testset "Convection" begin @@ -57,6 +58,6 @@ end @test d isa Tuple @test d[1] isa Array{T} - @test uDu ≥ 0 # Check positivity + @test uDu ≤ 0 # Check negativity (dissipation) end end diff --git a/test/pressure_solvers.jl b/test/pressure_solvers.jl index 8fb836497..bc473f345 100644 --- a/test/pressure_solvers.jl +++ b/test/pressure_solvers.jl @@ -12,7 +12,7 @@ direct = DirectPressureSolver(setup) cg = CGPressureSolver(setup) spectral = SpectralPressureSolver(setup) - lmspectral = LowerMemorySpectralPressureSolver(setup) + lmspectral = LowMemorySpectralPressureSolver(setup) initial_pressure(x, y) = 1 / 4 * (cos(2x) + cos(2y)) p_exact = diff --git a/test/runtests.jl b/test/runtests.jl index 14220eff4..a3dfea923 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -2,7 +2,9 @@ using Aqua using CairoMakie using ChainRulesTestUtils using IncompressibleNavierStokes +using IncompressibleNavierStokes: divergence, pressuregradient, convection, diffusion, apply_bc_u, apply_bc_p using LinearAlgebra +using Random using Statistics using Test From bbd86f07547eb825a205ce602acf7b5073bb59d0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Fri, 3 May 2024 13:19:10 +0200 Subject: [PATCH 337/379] Fix dependency --- docs/src/features/les.md | 9 ++------- docs/src/features/operators.md | 2 -- src/IncompressibleNavierStokes.jl | 2 +- src/time_steppers/tableaux.jl | 2 ++ 4 files changed, 5 insertions(+), 10 deletions(-) diff --git a/docs/src/features/les.md b/docs/src/features/les.md index dd67c2c80..9b1e741ac 100644 --- a/docs/src/features/les.md +++ b/docs/src/features/les.md @@ -32,12 +32,7 @@ atomic scales. The new turbulent viscosity on the other hand, models energy transfer from resolved to unresolved scales. This non-constant field is computed from the local velocity field. -The following eddy viscosity models are available: - ```@docs -AbstractViscosityModel -LaminarModel -SmagorinskyModel -QRModel -MixingLengthModel +smagtensor! +smagorinsky_closure ``` diff --git a/docs/src/features/operators.md b/docs/src/features/operators.md index de4f7fc9b..ae9845a84 100644 --- a/docs/src/features/operators.md +++ b/docs/src/features/operators.md @@ -44,8 +44,6 @@ kinetic_energy kinetic_energy! total_kinetic_energy tensorbasis -smagtensor! -smagorinsky_closure divoftensor! tensorbasis! reconstruct! diff --git a/src/IncompressibleNavierStokes.jl b/src/IncompressibleNavierStokes.jl index 4ac3719d1..ced36027f 100644 --- a/src/IncompressibleNavierStokes.jl +++ b/src/IncompressibleNavierStokes.jl @@ -50,12 +50,12 @@ include("solvers/pressure/project.jl") # Time steppers include("time_steppers/methods.jl") -include("time_steppers/tableaux.jl") include("time_steppers/nstage.jl") include("time_steppers/time_stepper_caches.jl") include("time_steppers/step.jl") include("time_steppers/isexplicit.jl") include("time_steppers/lambda_max.jl") +include("time_steppers/tableaux.jl") # Preprocess include("create_initial_conditions.jl") diff --git a/src/time_steppers/tableaux.jl b/src/time_steppers/tableaux.jl index c4dc7e8b8..ba9bbc49e 100644 --- a/src/time_steppers/tableaux.jl +++ b/src/time_steppers/tableaux.jl @@ -8,6 +8,8 @@ Original (MATLAB) by David Ketcheson, extended by Benjamin Sanderse. """ module RKMethods +using IncompressibleNavierStokes: runge_kutta_method + # Explicit Methods export FE11, SSP22, SSP42, SSP33, SSP43, SSP104, rSSPs2, rSSPs3, Wray3, RK56, DOPRI6 From 00509d11787fc5b075d85b02f81448283a57f411 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Fri, 3 May 2024 13:20:08 +0200 Subject: [PATCH 338/379] chore: Format files --- src/operators.jl | 3 +-- src/time_steppers/methods.jl | 8 ++------ test/chainrules.jl | 3 +-- test/operators.jl | 4 ++-- test/runtests.jl | 3 ++- 5 files changed, 8 insertions(+), 13 deletions(-) diff --git a/src/operators.jl b/src/operators.jl index a3b165a2a..90ecffe37 100644 --- a/src/operators.jl +++ b/src/operators.jl @@ -423,7 +423,7 @@ function convection!(F, u, setup) # uβα1 = u[β][I-δ(β)] / 2 + u[β][I-δ(β)+δ(α)] / 2 # uαβ2 = (u[α][I] + u[α][I+δ(β)]) / 2 # uβα2 = u[β][I] / 2 + u[β][I+δ(α)] / 2 - + # # Interpolation # uαβ1 = A[α][β][2][I[β]-1] * u[α][I-δ(β)] + A[α][β][1][I[β]] * u[α][I] # uβα1 = @@ -824,7 +824,6 @@ function vorticity!(::Dimension{3}, ω, u, setup) ω end - @inline ∂x(uα, I::CartesianIndex{D}, α, β, Δβ, Δuβ; δ = Offset{D}()) where {D} = α == β ? (uα[I] - uα[I-δ(β)]) / Δβ[I[β]] : ( diff --git a/src/time_steppers/methods.jl b/src/time_steppers/methods.jl index 56a432b11..7921872cb 100644 --- a/src/time_steppers/methods.jl +++ b/src/time_steppers/methods.jl @@ -57,12 +57,8 @@ struct OneLegMethod{T,M} <: AbstractODEMethod{T} β::T p_add_solve::Bool method_startup::M - OneLegMethod( - T = Float64; - β = T(1 // 2), - p_add_solve = true, - method_startup, - ) = new{T,typeof(method_startup)}(β, p_add_solve, method_startup) + OneLegMethod(T = Float64; β = T(1 // 2), p_add_solve = true, method_startup) = + new{T,typeof(method_startup)}(β, p_add_solve, method_startup) end """ diff --git a/test/chainrules.jl b/test/chainrules.jl index bf84661d6..df7bd2164 100644 --- a/test/chainrules.jl +++ b/test/chainrules.jl @@ -1,2 +1 @@ -@testset "Chain rules" begin -end +@testset "Chain rules" begin end diff --git a/test/operators.jl b/test/operators.jl index 258e183e3..c0e0a7820 100644 --- a/test/operators.jl +++ b/test/operators.jl @@ -15,7 +15,7 @@ @test div isa Array{T} @test all(!isnan, div) end - + @testset "Pressure gradient" begin v = randn!.(similar.(u)) p = randn!(similar(u[1])) @@ -24,7 +24,7 @@ Dv = divergence(v, setup) Gp = pressuregradient(p, setup) pDv = if length(v) == 2 - sum((p .* Ω .* Dv)[Ip]) + sum((p.*Ω.*Dv)[Ip]) end vGp = if length(v) == 2 vGpx = v[1] .* setup.grid.Δu[1] .* setup.grid.Δ[2]' .* Gp[1] diff --git a/test/runtests.jl b/test/runtests.jl index a3dfea923..32159b40a 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -2,7 +2,8 @@ using Aqua using CairoMakie using ChainRulesTestUtils using IncompressibleNavierStokes -using IncompressibleNavierStokes: divergence, pressuregradient, convection, diffusion, apply_bc_u, apply_bc_p +using IncompressibleNavierStokes: + divergence, pressuregradient, convection, diffusion, apply_bc_u, apply_bc_p using LinearAlgebra using Random using Statistics From a34e2eee349b6d0987080204204b51e66498a99a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Fri, 3 May 2024 13:20:55 +0200 Subject: [PATCH 339/379] Update format options --- .JuliaFormatter.toml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.JuliaFormatter.toml b/.JuliaFormatter.toml index c2ccc2df2..99fcf989a 100644 --- a/.JuliaFormatter.toml +++ b/.JuliaFormatter.toml @@ -1,3 +1,5 @@ whitespace_in_kwargs = true remove_extra_newlines = true trailing_comma = true +separate_kwargs_with_semicolon = true +long_to_short_function_def = true From ef8e7585f8b6922795615c6bb736e13534606f1d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Fri, 3 May 2024 13:37:34 +0200 Subject: [PATCH 340/379] Add 3D tests --- test/operators.jl | 62 ++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 50 insertions(+), 12 deletions(-) diff --git a/test/operators.jl b/test/operators.jl index c0e0a7820..e080df633 100644 --- a/test/operators.jl +++ b/test/operators.jl @@ -1,10 +1,15 @@ -@testset "Operators" begin +testops(dim) = @testset "Operators $(dim())D" begin # Setup + D = dim() T = Float64 Re = T(1_000) n = 16 lims = T(0), T(1) - x = stretched_grid(lims..., n, 1.2), cosine_grid(lims..., n) + x = if D == 2 + stretched_grid(lims..., n, 1.2), cosine_grid(lims..., n) + elseif D == 3 + stretched_grid(lims..., n, 1.2), cosine_grid(lims..., n), cosine_grid(lims..., n) + end setup = Setup(x...; Re) psolver = DirectPressureSolver(setup) u = random_field(setup, T(0); psolver) @@ -23,26 +28,45 @@ p = apply_bc_p(p, T(0), setup) Dv = divergence(v, setup) Gp = pressuregradient(p, setup) - pDv = if length(v) == 2 - sum((p.*Ω.*Dv)[Ip]) - end - vGp = if length(v) == 2 + pDv = sum((p.*Ω.*Dv)[Ip]) + vGp = if D == 2 vGpx = v[1] .* setup.grid.Δu[1] .* setup.grid.Δ[2]' .* Gp[1] - vGpx = v[2] .* setup.grid.Δ[1] .* setup.grid.Δu[2]' .* Gp[2] - sum(vGpx[Iu[1]]) + sum(vGpx[Iu[2]]) + vGpy = v[2] .* setup.grid.Δ[1] .* setup.grid.Δu[2]' .* Gp[2] + sum(vGpx[Iu[1]]) + sum(vGpy[Iu[2]]) + elseif D == 3 + vGpx = + v[1] .* setup.grid.Δu[1] .* reshape(setup.grid.Δ[2], 1, :) .* + reshape(setup.grid.Δ[3], 1, 1, :) .* Gp[1] + vGpy = + v[2] .* setup.grid.Δ[1] .* reshape(setup.grid.Δu[2], 1, :) .* + reshape(setup.grid.Δ[3], 1, 1, :) .* Gp[2] + vGpz = + v[3] .* setup.grid.Δ[1] .* reshape(setup.grid.Δ[2], 1, :) .* + reshape(setup.grid.Δu[3], 1, 1, :) .* Gp[3] + sum(vGpx[Iu[1]]) + sum(vGpy[Iu[2]]) + sum(vGpz[Iu[3]]) end @test Gp isa Tuple @test Gp[1] isa Array{T} - # FIXME: Find how to put Ω - @test_broken pDv ≈ -vGp # Check that D = -G' + @test pDv ≈ -vGp # Check that D = -G' end @testset "Convection" begin c = convection(u, setup) - uCu = if length(u) == 2 + uCu = if D == 2 uCux = u[1] .* setup.grid.Δu[1] .* setup.grid.Δ[2]' .* c[1] uCuy = u[2] .* setup.grid.Δ[1] .* setup.grid.Δu[2]' .* c[2] sum(uCux[Iu[1]]) + sum(uCuy[Iu[2]]) + elseif D == 3 + uCux = + u[1] .* setup.grid.Δu[1] .* reshape(setup.grid.Δ[2], 1, :) .* + reshape(setup.grid.Δ[3], 1, 1, :) .* c[1] + uCuy = + u[2] .* setup.grid.Δ[1] .* reshape(setup.grid.Δu[2], 1, :) .* + reshape(setup.grid.Δ[3], 1, 1, :) .* c[2] + uCuz = + u[3] .* setup.grid.Δ[1] .* reshape(setup.grid.Δ[2], 1, :) .* + reshape(setup.grid.Δu[3], 1, 1, :) .* c[3] + sum(uCux[Iu[1]]) + sum(uCuy[Iu[2]]) + sum(uCuz[Iu[3]]) end @test c isa Tuple @test c[1] isa Array{T} @@ -51,13 +75,27 @@ @testset "Diffusion" begin d = diffusion(u, setup) - uDu = if length(u) == 2 + uDu = if D == 2 uDux = u[1] .* setup.grid.Δu[1] .* setup.grid.Δ[2]' .* d[1] uDuy = u[2] .* setup.grid.Δ[1] .* setup.grid.Δu[2]' .* d[2] sum(uDux[Iu[1]]) + sum(uDuy[Iu[2]]) + elseif D == 3 + uDux = + u[1] .* setup.grid.Δu[1] .* reshape(setup.grid.Δ[2], 1, :) .* + reshape(setup.grid.Δ[3], 1, 1, :) .* d[1] + uDuy = + u[2] .* setup.grid.Δ[1] .* reshape(setup.grid.Δu[2], 1, :) .* + reshape(setup.grid.Δ[3], 1, 1, :) .* d[2] + uDuz = + u[3] .* setup.grid.Δ[1] .* reshape(setup.grid.Δ[2], 1, :) .* + reshape(setup.grid.Δu[3], 1, 1, :) .* d[3] + sum(uDux[Iu[1]]) + sum(uDuy[Iu[2]]) + sum(uDuz[Iu[3]]) end @test d isa Tuple @test d[1] isa Array{T} @test uDu ≤ 0 # Check negativity (dissipation) end end + +testops(IncompressibleNavierStokes.Dimension(2)) +testops(IncompressibleNavierStokes.Dimension(3)) From 303ccc4c51fd903464c65624d778e97533b9f4ec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Fri, 3 May 2024 15:22:56 +0200 Subject: [PATCH 341/379] Add and fix chain rule tests --- src/operators.jl | 18 ++++++++++++++--- test/Project.toml | 1 + test/chainrules.jl | 50 +++++++++++++++++++++++++++++++++++++++++++++- test/runtests.jl | 3 ++- 4 files changed, 67 insertions(+), 5 deletions(-) diff --git a/src/operators.jl b/src/operators.jl index 90ecffe37..437510d99 100644 --- a/src/operators.jl +++ b/src/operators.jl @@ -101,7 +101,11 @@ divergence(u, setup) = divergence!(fill!(similar(u[1], setup.grid.N), 0), u, set ChainRulesCore.rrule(::typeof(divergence), u, setup) = ( divergence(u, setup), - φ -> (NoTangent(), divergence_adjoint!(similar.(u), φ, setup), NoTangent()), + φ -> ( + NoTangent(), + divergence_adjoint!(Tangent{typeof(u)}(similar.(u)...), φ, setup), + NoTangent(), + ), ) """ @@ -546,7 +550,12 @@ convection(u, setup) = convection!(zero.(u), u, setup) ChainRulesCore.rrule(::typeof(convection), u, setup) = ( convection(u, setup), - φ -> (NoTangent(), convection_adjoint!(zero.(u), (φ...,), u, setup), NoTangent()), + φ -> ( + NoTangent(), + # convection_adjoint!(Tangent{typeof(u)}(zero.(u)...), (φ...,), u, setup), + convection_adjoint!(Tangent{typeof(u)}(zero.(u)...), (φ...,), u, setup), + NoTangent(), + ), ) """ @@ -624,7 +633,10 @@ diffusion(u, setup) = diffusion!(zero.(u), u, setup) ChainRulesCore.rrule(::typeof(diffusion), u, setup) = ( diffusion(u, setup), - φ -> (NoTangent(), diffusion_adjoint!(zero.(u), (φ...,), setup), NoTangent()), + φ -> (NoTangent(), diffusion_adjoint!( + # zero.(u), + Tangent{typeof(u)}(zero.(u)...), + (φ...,), setup), NoTangent()), ) function convectiondiffusion!(F, u, setup) diff --git a/test/Project.toml b/test/Project.toml index 7a0313284..eebe741f6 100644 --- a/test/Project.toml +++ b/test/Project.toml @@ -1,6 +1,7 @@ [deps] Aqua = "4c88cf16-eb10-579e-8560-4a9242c79595" CairoMakie = "13f3f980-e62b-5c42-98c6-ff1f3baf88f0" +ChainRulesCore = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4" ChainRulesTestUtils = "cdddcdb0-9152-4a09-a978-84456f9df70a" LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" diff --git a/test/chainrules.jl b/test/chainrules.jl index df7bd2164..da484e2a6 100644 --- a/test/chainrules.jl +++ b/test/chainrules.jl @@ -1 +1,49 @@ -@testset "Chain rules" begin end +testchainrules(dim) = @testset "Chain rules $(dim())D" begin + # Setup + D = dim() + T = Float64 + Re = T(1_000) + n = if D == 2 + 8 + elseif D == 3 + # 4^3 = 64 grid points + # 3*64 = 192 velocity components + # 192^2 = 36864 fininite difference pairs in convection/diffusion + # TODO: Check if `test_rrule` computes all combinations or only a subset + 4 + end + lims = T(0), T(1) + x = if D == 2 + stretched_grid(lims..., n, 1.2), cosine_grid(lims..., n) + elseif D == 3 + stretched_grid(lims..., n, 1.2), cosine_grid(lims..., n), cosine_grid(lims..., n) + end + setup = Setup(x...; Re) + psolver = DirectPressureSolver(setup) + u = random_field(setup, T(0); psolver) + (; Iu, Ip, Ω) = setup.grid + randn!.(u) + p = randn!(similar(u[1])) + @testset "Divergence" begin + test_rrule(divergence, u, setup ⊢ NoTangent()) + end + @testset "Pressure gradient" begin + test_rrule(pressuregradient, p, setup ⊢ NoTangent()) + end + @testset "Pressure gradient" begin + test_rrule(pressuregradient, p, setup ⊢ NoTangent()) + end + @testset "Convection" begin + test_rrule(convection, u, setup ⊢ NoTangent()) + end + @testset "Diffusion" begin + test_rrule(diffusion, u, setup ⊢ NoTangent()) + end + @testset "Bodyforce" begin + @test_broken 1 == 2 # Just to identify location for broken rrule test + # test_rrule(bodyforce, u, T(0) ⊢ NoTangent(), setup ⊢ NoTangent()) + end +end + +testchainrules(IncompressibleNavierStokes.Dimension(2)) +testchainrules(IncompressibleNavierStokes.Dimension(3)); diff --git a/test/runtests.jl b/test/runtests.jl index 32159b40a..67bd9e019 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -1,9 +1,10 @@ using Aqua using CairoMakie +using ChainRulesCore using ChainRulesTestUtils using IncompressibleNavierStokes using IncompressibleNavierStokes: - divergence, pressuregradient, convection, diffusion, apply_bc_u, apply_bc_p + divergence, pressuregradient, convection, diffusion, bodyforce, apply_bc_u, apply_bc_p using LinearAlgebra using Random using Statistics From 197517cdc6b75c66ad69a774ac70f77377b97652 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Fri, 3 May 2024 15:23:24 +0200 Subject: [PATCH 342/379] Remove undefined rule --- src/operators.jl | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/operators.jl b/src/operators.jl index 437510d99..2d00e65c9 100644 --- a/src/operators.jl +++ b/src/operators.jl @@ -672,13 +672,13 @@ function convectiondiffusion!(F, u, setup) F end -convectiondiffusion(u, setup) = convectiondiffusion!(zero.(u), u, setup) - -ChainRulesCore.rrule(::typeof(convectiondiffusion), u, setup) = ( - convection(u, setup), - φ -> - (NoTangent(), convectiondiffusion_adjoint!(similar.(u), φ, setup), NoTangent()), -) +# convectiondiffusion(u, setup) = convectiondiffusion!(zero.(u), u, setup) +# +# ChainRulesCore.rrule(::typeof(convectiondiffusion), u, setup) = ( +# convection(u, setup), +# φ -> +# (NoTangent(), convectiondiffusion_adjoint!(similar.(u), φ, setup), NoTangent()), +# ) """ bodyforce!(F, u, t, setup) From 2516deec2a85319fef021f4225175e5e3db89205 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Fri, 3 May 2024 15:26:22 +0200 Subject: [PATCH 343/379] chore: Format --- src/operators.jl | 9 +++++---- test/chainrules.jl | 2 +- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/src/operators.jl b/src/operators.jl index 2d00e65c9..56af6554b 100644 --- a/src/operators.jl +++ b/src/operators.jl @@ -633,10 +633,11 @@ diffusion(u, setup) = diffusion!(zero.(u), u, setup) ChainRulesCore.rrule(::typeof(diffusion), u, setup) = ( diffusion(u, setup), - φ -> (NoTangent(), diffusion_adjoint!( - # zero.(u), - Tangent{typeof(u)}(zero.(u)...), - (φ...,), setup), NoTangent()), + φ -> ( + NoTangent(), + diffusion_adjoint!(Tangent{typeof(u)}(zero.(u)...), (φ...,), setup), + NoTangent(), + ), ) function convectiondiffusion!(F, u, setup) diff --git a/test/chainrules.jl b/test/chainrules.jl index da484e2a6..3b5f6f356 100644 --- a/test/chainrules.jl +++ b/test/chainrules.jl @@ -3,7 +3,7 @@ testchainrules(dim) = @testset "Chain rules $(dim())D" begin D = dim() T = Float64 Re = T(1_000) - n = if D == 2 + n = if D == 2 8 elseif D == 3 # 4^3 = 64 grid points From b08fd11b8f62b55b95326c5c882ae9311ac44536 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Fri, 3 May 2024 16:12:41 +0200 Subject: [PATCH 344/379] Rename symbols --- src/operators.jl | 270 +++++++++++++++++++++++------------------------ 1 file changed, 135 insertions(+), 135 deletions(-) diff --git a/src/operators.jl b/src/operators.jl index 56af6554b..230b3a31b 100644 --- a/src/operators.jl +++ b/src/operators.jl @@ -37,10 +37,10 @@ # In the future, Enzyme.jl might be able to do this automatically. """ - δ = Offset{D}() + e = Offset{D}() Cartesian index unit vector in `D = 2` or `D = 3` dimensions. -Calling `δ(α)` returns a Cartesian index with `1` in the dimension `α` and zeros +Calling `e(α)` returns a Cartesian index with `1` in the dimension `α` and zeros elsewhere. See @@ -59,13 +59,13 @@ function divergence!(div, u, setup) (; grid, workgroupsize) = setup (; Δ, N, Ip, Np) = grid D = length(u) - δ = Offset{D}() + e = Offset{D}() @kernel function div!(div, u, I0) I = @index(Global, Cartesian) I = I + I0 d = zero(eltype(div)) for α = 1:D - d += (u[α][I] - u[α][I-δ(α)]) / Δ[α][I[α]] + d += (u[α][I] - u[α][I-e(α)]) / Δ[α][I[α]] end div[I] = d end @@ -79,13 +79,13 @@ function divergence_adjoint!(u, φ, setup) (; grid, workgroupsize) = setup (; Δ, N, Ip) = grid D = length(u) - δ = Offset{D}() + e = Offset{D}() @kernel function adj!(u, φ) I = @index(Global, Cartesian) for α = 1:D u[α][I] = zero(eltype(u[1])) I ∈ Ip && (u[α][I] += φ[I] / Δ[α][I[α]]) - I + δ(α) ∈ Ip && (u[α][I] -= φ[I+δ(α)] / Δ[α][I[α]+1]) + I + e(α) ∈ Ip && (u[α][I] -= φ[I+e(α)] / Δ[α][I[α]+1]) end end adj!(get_backend(u[1]), workgroupsize)(u, φ; ndrange = N) @@ -117,11 +117,11 @@ function pressuregradient!(G, p, setup) (; grid, workgroupsize) = setup (; dimension, Δu, Nu, Iu) = grid D = dimension() - δ = Offset{D}() + e = Offset{D}() @kernel function G!(G, p, ::Val{α}, I0) where {α} I = @index(Global, Cartesian) I = I0 + I - G[α][I] = (p[I+δ(α)] - p[I]) / Δu[α][I[α]] + G[α][I] = (p[I+e(α)] - p[I]) / Δu[α][I[α]] end D = dimension() for α = 1:D @@ -136,12 +136,12 @@ function pressuregradient_adjoint!(pbar, φ, setup) (; grid, workgroupsize) = setup (; dimension, Δu, N, Iu) = grid D = dimension() - δ = Offset{D}() + e = Offset{D}() @kernel function adj!(p, φ) I = @index(Global, Cartesian) p[I] = zero(eltype(p)) for α = 1:D - I - δ(α) ∈ Iu[α] && (p[I] += φ[α][I-δ(α)] / Δu[α][I[α]-1]) + I - e(α) ∈ Iu[α] && (p[I] += φ[α][I-e(α)] / Δu[α][I[α]-1]) I ∈ Iu[α] && (p[I] -= φ[α][I] / Δu[α][I[α]]) end end @@ -171,11 +171,11 @@ function applypressure!(u, p, setup) (; grid, workgroupsize) = setup (; dimension, Δu, Nu, Iu) = grid D = dimension() - δ = Offset{D}() + e = Offset{D}() @kernel function apply!(u, p, ::Val{α}, I0) where {α} I = @index(Global, Cartesian) I = I0 + I - u[α][I] -= (p[I+δ(α)] - p[I]) / Δu[α][I[α]] + u[α][I] -= (p[I+e(α)] - p[I]) / Δu[α][I[α]] end D = dimension() for α = 1:D @@ -190,12 +190,12 @@ end # (; grid, workgroupsize) = setup # (; dimension, Δu, N, Iu) = grid # D = dimension() -# δ = Offset{D}() +# e = Offset{D}() # @kernel function adj!(p, φ) # I = @index(Global, Cartesian) # p[I] = zero(eltype(p)) # for α = 1:D -# I - δ(α) ∈ Iu[α] && (p[I] += φ[α][I-δ(α)] / Δu[α][I[α]-1]) +# I - e(α) ∈ Iu[α] && (p[I] += φ[α][I-e(α)] / Δu[α][I[α]-1]) # I ∈ Iu[α] && (p[I] -= φ[α][I] / Δu[α][I[α]]) # end # end @@ -225,7 +225,7 @@ function laplacian!(L, p, setup) (; grid, workgroupsize, boundary_conditions) = setup (; dimension, Δ, Δu, N, Np, Ip, Ω) = grid D = dimension() - δ = Offset{D}() + e = Offset{D}() # @kernel function lap!(L, p, I0) # I = @index(Global, Cartesian) # I = I + I0 @@ -235,19 +235,19 @@ function laplacian!(L, p, setup) # if bc[1] isa PressureBC && I[α] == I0[α] + 1 # lap += # Ω[I] / Δ[α][I[α]] * - # ((p[I+δ(α)] - p[I]) / Δu[α][I[α]] - (p[I]) / Δu[α][I[α]-1]) + # ((p[I+e(α)] - p[I]) / Δu[α][I[α]] - (p[I]) / Δu[α][I[α]-1]) # elseif bc[2] isa PressureBC && I[α] == I0[α] + Np[α] # lap += # Ω[I] / Δ[α][I[α]] * - # ((-p[I]) / Δu[α][I[α]] - (p[I] - p[I-δ(α)]) / Δu[α][I[α]-1]) + # ((-p[I]) / Δu[α][I[α]] - (p[I] - p[I-e(α)]) / Δu[α][I[α]-1]) # elseif bc[1] isa DirichletBC && I[α] == I0[α] + 1 - # lap += Ω[I] / Δ[α][I[α]] * ((p[I+δ(α)] - p[I]) / Δu[α][I[α]]) + # lap += Ω[I] / Δ[α][I[α]] * ((p[I+e(α)] - p[I]) / Δu[α][I[α]]) # elseif bc[2] isa DirichletBC && I[α] == I0[α] + Np[α] - # lap += Ω[I] / Δ[α][I[α]] * (-(p[I] - p[I-δ(α)]) / Δu[α][I[α]-1]) + # lap += Ω[I] / Δ[α][I[α]] * (-(p[I] - p[I-e(α)]) / Δu[α][I[α]-1]) # else # lap += # Ω[I] / Δ[α][I[α]] * - # ((p[I+δ(α)] - p[I]) / Δu[α][I[α]] - (p[I] - p[I-δ(α)]) / Δu[α][I[α]-1]) + # ((p[I+e(α)] - p[I]) / Δu[α][I[α]] - (p[I] - p[I-e(α)]) / Δu[α][I[α]-1]) # end # end # L[I] = lap @@ -259,19 +259,19 @@ function laplacian!(L, p, setup) if bc[1] isa PressureBC && I[α] == I0[α] + 1 L[I] += Ω[I] / Δ[α][I[α]] * - ((p[I+δ(α)] - p[I]) / Δu[α][I[α]] - (p[I]) / Δu[α][I[α]-1]) + ((p[I+e(α)] - p[I]) / Δu[α][I[α]] - (p[I]) / Δu[α][I[α]-1]) elseif bc[2] isa PressureBC && I[α] == I0[α] + Np[α] L[I] += Ω[I] / Δ[α][I[α]] * - ((-p[I]) / Δu[α][I[α]] - (p[I] - p[I-δ(α)]) / Δu[α][I[α]-1]) + ((-p[I]) / Δu[α][I[α]] - (p[I] - p[I-e(α)]) / Δu[α][I[α]-1]) elseif bc[1] isa DirichletBC && I[α] == I0[α] + 1 - L[I] += Ω[I] / Δ[α][I[α]] * ((p[I+δ(α)] - p[I]) / Δu[α][I[α]]) + L[I] += Ω[I] / Δ[α][I[α]] * ((p[I+e(α)] - p[I]) / Δu[α][I[α]]) elseif bc[2] isa DirichletBC && I[α] == I0[α] + Np[α] - L[I] += Ω[I] / Δ[α][I[α]] * (-(p[I] - p[I-δ(α)]) / Δu[α][I[α]-1]) + L[I] += Ω[I] / Δ[α][I[α]] * (-(p[I] - p[I-e(α)]) / Δu[α][I[α]-1]) else L[I] += Ω[I] / Δ[α][I[α]] * - ((p[I+δ(α)] - p[I]) / Δu[α][I[α]] - (p[I] - p[I-δ(α)]) / Δu[α][I[α]-1]) + ((p[I+e(α)] - p[I]) / Δu[α][I[α]] - (p[I] - p[I-e(α)]) / Δu[α][I[α]-1]) end # L[I] = lap end @@ -309,7 +309,7 @@ function laplacian_mat(setup) backend = get_backend(x[1]) T = eltype(x[1]) D = dimension() - δ = Offset{D}() + e = Offset{D}() Ia = first(Ip) Ib = last(Ip) I = similar(x[1], CartesianIndex{D}, 0) @@ -324,7 +324,7 @@ function laplacian_mat(setup) for (aa, bb, j) in [(a, nothing, ia), (nothing, nothing, i), (nothing, b, ib)] vala = @.(Ω[j] / Δ[α][getindex.(j, α)] / Δu[α][getindex.(j, α)-1]) if isnothing(aa) - J = [J; j .- [δ(α)]; j] + J = [J; j .- [e(α)]; j] I = [I; j; j] val = [val; vala; -vala] elseif aa isa PressureBC @@ -344,7 +344,7 @@ function laplacian_mat(setup) valb = @.(Ω[j] / Δ[α][getindex.(j, α)] / Δu[α][getindex.(j, α)]) if isnothing(bb) - J = [J; j; j .+ [δ(α)]] + J = [J; j; j .+ [e(α)]] I = [I; j; j] val = [val; -valb; valb] elseif bb isa PressureBC @@ -402,7 +402,7 @@ function convection!(F, u, setup) (; grid, workgroupsize) = setup (; dimension, Δ, Δu, Nu, Iu, A) = grid D = dimension() - δ = Offset{D}() + e = Offset{D}() @kernel function conv!(F, u, ::Val{α}, ::Val{βrange}, I0) where {α,βrange} I = @index(Global, Cartesian) I = I + I0 @@ -413,28 +413,28 @@ function convection!(F, u, setup) # Half for u[α], (reverse!) interpolation for u[β] # Note: # In matrix version, uses - # 1*u[α][I-δ(β)] + 0*u[α][I] - # instead of 1/2 when u[α][I-δ(β)] is at Dirichlet boundary. - uαβ1 = (u[α][I-δ(β)] + u[α][I]) / 2 - uαβ2 = (u[α][I] + u[α][I+δ(β)]) / 2 + # 1*u[α][I-e(β)] + 0*u[α][I] + # instead of 1/2 when u[α][I-e(β)] is at Dirichlet boundary. + uαβ1 = (u[α][I-e(β)] + u[α][I]) / 2 + uαβ2 = (u[α][I] + u[α][I+e(β)]) / 2 uβα1 = - A[β][α][2][I[α]-(α==β)] * u[β][I-δ(β)] + - A[β][α][1][I[α]+(α!=β)] * u[β][I-δ(β)+δ(α)] - uβα2 = A[β][α][2][I[α]] * u[β][I] + A[β][α][1][I[α]+1] * u[β][I+δ(α)] + A[β][α][2][I[α]-(α==β)] * u[β][I-e(β)] + + A[β][α][1][I[α]+(α!=β)] * u[β][I-e(β)+e(α)] + uβα2 = A[β][α][2][I[α]] * u[β][I] + A[β][α][1][I[α]+1] * u[β][I+e(α)] # # Half - # uαβ1 = (u[α][I-δ(β)] + u[α][I]) / 2 - # uβα1 = u[β][I-δ(β)] / 2 + u[β][I-δ(β)+δ(α)] / 2 - # uαβ2 = (u[α][I] + u[α][I+δ(β)]) / 2 - # uβα2 = u[β][I] / 2 + u[β][I+δ(α)] / 2 + # uαβ1 = (u[α][I-e(β)] + u[α][I]) / 2 + # uβα1 = u[β][I-e(β)] / 2 + u[β][I-e(β)+e(α)] / 2 + # uαβ2 = (u[α][I] + u[α][I+e(β)]) / 2 + # uβα2 = u[β][I] / 2 + u[β][I+e(α)] / 2 # # Interpolation - # uαβ1 = A[α][β][2][I[β]-1] * u[α][I-δ(β)] + A[α][β][1][I[β]] * u[α][I] + # uαβ1 = A[α][β][2][I[β]-1] * u[α][I-e(β)] + A[α][β][1][I[β]] * u[α][I] # uβα1 = - # A[β][α][2][I[α]-(α==β)] * u[β][I-δ(β)] + - # A[β][α][1][I[α]+(α!=β)] * u[β][I-δ(β)+δ(α)] - # uαβ2 = A[α][β][2][I[β]] * u[α][I] + A[α][β][1][I[β]+1] * u[α][I+δ(β)] - # uβα2 = A[β][α][2][I[α]] * u[β][I] + A[β][α][1][I[α]+1] * u[β][I+δ(α)] + # A[β][α][2][I[α]-(α==β)] * u[β][I-e(β)] + + # A[β][α][1][I[α]+(α!=β)] * u[β][I-e(β)+e(α)] + # uαβ2 = A[α][β][2][I[β]] * u[α][I] + A[α][β][1][I[β]+1] * u[α][I+e(β)] + # uβα2 = A[β][α][2][I[α]] * u[β][I] + A[β][α][1][I[α]+1] * u[β][I+e(α)] F[α][I] -= (uαβ2 * uβα2 - uαβ1 * uβα1) / Δuαβ[I[β]] end @@ -451,7 +451,7 @@ function convection_adjoint!(ubar, φbar, u, setup) (; grid, workgroupsize) = setup (; dimension, Δ, Δu, N, Iu, A) = grid D = dimension() - δ = Offset{D}() + e = Offset{D}() T = eltype(u[1]) h = T(1) / 2 @kernel function adj!(ubar, φbar, u, ::Val{γ}, ::Val{looprange}) where {γ,looprange} @@ -466,16 +466,16 @@ function convection_adjoint!(ubar, φbar, u, setup) I = J if α == γ && I in Iu[α] uαβ2 = h - uβα2 = Aβα2[I[α]] * u[β][I] + Aβα1[I[α]+1] * u[β][I+δ(α)] + uβα2 = Aβα2[I[α]] * u[β][I] + Aβα1[I[α]+1] * u[β][I+e(α)] dφdu = -uαβ2 * uβα2 / Δuαβ[I[β]] ubar[γ][J] += φbar[α][I] * dφdu end # 2 - I = J - δ(β) + I = J - e(β) if α == γ && I in Iu[α] uαβ2 = h - uβα2 = Aβα2[I[α]] * u[β][I] + Aβα1[I[α]+1] * u[β][I+δ(α)] + uβα2 = Aβα2[I[α]] * u[β][I] + Aβα1[I[α]+1] * u[β][I+e(α)] dφdu = -uαβ2 * uβα2 / Δuαβ[I[β]] ubar[γ][J] += φbar[α][I] * dφdu end @@ -483,28 +483,28 @@ function convection_adjoint!(ubar, φbar, u, setup) # 3 I = J if β == γ && I in Iu[α] - uαβ2 = h * u[α][I] + h * u[α][I+δ(β)] + uαβ2 = h * u[α][I] + h * u[α][I+e(β)] uβα2 = Aβα2[I[α]] dφdu = -uαβ2 * uβα2 / Δuαβ[I[β]] ubar[γ][J] += φbar[α][I] * dφdu end # 4 - I = J - δ(α) + I = J - e(α) if β == γ && I in Iu[α] - uαβ2 = h * u[α][I] + h * u[α][I+δ(β)] + uαβ2 = h * u[α][I] + h * u[α][I+e(β)] uβα2 = Aβα1[I[α]+1] dφdu = -uαβ2 * uβα2 / Δuαβ[I[β]] ubar[γ][J] += φbar[α][I] * dφdu end # 5 - I = J + δ(β) + I = J + e(β) if α == γ && I in Iu[α] uαβ1 = h uβα1 = - Aβα2[I[α]-(α==β)] * u[β][I-δ(β)] + - Aβα1[I[α]+(α!=β)] * u[β][I-δ(β)+δ(α)] + Aβα2[I[α]-(α==β)] * u[β][I-e(β)] + + Aβα1[I[α]+(α!=β)] * u[β][I-e(β)+e(α)] dφdu = uαβ1 * uβα1 / Δuαβ[I[β]] ubar[γ][J] += φbar[α][I] * dφdu end @@ -514,25 +514,25 @@ function convection_adjoint!(ubar, φbar, u, setup) if α == γ && I in Iu[α] uαβ1 = h uβα1 = - Aβα2[I[α]-(α==β)] * u[β][I-δ(β)] + - Aβα1[I[α]+(α!=β)] * u[β][I-δ(β)+δ(α)] + Aβα2[I[α]-(α==β)] * u[β][I-e(β)] + + Aβα1[I[α]+(α!=β)] * u[β][I-e(β)+e(α)] dφdu = uαβ1 * uβα1 / Δuαβ[I[β]] ubar[γ][J] += φbar[α][I] * dφdu end # 7 - I = J + δ(β) + I = J + e(β) if β == γ && I in Iu[α] - uαβ1 = h * u[α][I-δ(β)] + h * u[α][I] + uαβ1 = h * u[α][I-e(β)] + h * u[α][I] uβα1 = Aβα2[I[α]-(α==β)] dφdu = uαβ1 * uβα1 / Δuαβ[I[β]] ubar[γ][J] += φbar[α][I] * dφdu end # 8 - I = J + δ(β) - δ(α) + I = J + e(β) - e(α) if β == γ && I in Iu[α] - uαβ1 = h * u[α][I-δ(β)] + h * u[α][I] + uαβ1 = h * u[α][I-e(β)] + h * u[α][I] uβα1 = Aβα1[I[α]+(α!=β)] dφdu = uαβ1 * uβα1 / Δuαβ[I[β]] ubar[γ][J] += φbar[α][I] * dφdu @@ -567,7 +567,7 @@ function diffusion!(F, u, setup) (; grid, workgroupsize, Re) = setup (; dimension, Δ, Δu, Nu, Iu) = grid D = dimension() - δ = Offset{D}() + e = Offset{D}() ν = 1 / Re @kernel function diff!(F, u, ::Val{α}, ::Val{βrange}, I0) where {α,βrange} I = @index(Global, Cartesian) @@ -577,8 +577,8 @@ function diffusion!(F, u, setup) Δuαβ = (α == β ? Δu[β] : Δ[β]) F[α][I] += ν * ( - (u[α][I+δ(β)] - u[α][I]) / (β == α ? Δ[β][I[β]+1] : Δu[β][I[β]]) - - (u[α][I] - u[α][I-δ(β)]) / (β == α ? Δ[β][I[β]] : Δu[β][I[β]-1]) + (u[α][I+e(β)] - u[α][I]) / (β == α ? Δ[β][I[β]+1] : Δu[β][I[β]]) - + (u[α][I] - u[α][I-e(β)]) / (β == α ? Δ[β][I[β]] : Δu[β][I[β]-1]) ) / Δuαβ[I[β]] end end @@ -594,20 +594,20 @@ function diffusion_adjoint!(u, φ, setup) (; grid, workgroupsize, Re) = setup (; dimension, N, Δ, Δu, Iu) = grid D = dimension() - δ = Offset{D}() + e = Offset{D}() ν = 1 / Re @kernel function adj!(u, φ, ::Val{α}, ::Val{βrange}) where {α,βrange} I = @index(Global, Cartesian) # for β = 1:D KernelAbstractions.Extras.LoopInfo.@unroll for β in βrange Δuαβ = (α == β ? Δu[β] : Δ[β]) - # F[α][I] += ν * u[α][I+δ(β)] / (β == α ? Δ[β][I[β]+1] : Δu[β][I[β]]) + # F[α][I] += ν * u[α][I+e(β)] / (β == α ? Δ[β][I[β]+1] : Δu[β][I[β]]) # F[α][I] -= ν * u[α][I] / (β == α ? Δ[β][I[β]+1] : Δu[β][I[β]]) # F[α][I] -= ν * u[α][I] / (β == α ? Δ[β][I[β]] : Δu[β][I[β]-1]) - # F[α][I] += ν * u[α][I-δ(β)] / (β == α ? Δ[β][I[β]] : Δu[β][I[β]-1]) - I - δ(β) ∈ Iu[α] && ( + # F[α][I] += ν * u[α][I-e(β)] / (β == α ? Δ[β][I[β]] : Δu[β][I[β]-1]) + I - e(β) ∈ Iu[α] && ( u[α][I] += - ν * φ[α][I-δ(β)] / (β == α ? Δ[β][I[β]] : Δu[β][I[β]-1]) / Δuαβ[I[β]-1] + ν * φ[α][I-e(β)] / (β == α ? Δ[β][I[β]] : Δu[β][I[β]-1]) / Δuαβ[I[β]-1] ) I ∈ Iu[α] && ( u[α][I] -= @@ -617,9 +617,9 @@ function diffusion_adjoint!(u, φ, setup) u[α][I] -= ν * φ[α][I] / (β == α ? Δ[β][I[β]] : Δu[β][I[β]-1]) / Δuαβ[I[β]] ) - I + δ(β) ∈ Iu[α] && ( + I + e(β) ∈ Iu[α] && ( u[α][I] += - ν * φ[α][I+δ(β)] / (β == α ? Δ[β][I[β]+1] : Δu[β][I[β]]) / Δuαβ[I[β]+1] + ν * φ[α][I+e(β)] / (β == α ? Δ[β][I[β]+1] : Δu[β][I[β]]) / Δuαβ[I[β]+1] ) end end @@ -644,7 +644,7 @@ function convectiondiffusion!(F, u, setup) (; grid, workgroupsize, Re) = setup (; dimension, Δ, Δu, Nu, Iu, A) = grid D = dimension() - δ = Offset{D}() + e = Offset{D}() ν = 1 / Re @kernel function cd!(F, u, ::Val{α}, ::Val{βrange}, I0) where {α,βrange} I = @index(Global, Cartesian) @@ -652,16 +652,16 @@ function convectiondiffusion!(F, u, setup) # for β = 1:D KernelAbstractions.Extras.LoopInfo.@unroll for β in βrange Δuαβ = α == β ? Δu[β] : Δ[β] - uαβ1 = (u[α][I-δ(β)] + u[α][I]) / 2 - uαβ2 = (u[α][I] + u[α][I+δ(β)]) / 2 + uαβ1 = (u[α][I-e(β)] + u[α][I]) / 2 + uαβ2 = (u[α][I] + u[α][I+e(β)]) / 2 uβα1 = - A[β][α][2][I[α]-(α==β)] * u[β][I-δ(β)] + - A[β][α][1][I[α]+(α!=β)] * u[β][I-δ(β)+δ(α)] - uβα2 = A[β][α][2][I[α]] * u[β][I] + A[β][α][1][I[α]+1] * u[β][I+δ(α)] + A[β][α][2][I[α]-(α==β)] * u[β][I-e(β)] + + A[β][α][1][I[α]+(α!=β)] * u[β][I-e(β)+e(α)] + uβα2 = A[β][α][2][I[α]] * u[β][I] + A[β][α][1][I[α]+1] * u[β][I+e(α)] uαuβ1 = uαβ1 * uβα1 uαuβ2 = uαβ2 * uβα2 - ∂βuα1 = (u[α][I] - u[α][I-δ(β)]) / (β == α ? Δ[β][I[β]] : Δu[β][I[β]-1]) - ∂βuα2 = (u[α][I+δ(β)] - u[α][I]) / (β == α ? Δ[β][I[β]+1] : Δu[β][I[β]]) + ∂βuα1 = (u[α][I] - u[α][I-e(β)]) / (β == α ? Δ[β][I[β]] : Δu[β][I[β]-1]) + ∂βuα2 = (u[α][I+e(β)] - u[α][I]) / (β == α ? Δ[β][I[β]+1] : Δu[β][I[β]]) F[α][I] += (ν * (∂βuα2 - ∂βuα1) - (uαuβ2 - uαuβ1)) / Δuαβ[I[β]] end end @@ -691,7 +691,7 @@ function bodyforce!(F, u, t, setup) (; dimension, Δ, Δu, Nu, Iu, x, xp) = grid isnothing(bodyforce) && return F D = dimension() - δ = Offset{D}() + e = Offset{D}() @assert D == 2 @kernel function f!(F, ::Val{α}, t, I0) where {α} I = @index(Global, Cartesian) @@ -801,12 +801,12 @@ function vorticity!(::Dimension{2}, ω, u, setup) (; grid, workgroupsize) = setup (; dimension, Δu, N) = grid D = dimension() - δ = Offset{D}() + e = Offset{D}() @kernel function ω!(ω, u, I0) I = @index(Global, Cartesian) I = I + I0 ω[I] = - (u[2][I+δ(1)] - u[2][I]) / Δu[1][I[1]] - (u[1][I+δ(2)] - u[1][I]) / Δu[2][I[2]] + (u[2][I+e(1)] - u[2][I]) / Δu[1][I[1]] - (u[1][I+e(2)] - u[1][I]) / Δu[2][I[2]] end I0 = CartesianIndex(ntuple(Returns(1), D)) I0 -= oneunit(I0) @@ -818,7 +818,7 @@ function vorticity!(::Dimension{3}, ω, u, setup) (; grid, workgroupsize) = setup (; dimension, Δu, N) = grid D = dimension() - δ = Offset{D}() + e = Offset{D}() @kernel function ω!(ω, u, I0) T = eltype(ω) I = @index(Global, Cartesian) @@ -827,8 +827,8 @@ function vorticity!(::Dimension{3}, ω, u, setup) # α₊ = mod1(α + 1, D) # α₋ = mod1(α - 1, D) ω[α][I] = - (u[α₋][I+δ(α₊)] - u[α₋][I]) / Δu[α₊][I[α₊]] - - (u[α₊][I+δ(α₋)] - u[α₊][I]) / Δu[α₋][I[α₋]] + (u[α₋][I+e(α₊)] - u[α₋][I]) / Δu[α₊][I[α₊]] - + (u[α₊][I+e(α₋)] - u[α₊][I]) / Δu[α₋][I[α₋]] end end I0 = CartesianIndex(ntuple(Returns(1), D)) @@ -837,13 +837,13 @@ function vorticity!(::Dimension{3}, ω, u, setup) ω end -@inline ∂x(uα, I::CartesianIndex{D}, α, β, Δβ, Δuβ; δ = Offset{D}()) where {D} = - α == β ? (uα[I] - uα[I-δ(β)]) / Δβ[I[β]] : +@inline ∂x(uα, I::CartesianIndex{D}, α, β, Δβ, Δuβ; e = Offset{D}()) where {D} = + α == β ? (uα[I] - uα[I-e(β)]) / Δβ[I[β]] : ( - (uα[I+δ(β)] - uα[I]) / Δuβ[I[β]] + - (uα[I-δ(α)+δ(β)] - uα[I-δ(α)]) / Δuβ[I[β]] + - (uα[I] - uα[I-δ(β)]) / Δuβ[I[β]-1] + - (uα[I-δ(α)] - uα[I-δ(α)-δ(β)]) / Δuβ[I[β]-1] + (uα[I+e(β)] - uα[I]) / Δuβ[I[β]] + + (uα[I-e(α)+e(β)] - uα[I-e(α)]) / Δuβ[I[β]] + + (uα[I] - uα[I-e(β)]) / Δuβ[I[β]-1] + + (uα[I-e(α)] - uα[I-e(α)-e(β)]) / Δuβ[I[β]-1] ) / 4 @inline ∇(u, I::CartesianIndex{2}, Δ, Δu) = @SMatrix [∂x(u[α], I, α, β, Δ[β], Δu[β]) for α = 1:2, β = 1:2] @@ -894,7 +894,7 @@ function divoftensor!(s, σ, setup) (; grid, workgroupsize) = setup (; dimension, Nu, Iu, Δ, Δu, A) = grid D = dimension() - δ = Offset{D}() + e = Offset{D}() @kernel function s!(s, σ, ::Val{α}, ::Val{βrange}, I0) where {α,βrange} I = @index(Global, Cartesian) I = I + I0 @@ -903,23 +903,23 @@ function divoftensor!(s, σ, setup) KernelAbstractions.Extras.LoopInfo.@unroll for β in βrange Δuαβ = α == β ? Δu[β] : Δ[β] if α == β - σαβ2 = σ[I+δ(β)][α, β] + σαβ2 = σ[I+e(β)][α, β] σαβ1 = σ[I][α, β] else # TODO: Add interpolation weights for non-uniform case σαβ2 = ( σ[I][α, β] + - σ[I+δ(β)][α, β] + - σ[I+δ(α)+δ(β)][α, β] + - σ[I+δ(α)][α, β] + σ[I+e(β)][α, β] + + σ[I+e(α)+e(β)][α, β] + + σ[I+e(α)][α, β] ) / 4 σαβ1 = ( - σ[I-δ(β)][α, β] + + σ[I-e(β)][α, β] + σ[I][α, β] + - σ[I+δ(α)-δ(β)][α, β] + - σ[I+δ(α)][α, β] + σ[I+e(α)-e(β)][α, β] + + σ[I+e(α)][α, β] ) / 4 end s[α][I] += (σαβ2 - σαβ1) / Δuαβ[I[β]] @@ -1016,8 +1016,8 @@ function tensorbasis(u, setup) T = eltype(u[1]) D = setup.grid.dimension() tensorbasis!( - ntuple(k -> similar(u[1], SMatrix{D,D,T,D * D}, setup.grid.N), D == 2 ? 3 : 11), - ntuple(k -> similar(u[1], setup.grid.N), D == 2 ? 2 : 5), + ntuple(α -> similar(u[1], SMatrix{D,D,T,D * D}, setup.grid.N), D == 2 ? 3 : 11), + ntuple(α -> similar(u[1], setup.grid.N), D == 2 ? 2 : 5), u, setup, ) @@ -1040,11 +1040,11 @@ function interpolate_u_p!(up, u, setup) (; grid, workgroupsize) = setup (; dimension, Np, Ip) = grid D = dimension() - δ = Offset{D}() + e = Offset{D}() @kernel function int!(up, u, ::Val{α}, I0) where {α} I = @index(Global, Cartesian) I = I + I0 - up[α][I] = (u[α][I-δ(α)] + u[α][I]) / 2 + up[α][I] = (u[α][I-e(α)] + u[α][I]) / 2 end for α = 1:D I0 = first(Ip) @@ -1077,11 +1077,11 @@ function interpolate_ω_p!(::Dimension{2}, ωp, ω, setup) (; grid, workgroupsize) = setup (; dimension, Np, Ip) = grid D = dimension() - δ = Offset{D}() + e = Offset{D}() @kernel function int!(ωp, ω, I0) I = @index(Global, Cartesian) I = I + I0 - ωp[I] = (ω[I-δ(1)-δ(2)] + ω[I]) / 2 + ωp[I] = (ω[I-e(1)-e(2)] + ω[I]) / 2 end I0 = first(Ip) I0 -= oneunit(I0) @@ -1093,13 +1093,13 @@ function interpolate_ω_p!(::Dimension{3}, ωp, ω, setup) (; grid, workgroupsize) = setup (; dimension, Np, Ip) = grid D = dimension() - δ = Offset{D}() + e = Offset{D}() @kernel function int!(ωp, ω, ::Val{α}, I0) where {α} I = @index(Global, Cartesian) I = I + I0 α₊ = mod1(α + 1, D) α₋ = mod1(α - 1, D) - ωp[α][I] = (ω[α][I-δ(α₊)-δ(α₋)] + ω[α][I]) / 2 + ωp[α][I] = (ω[α][I-e(α₊)-e(α₋)] + ω[α][I]) / 2 end I0 = first(Ip) I0 -= oneunit(I0) @@ -1123,25 +1123,25 @@ function Dfield!(d, G, p, setup) (; dimension, Np, Ip, Δ) = grid T = eltype(p) D = dimension() - δ = Offset{D}() + e = Offset{D}() @kernel function D!(d, G, p, I0) I = @index(Global, Cartesian) I = I + I0 g = zero(eltype(p)) for α = 1:D - g += (G[α][I-δ(α)] + G[α][I])^2 + g += (G[α][I-e(α)] + G[α][I])^2 end lap = zero(eltype(p)) # for α = 1:D - # lap += (G[α][I] - G[α][I-δ(α)]) / Δ[α][I[α]] + # lap += (G[α][I] - G[α][I-e(α)]) / Δ[α][I[α]] # end if D == 2 - lap += (G[1][I] - G[1][I-δ(1)]) / Δ[1][I[1]] - lap += (G[2][I] - G[2][I-δ(2)]) / Δ[2][I[2]] + lap += (G[1][I] - G[1][I-e(1)]) / Δ[1][I[1]] + lap += (G[2][I] - G[2][I-e(2)]) / Δ[2][I[2]] elseif D == 3 - lap += (G[1][I] - G[1][I-δ(1)]) / Δ[1][I[1]] - lap += (G[2][I] - G[2][I-δ(2)]) / Δ[2][I[2]] - lap += (G[3][I] - G[3][I-δ(3)]) / Δ[3][I[3]] + lap += (G[1][I] - G[1][I-e(1)]) / Δ[1][I[1]] + lap += (G[2][I] - G[2][I-e(2)]) / Δ[2][I[2]] + lap += (G[3][I] - G[3][I-e(3)]) / Δ[3][I[3]] end lap = lap > 0 ? max(lap, ϵ) : min(lap, -ϵ) # lap = abs(lap) @@ -1180,14 +1180,14 @@ function Qfield!(Q, u, setup) (; grid, workgroupsize) = setup (; dimension, Np, Ip, Δ) = grid D = dimension() - δ = Offset{D}() + e = Offset{D}() @kernel function Q!(Q, u, I0) I = @index(Global, Cartesian) I = I + I0 q = zero(eltype(Q)) for α = 1:D, β = 1:D q -= - (u[α][I] - u[α][I-δ(β)]) / Δ[β][I[β]] * (u[β][I] - u[β][I-δ(α)]) / + (u[α][I] - u[α][I-e(β)]) / Δ[β][I[β]] * (u[β][I] - u[β][I-e(α)]) / Δ[α][I[α]] / 2 end Q[I] = q @@ -1239,9 +1239,9 @@ as proposed by Jeong and Hussain [Jeong1995](@cite). eig2field(u, setup) = eig2field!(similar(u[1], setup.grid.N), u, setup) """ - kinetic_energy!(e, u, setup; interpolate_first = false) + kinetic_energy!(k, u, setup; interpolate_first = false) -Compute kinetic energy field ``e`` (in-place version). +Compute kinetic energy field ``k`` (in-place version). If `interpolate_first` is true, it is given by ```math @@ -1256,36 +1256,36 @@ e_I = \\frac{1}{4} \\sum_\\alpha (u^\\alpha_{I + \\delta(\\alpha) / 2}^2 + u^\\a as in [Sanderse2023](@cite). """ -function kinetic_energy!(e, u, setup; interpolate_first = false) +function kinetic_energy!(k, u, setup; interpolate_first = false) (; grid, workgroupsize) = setup (; dimension, Np, Ip) = grid D = dimension() - δ = Offset{D}() - @kernel function efirst!(e, u, I0) + e = Offset{D}() + @kernel function efirst!(ke, u, I0) I = @index(Global, Cartesian) I = I + I0 - k = zero(eltype(e)) + k = zero(eltype(ke)) for α = 1:D - k += (u[α][I] + u[α][I-δ(α)])^2 + k += (u[α][I] + u[α][I-e(α)])^2 end k = k / 8 - e[I] = k + ke[I] = k end - @kernel function elast!(e, u, I0) + @kernel function elast!(ke, u, I0) I = @index(Global, Cartesian) I = I + I0 - k = zero(eltype(e)) + k = zero(eltype(ke)) for α = 1:D - k += u[α][I]^2 + u[α][I-δ(α)]^2 + k += u[α][I]^2 + u[α][I-e(α)]^2 end k = k / 4 - e[I] = k + ke[I] = k end - e! = interpolate_first ? efirst! : elast! + ke! = interpolate_first ? efirst! : elast! I0 = first(Ip) I0 -= oneunit(I0) - e!(get_backend(u[1]), workgroupsize)(e, u, I0; ndrange = Np) - e + ke!(get_backend(u[1]), workgroupsize)(k, u, I0; ndrange = Np) + k end """ @@ -1304,7 +1304,7 @@ volume centers and squared. """ function total_kinetic_energy(u, setup; kwargs...) (; Ω, Ip) = setup.grid - e = kinetic_energy(u, setup; kwargs...) - e .*= Ω - sum(e[Ip]) + k = kinetic_energy(u, setup; kwargs...) + k .*= Ω + sum(k[Ip]) end From 6f08a37aad862a28f5b7e3db97a08a53c4810dfb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Fri, 3 May 2024 16:19:27 +0200 Subject: [PATCH 345/379] Add load path --- test/runtests.jl | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/runtests.jl b/test/runtests.jl index 67bd9e019..6063a7962 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -1,3 +1,5 @@ +push!(LOAD_PATH, joinpath(@__DIR__, "..")) + using Aqua using CairoMakie using ChainRulesCore From 0a8494d6525bfafebcf9fae7348552fef7d1d1c2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Sat, 4 May 2024 07:19:45 +0200 Subject: [PATCH 346/379] Rename symbol --- src/boundary_conditions.jl | 54 +++++++++++++++++++------------------- 1 file changed, 27 insertions(+), 27 deletions(-) diff --git a/src/boundary_conditions.jl b/src/boundary_conditions.jl index fc2731c0e..78a870835 100644 --- a/src/boundary_conditions.jl +++ b/src/boundary_conditions.jl @@ -193,14 +193,14 @@ function apply_bc_u!(::PeriodicBC, u, β, t, setup; isright, kwargs...) (; grid, workgroupsize) = setup (; dimension, N) = grid D = dimension() - δ = Offset{D}() + e = Offset{D}() @kernel function _bc_a!(u, ::Val{α}, ::Val{β}) where {α,β} I = @index(Global, Cartesian) - u[α][I] = u[α][I+(N[β]-2)*δ(β)] + u[α][I] = u[α][I+(N[β]-2)*e(β)] end @kernel function _bc_b!(u, ::Val{α}, ::Val{β}) where {α,β} I = @index(Global, Cartesian) - u[α][I+(N[β]-1)*δ(β)] = u[α][I+δ(β)] + u[α][I+(N[β]-1)*e(β)] = u[α][I+e(β)] end ndrange = ntuple(γ -> γ == β ? 1 : N[γ], D) for α = 1:D @@ -217,16 +217,16 @@ function apply_bc_u_pullback!(::PeriodicBC, φbar, β, t, setup; isright, kwargs (; grid, workgroupsize) = setup (; dimension, N) = grid D = dimension() - δ = Offset{D}() + e = Offset{D}() @kernel function adj_a!(φ, ::Val{α}, ::Val{β}) where {α,β} I = @index(Global, Cartesian) - φ[α][I+(N[β]-2)*δ(β)] += φ[α][I] + φ[α][I+(N[β]-2)*e(β)] += φ[α][I] φ[α][I] = 0 end @kernel function adj_b!(φ, ::Val{α}, ::Val{β}) where {α,β} I = @index(Global, Cartesian) - φ[α][I+δ(β)] += φ[α][I+(N[β]-1)*δ(β)] - φ[α][I+(N[β]-1)*δ(β)] = 0 + φ[α][I+e(β)] += φ[α][I+(N[β]-1)*e(β)] + φ[α][I+(N[β]-1)*e(β)] = 0 end ndrange = ntuple(γ -> γ == β ? 1 : N[γ], D) for α = 1:D @@ -243,14 +243,14 @@ function apply_bc_p!(::PeriodicBC, p, β, t, setup; isright, kwargs...) (; grid, workgroupsize) = setup (; dimension, N) = grid D = dimension() - δ = Offset{D}() + e = Offset{D}() @kernel function _bc_a(p, ::Val{β}) where {β} I = @index(Global, Cartesian) - p[I] = p[I+(N[β]-2)*δ(β)] + p[I] = p[I+(N[β]-2)*e(β)] end @kernel function _bc_b(p, ::Val{β}) where {β} I = @index(Global, Cartesian) - p[I+(N[β]-1)*δ(β)] = p[I+δ(β)] + p[I+(N[β]-1)*e(β)] = p[I+e(β)] end ndrange = ntuple(γ -> γ == β ? 1 : N[γ], D) if isright @@ -265,16 +265,16 @@ function apply_bc_p_pullback!(::PeriodicBC, φbar, β, t, setup; isright, kwargs (; grid, workgroupsize) = setup (; dimension, N) = grid D = dimension() - δ = Offset{D}() + e = Offset{D}() @kernel function adj_a!(φ, ::Val{β}) where {β} I = @index(Global, Cartesian) - φ[I+(N[β]-2)*δ(β)] += φ[I] + φ[I+(N[β]-2)*e(β)] += φ[I] φ[I] = 0 end @kernel function adj_b!(φ, ::Val{β}) where {β} I = @index(Global, Cartesian) - φ[I+δ(β)] += φ[I+(N[β]-1)*δ(β)] - φ[I+(N[β]-1)*δ(β)] = 0 + φ[I+e(β)] += φ[I+(N[β]-1)*e(β)] + φ[I+(N[β]-1)*e(β)] = 0 end ndrange = ntuple(γ -> γ == β ? 1 : N[γ], D) if isright @@ -288,7 +288,7 @@ end function apply_bc_u!(bc::DirichletBC, u, β, t, setup; isright, dudt = false, kwargs...) (; dimension, x, xp, N) = setup.grid D = dimension() - δ = Offset{D}() + e = Offset{D}() # isnothing(bc.u) && return bcfunc = dudt ? bc.dudt : bc.u for α = 1:D @@ -320,13 +320,13 @@ end function apply_bc_p!(::DirichletBC, p, β, t, setup; isright, kwargs...) (; dimension, N) = setup.grid D = dimension() - δ = Offset{D}() + e = Offset{D}() if isright I = CartesianIndices(ntuple(γ -> γ == β ? (N[γ]:N[γ]) : (1:N[γ]), D)) - p[I] .= p[I.-δ(β)] + p[I] .= p[I.-e(β)] else I = CartesianIndices(ntuple(γ -> γ == β ? (1:1) : (1:N[γ]), D)) - p[I] .= p[I.+δ(β)] + p[I] .= p[I.+e(β)] end p end @@ -334,15 +334,15 @@ end function apply_bc_u!(::SymmetricBC, u, β, t, setup; isright, kwargs...) (; dimension, N) = setup.grid D = dimension() - δ = Offset{D}() + e = Offset{D}() for α = 1:D if α != β if isright I = CartesianIndices(ntuple(γ -> γ == β ? (N[γ]:N[γ]) : (1:N[γ]), D)) - u[α][I] .= u[α][I.-δ(β)] + u[α][I] .= u[α][I.-e(β)] else I = CartesianIndices(ntuple(γ -> γ == β ? (1:1) : (1:N[γ]), D)) - u[α][I] .= u[α][I.+δ(β)] + u[α][I] .= u[α][I.+e(β)] end end end @@ -352,13 +352,13 @@ end function apply_bc_p!(::SymmetricBC, p, β, t, setup; isright, kwargs...) (; dimension, N) = setup.grid D = dimension() - δ = Offset{D}() + e = Offset{D}() if isright I = CartesianIndices(ntuple(γ -> γ == β ? (N[γ]:N[γ]) : (1:N[γ]), D)) - p[I] .= p[I.-δ(β)] + p[I] .= p[I.-e(β)] else I = CartesianIndices(ntuple(γ -> γ == β ? (1:1) : (1:N[γ]), D)) - p[I] .= p[I.+δ(β)] + p[I] .= p[I.+e(β)] end p end @@ -367,16 +367,16 @@ function apply_bc_u!(bc::PressureBC, u, β, t, setup; isright, kwargs...) (; grid, workgroupsize) = setup (; dimension, N, Nu, Iu) = grid D = dimension() - δ = Offset{D}() + e = Offset{D}() @kernel function _bc_a!(u, ::Val{α}, ::Val{β}, I0) where {α,β} I = @index(Global, Cartesian) I = I + I0 - u[α][I] = u[α][I+δ(β)] + u[α][I] = u[α][I+e(β)] end @kernel function _bc_b!(u, ::Val{α}, ::Val{β}, I0) where {α,β} I = @index(Global, Cartesian) I = I + I0 - u[α][I] = u[α][I-δ(β)] + u[α][I] = u[α][I-e(β)] end ndrange = (N[1:β-1]..., 1, N[β+1:end]...) for α = 1:D From e4fc097cd6ea6858f39d281dd050f461a92b93d4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Sat, 4 May 2024 09:42:41 +0200 Subject: [PATCH 347/379] Add rrule tests --- src/boundary_conditions.jl | 48 ++++++++++++++++++++++++++++++++++---- test/chainrules.jl | 23 +++++++++++++++++- 2 files changed, 66 insertions(+), 5 deletions(-) diff --git a/src/boundary_conditions.jl b/src/boundary_conditions.jl index 78a870835..2683db87e 100644 --- a/src/boundary_conditions.jl +++ b/src/boundary_conditions.jl @@ -113,7 +113,7 @@ ChainRulesCore.rrule(::typeof(apply_bc_u), u, t, setup; kwargs...) = ( # With respect to (apply_bc_u, u, t, setup) φbar -> ( NoTangent(), - apply_bc_u_pullback!(copy.((φbar...,)), t, setup; kwargs...), + apply_bc_u_pullback!(Tangent{typeof(u)}(copy.((φbar...,))...), t, setup; kwargs...), NoTangent(), NoTangent(), ), @@ -124,7 +124,13 @@ ChainRulesCore.rrule(::typeof(apply_bc_p), p, t, setup) = ( # With respect to (apply_bc_p, p, t, setup) φbar -> ( NoTangent(), - apply_bc_p_pullback!(copy(φbar), t, setup), + apply_bc_p_pullback!( + # copy(φbar), + copy(unthunk(φbar)), + # Tangent{typeof{p}}(copy((φbar...,))), + t, + setup, + ), NoTangent(), NoTangent(), ), @@ -183,8 +189,24 @@ function apply_bc_p_pullback!(φbar, t, setup; kwargs...) (; dimension) = grid D = dimension() for β = 1:D - apply_bc_p_pullback!(boundary_conditions[β][1], φbar, β, t, setup; isright = false) - apply_bc_p_pullback!(boundary_conditions[β][2], φbar, β, t, setup; isright = true) + apply_bc_p_pullback!( + boundary_conditions[β][1], + φbar, + β, + t, + setup; + isright = false, + kwargs..., + ) + apply_bc_p_pullback!( + boundary_conditions[β][2], + φbar, + β, + t, + setup; + isright = true, + kwargs..., + ) end φbar end @@ -317,6 +339,9 @@ function apply_bc_u!(bc::DirichletBC, u, β, t, setup; isright, dudt = false, kw u end +# apply_bc_u_pullback!(::DirichletBC, φbar, β, t, setup; isright, kwargs...) = +# @not_implemented("DirichletBC pullback not yet implemented.") + function apply_bc_p!(::DirichletBC, p, β, t, setup; isright, kwargs...) (; dimension, N) = setup.grid D = dimension() @@ -331,6 +356,9 @@ function apply_bc_p!(::DirichletBC, p, β, t, setup; isright, kwargs...) p end +# apply_bc_p_pullback!(::DirichletBC, φbar, β, t, setup; isright, kwargs...) = +# @not_implemented("DirichletBC pullback not yet implemented.") + function apply_bc_u!(::SymmetricBC, u, β, t, setup; isright, kwargs...) (; dimension, N) = setup.grid D = dimension() @@ -349,6 +377,9 @@ function apply_bc_u!(::SymmetricBC, u, β, t, setup; isright, kwargs...) u end +# apply_bc_u_pullback!(::SymmetricBC, φbar, β, t, setup; isright, kwargs...) = +# @not_implemented("SymmetricBC pullback not yet implemented.") + function apply_bc_p!(::SymmetricBC, p, β, t, setup; isright, kwargs...) (; dimension, N) = setup.grid D = dimension() @@ -363,6 +394,9 @@ function apply_bc_p!(::SymmetricBC, p, β, t, setup; isright, kwargs...) p end +# apply_bc_p_pullback!(::SymmetricBC, φbar, β, t, setup; isright, kwargs...) = +# @not_implemented("SymmetricBC pullback not yet implemented.") + function apply_bc_u!(bc::PressureBC, u, β, t, setup; isright, kwargs...) (; grid, workgroupsize) = setup (; dimension, N, Nu, Iu) = grid @@ -393,6 +427,9 @@ function apply_bc_u!(bc::PressureBC, u, β, t, setup; isright, kwargs...) u end +# apply_bc_u_pullback!(::PressureBC, φbar, β, t, setup; isright, kwargs...) = +# @not_implemented("PressureBC pullback not yet implemented.") + function apply_bc_p!(bc::PressureBC, p, β, t, setup; isright, kwargs...) (; dimension, N) = setup.grid D = dimension() @@ -404,3 +441,6 @@ function apply_bc_p!(bc::PressureBC, p, β, t, setup; isright, kwargs...) p[I] .= 0 p end + +apply_bc_p_pullback!(::PressureBC, φbar, β, t, setup; isright, kwargs...) = + @not_implemented("PressureBC pullback not yet implemented.") diff --git a/test/chainrules.jl b/test/chainrules.jl index 3b5f6f356..d08cb4f19 100644 --- a/test/chainrules.jl +++ b/test/chainrules.jl @@ -1,3 +1,21 @@ +# @testset "Chain rules boundary conditions" begin +# T = Float64 +# Re = T(1_000) +# n = 8 +# boundary_conditions = ((DirichletBC(), PressureBC()), (PeriodicBC(), PeriodicBC())) +# lims = T(0), T(1) +# x = stretched_grid(lims..., n, 1.2), cosine_grid(lims..., n) +# setup = Setup(x...; Re, +# boundary_conditions, +# ) +# psolver = DirectPressureSolver(setup) +# u = random_field(setup, T(0); psolver) +# randn!.(u) +# p = randn!(similar(u[1])) +# test_rrule(apply_bc_u, u, T(0) ⊢ NoTangent(), setup ⊢ NoTangent()) +# test_rrule(apply_bc_p, p, T(0) ⊢ NoTangent(), setup ⊢ NoTangent()) +# end; + testchainrules(dim) = @testset "Chain rules $(dim())D" begin # Setup D = dim() @@ -21,9 +39,12 @@ testchainrules(dim) = @testset "Chain rules $(dim())D" begin setup = Setup(x...; Re) psolver = DirectPressureSolver(setup) u = random_field(setup, T(0); psolver) - (; Iu, Ip, Ω) = setup.grid randn!.(u) p = randn!(similar(u[1])) + @testset "Boundary conditions" begin + test_rrule(apply_bc_u, u, T(0) ⊢ NoTangent(), setup ⊢ NoTangent()) + test_rrule(apply_bc_p, p, T(0) ⊢ NoTangent(), setup ⊢ NoTangent()) + end @testset "Divergence" begin test_rrule(divergence, u, setup ⊢ NoTangent()) end From e8c4dd6a34e35e3a21858af37480d1766043552f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Sat, 4 May 2024 09:49:31 +0200 Subject: [PATCH 348/379] Add rrule test for Poisson --- src/boundary_conditions.jl | 6 ++++-- src/solvers/pressure/poisson.jl | 2 +- test/chainrules.jl | 4 ++-- test/runtests.jl | 9 ++++++++- 4 files changed, 15 insertions(+), 6 deletions(-) diff --git a/src/boundary_conditions.jl b/src/boundary_conditions.jl index 2683db87e..42e06850f 100644 --- a/src/boundary_conditions.jl +++ b/src/boundary_conditions.jl @@ -113,6 +113,8 @@ ChainRulesCore.rrule(::typeof(apply_bc_u), u, t, setup; kwargs...) = ( # With respect to (apply_bc_u, u, t, setup) φbar -> ( NoTangent(), + # Important: identity operator should be part of `apply_bc_u_pullback`, + # but is actually implemented via the `copy` below instead. apply_bc_u_pullback!(Tangent{typeof(u)}(copy.((φbar...,))...), t, setup; kwargs...), NoTangent(), NoTangent(), @@ -125,9 +127,9 @@ ChainRulesCore.rrule(::typeof(apply_bc_p), p, t, setup) = ( φbar -> ( NoTangent(), apply_bc_p_pullback!( - # copy(φbar), + # Important: identity operator should be part of `apply_bc_p_pullback`, + # but is actually implemented via the `copy` below instead. copy(unthunk(φbar)), - # Tangent{typeof{p}}(copy((φbar...,))), t, setup, ), diff --git a/src/solvers/pressure/poisson.jl b/src/solvers/pressure/poisson.jl index d661eaeb4..7759ccfdb 100644 --- a/src/solvers/pressure/poisson.jl +++ b/src/solvers/pressure/poisson.jl @@ -12,7 +12,7 @@ poisson(solver, f) = poisson!(solver, zero(f), f) # Laplacian is auto-adjoint ChainRulesCore.rrule(::typeof(poisson), solver, f) = - (poisson(solver, f), φ -> (NoTangent(), NoTangent(), poisson(solver, φ))) + (poisson(solver, f), φ -> (NoTangent(), NoTangent(), poisson(solver, unthunk(φ)))) """ poisson!(solver, p, f) diff --git a/test/chainrules.jl b/test/chainrules.jl index d08cb4f19..4c6341f0d 100644 --- a/test/chainrules.jl +++ b/test/chainrules.jl @@ -51,8 +51,8 @@ testchainrules(dim) = @testset "Chain rules $(dim())D" begin @testset "Pressure gradient" begin test_rrule(pressuregradient, p, setup ⊢ NoTangent()) end - @testset "Pressure gradient" begin - test_rrule(pressuregradient, p, setup ⊢ NoTangent()) + @testset "Poisson" begin + test_rrule(poisson, psolver ⊢ NoTangent(), p) end @testset "Convection" begin test_rrule(convection, u, setup ⊢ NoTangent()) diff --git a/test/runtests.jl b/test/runtests.jl index 6063a7962..6fcdb7a00 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -6,7 +6,14 @@ using ChainRulesCore using ChainRulesTestUtils using IncompressibleNavierStokes using IncompressibleNavierStokes: - divergence, pressuregradient, convection, diffusion, bodyforce, apply_bc_u, apply_bc_p + divergence, + pressuregradient, + convection, + diffusion, + bodyforce, + poisson, + apply_bc_u, + apply_bc_p using LinearAlgebra using Random using Statistics From d7761dc275ddab041cfd183c289dc08b9ec389eb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Sat, 4 May 2024 09:51:17 +0200 Subject: [PATCH 349/379] Add comment --- test/chainrules.jl | 1 + 1 file changed, 1 insertion(+) diff --git a/test/chainrules.jl b/test/chainrules.jl index 4c6341f0d..7997b031e 100644 --- a/test/chainrules.jl +++ b/test/chainrules.jl @@ -16,6 +16,7 @@ # test_rrule(apply_bc_p, p, T(0) ⊢ NoTangent(), setup ⊢ NoTangent()) # end; +# Test chain rule correctness by comparing with finite differences testchainrules(dim) = @testset "Chain rules $(dim())D" begin # Setup D = dim() From 08a78378003e03977696edededb78341cd537f61 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Sat, 4 May 2024 10:21:50 +0200 Subject: [PATCH 350/379] Move filters to NeuralClosure --- docs/src/features/closure.md | 4 ++-- libs/NeuralClosure/src/NeuralClosure.jl | 2 ++ {src => libs/NeuralClosure/src}/filter.jl | 8 +++----- src/IncompressibleNavierStokes.jl | 4 ---- 4 files changed, 7 insertions(+), 11 deletions(-) rename {src => libs/NeuralClosure/src}/filter.jl (95%) diff --git a/docs/src/features/closure.md b/docs/src/features/closure.md index 55033c0eb..07446134c 100644 --- a/docs/src/features/closure.md +++ b/docs/src/features/closure.md @@ -35,8 +35,8 @@ M \bar{v} & = 0, \\ ``` ```@docs -IncompressibleNavierStokes.FaceAverage -IncompressibleNavierStokes.VolumeAverage +NeuralClosure.FaceAverage +NeuralClosure.VolumeAverage ``` ```@docs diff --git a/libs/NeuralClosure/src/NeuralClosure.jl b/libs/NeuralClosure/src/NeuralClosure.jl index 048cc9622..4518e8351 100644 --- a/libs/NeuralClosure/src/NeuralClosure.jl +++ b/libs/NeuralClosure/src/NeuralClosure.jl @@ -18,6 +18,7 @@ include("closure.jl") include("cnn.jl") include("fno.jl") include("training.jl") +include("filter.jl") include("create_les_data.jl") export smagorinsky_closure @@ -28,5 +29,6 @@ export create_loss_prior, create_loss_post export create_dataloader_prior, create_dataloader_post export create_callback, create_les_data, create_io_arrays export wrappedclosure +export FaceAverage, VolumeAverage, reconstruct, reconstruct! end diff --git a/src/filter.jl b/libs/NeuralClosure/src/filter.jl similarity index 95% rename from src/filter.jl rename to libs/NeuralClosure/src/filter.jl index a95fa1ce3..dfb1c4943 100644 --- a/src/filter.jl +++ b/libs/NeuralClosure/src/filter.jl @@ -19,7 +19,6 @@ function (::FaceAverage)(v, u, setup_les, comp) (; grid, workgroupsize) = setup_les (; Nu, Iu) = grid D = length(u) - δ = Offset{D}() @kernel function Φ!(v, u, ::Val{α}, face, I0) where {α} I = @index(Global, Cartesian) J = I0 + comp * (I - oneunit(I)) @@ -48,14 +47,14 @@ function reconstruct!(u, v, setup_dns, setup_les, comp) (; grid, boundary_conditions, workgroupsize) = setup_les (; N, Iu) = grid D = length(u) - δ = Offset{D}() + e = Offset{D}() @assert all(bc -> bc[1] isa PeriodicBC && bc[2] isa PeriodicBC, boundary_conditions) @kernel function R!(u, v, ::Val{α}, volume) where {α} J = @index(Global, Cartesian) I = oneunit(J) + comp * J J = oneunit(J) + J - Jleft = J - δ(α) - Jleft.I[α] == 1 && (Jleft += (N[α] - 2) * δ(α)) + Jleft = J - e(α) + Jleft.I[α] == 1 && (Jleft += (N[α] - 2) * e(α)) for i in volume s = zero(eltype(v[α])) s += (comp - i.I[α]) * v[α][J] @@ -88,7 +87,6 @@ function (::VolumeAverage)(v, u, setup_les, comp) (; grid, boundary_conditions, workgroupsize) = setup_les (; N, Nu, Iu) = grid D = length(u) - δ = Offset{D}() @assert all(bc -> bc[1] isa PeriodicBC && bc[2] isa PeriodicBC, boundary_conditions) @kernel function Φ!(v, u, ::Val{α}, volume, I0) where {α} I = @index(Global, Cartesian) diff --git a/src/IncompressibleNavierStokes.jl b/src/IncompressibleNavierStokes.jl index ced36027f..5fb532347 100644 --- a/src/IncompressibleNavierStokes.jl +++ b/src/IncompressibleNavierStokes.jl @@ -67,7 +67,6 @@ include("processors/animator.jl") # Discrete operators include("operators.jl") -include("filter.jl") # Solvers include("solvers/get_timestep.jl") @@ -112,9 +111,6 @@ export create_initial_conditions, random_field export plotgrid, save_vtk export plotmat -# Filter -export FaceAverage, VolumeAverage, reconstruct, reconstruct! - # ODE methods export AdamsBashforthCrankNicolsonMethod, OneLegMethod, RKMethods From be3d7133193ccf11f74851be95fcd2bbaaedd4ab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Sat, 4 May 2024 10:37:38 +0200 Subject: [PATCH 351/379] Update README --- README.md | 28 ++++++++++++++++++---------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index b41a3bb3b..04dba8ea1 100644 --- a/README.md +++ b/README.md @@ -31,6 +31,11 @@ See the for examples of some typical workflows. More examples can be found in the [`examples`](examples) directory. +## Source code for paper + +See [here](./libs/PaperDC) for the source code used in the paper +[Discretize first, filter next: learning divergence-consistent closure models for large-eddy simulation](https://arxiv.org/abs/2403.18088). + ## Gallery The velocity and pressure fields may be visualized in a live session using @@ -52,8 +57,16 @@ The following example code using a negative body force on a small rectangle with an unsteady inflow. It simulates a wind turbine (actuator) under varying wind conditions. +Make sure to have the `GLMakie` and `IncompressibleNavierStokes` installed: + +```julia +using Pkg +Pkg.add(["GLMakie", "IncompressibleNavierStokes"]) +``` + +Then run run the following code to make a short animation: + ```julia -# using Pkg; Pkg.add(["GLMakie", "IncompressibleNavierStokes"])) using GLMakie using IncompressibleNavierStokes @@ -62,7 +75,7 @@ n = 40 x = LinRange(0.0, 10.0, 5n + 1) y = LinRange(-2.0, 2.0, 2n + 1) -# Boundary conditions: Unsteady BC requires time derivatives +# Boundary conditions boundary_conditions = ( # Inlet, outlet ( @@ -81,14 +94,9 @@ boundary_conditions = ( (PressureBC(), PressureBC()), ) -# Actuator body force: A thrust coefficient `Cₜ` distributed over a thin rectangle -xc, yc = 2.0, 0.0 # Disk center -D = 1.0 # Disk diameter -δ = 0.11 # Disk thickness -Cₜ = 0.2 # Thrust coefficient -cₜ = Cₜ / (D * δ) -inside(x, y) = abs(x - xc) ≤ δ / 2 && abs(y - yc) ≤ D / 2 -bodyforce(dim, x, y, t) = dim() == 1 ? -cₜ * inside(x, y) : 0.0 +# Actuator body force: A thrust coefficient distributed over a thin rectangle +inside(x, y) = abs(x - 2.0) ≤ 0.055 && abs(y) ≤ 0.5 +bodyforce(dim, x, y, t) = dim() == 1 && inside(x, y) ? -1.82 : 0.0 # Build setup and assemble operators setup = Setup(x, y; Re = 100.0, boundary_conditions, bodyforce); From 4a110bda34b98cddcf35ecd50d8f60391c19c442 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Sat, 4 May 2024 10:43:39 +0200 Subject: [PATCH 352/379] Add missing dependency --- libs/NeuralClosure/Project.toml | 2 ++ libs/NeuralClosure/src/NeuralClosure.jl | 1 + 2 files changed, 3 insertions(+) diff --git a/libs/NeuralClosure/Project.toml b/libs/NeuralClosure/Project.toml index 82e8e9c58..138073932 100644 --- a/libs/NeuralClosure/Project.toml +++ b/libs/NeuralClosure/Project.toml @@ -7,6 +7,7 @@ version = "0.1.0" CUDA = "052768ef-5323-5732-b1bb-66c8b64840ba" ComponentArrays = "b0b7db55-cfe3-40fc-9ded-d10e2dbeff66" IncompressibleNavierStokes = "5e318141-6589-402b-868d-77d7df8c442e" +KernelAbstractions = "63c18a36-062a-441e-b654-da1e3ab1ce7c" Lux = "b2108857-7c20-44ae-9111-449ecde12c47" NNlib = "872c559c-99b0-510c-b3b7-b6c96a88d5cd" Observables = "510215fc-4207-5dde-b226-833fc4488ee2" @@ -18,6 +19,7 @@ Zygote = "e88e6eb3-aa80-5325-afca-941959d7151f" [compat] CUDA = "5" ComponentArrays = "0.15" +KernelAbstractions = "0.9" Lux = "0.5" NNlib = "0.9" Observables = "0.5" diff --git a/libs/NeuralClosure/src/NeuralClosure.jl b/libs/NeuralClosure/src/NeuralClosure.jl index 4518e8351..9b9084cc6 100644 --- a/libs/NeuralClosure/src/NeuralClosure.jl +++ b/libs/NeuralClosure/src/NeuralClosure.jl @@ -7,6 +7,7 @@ using CUDA using ComponentArrays: ComponentArray using IncompressibleNavierStokes using IncompressibleNavierStokes: Dimension, momentum!, apply_bc_u!, project! +using KernelAbstractions using Lux using NNlib using Observables From a91fa29208c8e7bbdbe9f5523841a44fe8889d15 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Sat, 4 May 2024 14:53:48 +0200 Subject: [PATCH 353/379] docs: Fix reference --- docs/src/features/closure.md | 1 + docs/src/features/operators.md | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/src/features/closure.md b/docs/src/features/closure.md index 07446134c..115edce44 100644 --- a/docs/src/features/closure.md +++ b/docs/src/features/closure.md @@ -37,6 +37,7 @@ M \bar{v} & = 0, \\ ```@docs NeuralClosure.FaceAverage NeuralClosure.VolumeAverage +NeuralClosure.reconstruct! ``` ```@docs diff --git a/docs/src/features/operators.md b/docs/src/features/operators.md index ae9845a84..54ac99408 100644 --- a/docs/src/features/operators.md +++ b/docs/src/features/operators.md @@ -46,5 +46,4 @@ total_kinetic_energy tensorbasis divoftensor! tensorbasis! -reconstruct! ``` From 13ce4e4fe985b85fe51b8acdc47ee6740719badc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Sun, 5 May 2024 13:58:55 +0200 Subject: [PATCH 354/379] Add tests --- src/operators.jl | 22 ++++++++++++---------- test/Project.toml | 1 + test/operators.jl | 44 ++++++++++++++++++++++++++++++++++++++++++++ test/runtests.jl | 28 +++++++++++++++++++++++----- 4 files changed, 80 insertions(+), 15 deletions(-) diff --git a/src/operators.jl b/src/operators.jl index 230b3a31b..f4684eb07 100644 --- a/src/operators.jl +++ b/src/operators.jl @@ -965,6 +965,7 @@ Note that `B[1]` corresponds to ``T_0`` in the paper, and `V` to ``I``. function tensorbasis!(B, V, u, setup) (; grid, workgroupsize) = setup (; Np, Ip, Δ, Δu, dimension) = grid + D = dimension() @kernel function basis2!(B, V, u, I0) I = @index(Global, Cartesian) I = I + I0 @@ -1003,7 +1004,7 @@ function tensorbasis!(B, V, u, setup) I0 = first(Ip) I0 -= oneunit(I0) basis! = D == 2 ? basis2! : basis3! - basis!(get_backend(u[1]), workgroupsize)(dimension, B, V, u, I0; ndrange = Np) + basis!(get_backend(u[1]), workgroupsize)(B, V, u, I0; ndrange = Np) B, V end @@ -1118,7 +1119,7 @@ Compute the ``D``-field [LiJiajia2019](@cite) given by D = \\frac{2 | \\nabla p |}{\\nabla^2 p}. ``` """ -function Dfield!(d, G, p, setup) +function Dfield!(d, G, p, setup; ϵ = eps(eltype(p))) (; grid, workgroupsize) = setup (; dimension, Np, Ip, Δ) = grid T = eltype(p) @@ -1155,15 +1156,16 @@ function Dfield!(d, G, p, setup) end """ - Dfield(p, setup) + Dfield(p, setup; kwargs...) Compute the ``D``-field. """ -Dfield(p, setup) = Dfield!( +Dfield(p, setup; kwargs...) = Dfield!( zero(p), ntuple(α -> similar(p, setup.grid.N), setup.grid.dimension()), p, - setup, + setup; + kwargs..., ) """ @@ -1222,6 +1224,7 @@ function eig2field!(λ, u, setup) ∇u = ∇(u, I, Δ, Δu) S = @. (∇u + ∇u') / 2 Ω = @. (∇u - ∇u') / 2 + # FIXME: Is not recognized as hermitian with Float64 on CPU λ[I] = eigvals(S^2 + Ω^2)[2] end I0 = first(Ip) @@ -1256,7 +1259,7 @@ e_I = \\frac{1}{4} \\sum_\\alpha (u^\\alpha_{I + \\delta(\\alpha) / 2}^2 + u^\\a as in [Sanderse2023](@cite). """ -function kinetic_energy!(k, u, setup; interpolate_first = false) +function kinetic_energy!(ke, u, setup; interpolate_first = false) (; grid, workgroupsize) = setup (; dimension, Np, Ip) = grid D = dimension() @@ -1284,8 +1287,8 @@ function kinetic_energy!(k, u, setup; interpolate_first = false) ke! = interpolate_first ? efirst! : elast! I0 = first(Ip) I0 -= oneunit(I0) - ke!(get_backend(u[1]), workgroupsize)(k, u, I0; ndrange = Np) - k + ke!(get_backend(u[1]), workgroupsize)(ke, u, I0; ndrange = Np) + ke end """ @@ -1293,8 +1296,7 @@ end Compute kinetic energy field ``e`` (out-of-place version). """ -kinetic_energy(u, setup; kwargs...) = - kinetic_energy!(similar(u[1], setup.grid.N), u, setup; kwargs...) +kinetic_energy(u, setup; kwargs...) = kinetic_energy!(similar(u[1]), u, setup; kwargs...) """ total_kinetic_energy(setup, u; kwargs...) diff --git a/test/Project.toml b/test/Project.toml index eebe741f6..47bb34b07 100644 --- a/test/Project.toml +++ b/test/Project.toml @@ -5,5 +5,6 @@ ChainRulesCore = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4" ChainRulesTestUtils = "cdddcdb0-9152-4a09-a978-84456f9df70a" LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" +SparseArrays = "2f01184e-e22b-5df5-ae63-d93ebab69eaf" Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2" Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" diff --git a/test/operators.jl b/test/operators.jl index e080df633..b32a8a47c 100644 --- a/test/operators.jl +++ b/test/operators.jl @@ -50,6 +50,17 @@ testops(dim) = @testset "Operators $(dim())D" begin @test pDv ≈ -vGp # Check that D = -G' end + @testset "Laplacian" begin + p = randn!(similar(u[1])) + p = apply_bc_p(p, T(0), setup) + Lp = laplacian(p, setup) + @test Lp isa Array{T} + @test sum((p.*Ω.*Lp)[Ip]) ≤ 0 # Check negativity + L = laplacian_mat(setup) + @test L isa SparseMatrixCSC + @test norm((Lp)[Ip][:] - (L * p[Ip][:])) ≈ 0 atol = 1e-12 + end + @testset "Convection" begin c = convection(u, setup) uCu = if D == 2 @@ -95,6 +106,39 @@ testops(dim) = @testset "Operators $(dim())D" begin @test d[1] isa Array{T} @test uDu ≤ 0 # Check negativity (dissipation) end + + @testset "Convection-Diffusion" begin + cd = convectiondiffusion!(zero.(u), u, setup) + c_and_d = zero.(u) + convection!(c_and_d, u, setup) + diffusion!(c_and_d, u, setup) + @test all(cd .≈ c_and_d) + end + + @testset "Momentum" begin + m = momentum(u, T(1), setup) + @test m isa Tuple + @test m[1] isa Array{T} + @test all(all.(!isnan, m)) + end + + @testset "Other fields" begin + p = randn!(similar(u[1])) + ω = vorticity(u, setup) + D == 2 && @test ω isa Array{T} + D == 3 && @test ω isa Tuple + @test smagorinsky_closure(setup)(u, 0.1) isa Tuple + @test tensorbasis(u, setup) isa Tuple + @test interpolate_u_p(u, setup) isa Tuple + D == 2 && @test interpolate_ω_p(ω, setup) isa Array{T} + D == 3 && @test interpolate_ω_p(ω, setup) isa Tuple + @test Dfield(p, setup) isa Array{T} + @test Qfield(u, setup) isa Array{T} + D == 2 && @test_throws AssertionError eig2field(u, setup) + D == 3 && @test eig2field(u, setup) isa Array{T} broken = D == 3 + @test kinetic_energy(u, setup) isa Array{T} + @test total_kinetic_energy(u, setup) isa T + end end testops(IncompressibleNavierStokes.Dimension(2)) diff --git a/test/runtests.jl b/test/runtests.jl index 6fcdb7a00..dfad363ce 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -6,16 +6,34 @@ using ChainRulesCore using ChainRulesTestUtils using IncompressibleNavierStokes using IncompressibleNavierStokes: - divergence, - pressuregradient, + apply_bc_p, + apply_bc_u, + bodyforce, convection, + convection!, + convectiondiffusion!, diffusion, - bodyforce, + diffusion!, + divergence, + eig2field, + kinetic_energy, + interpolate_u_p, + interpolate_ω_p, + laplacian, + laplacian_mat, + momentum, poisson, - apply_bc_u, - apply_bc_p + pressuregradient, + smagorinsky_closure, + tensorbasis, + total_kinetic_energy, + vorticity, + Dfield, + Qfield + using LinearAlgebra using Random +using SparseArrays using Statistics using Test From 4d91755324322d4acb94aaed9e2a0b580a32e798 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Sun, 5 May 2024 13:59:17 +0200 Subject: [PATCH 355/379] Simplify docs environment --- docs/make.jl | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/docs/make.jl b/docs/make.jl index 18c929896..96f5de836 100644 --- a/docs/make.jl +++ b/docs/make.jl @@ -1,16 +1,13 @@ # Show number of threads on GitHub Actions @info "" Threads.nthreads() -# Make paths relative to this file -cd(@__DIR__) - # Build docs environment using Pkg -Pkg.activate(".") +Pkg.activate(@__DIR__) Pkg.develop([ - PackageSpec(; path = ".."), - PackageSpec(; path = "../libs/NeuralClosure"), - PackageSpec(; path = "../examples"), + PackageSpec(; path = joinpath(@__DIR__, "..")), + PackageSpec(; path = joinpath(@__DIR__, "..", "libs", "NeuralClosure")), + PackageSpec(; path = joinpath(@__DIR__, "..", "examples")), ]) Pkg.instantiate() From a5bc3632a1b3046594298d78fef80ddadaa64708 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Sun, 5 May 2024 14:18:42 +0200 Subject: [PATCH 356/379] Move files --- README.md | 2 +- docs/make.jl | 2 +- {libs => lib}/NeuralClosure/LICENSE | 0 {libs => lib}/NeuralClosure/Project.toml | 0 {libs => lib}/NeuralClosure/README.md | 0 {libs => lib}/NeuralClosure/src/NeuralClosure.jl | 0 {libs => lib}/NeuralClosure/src/closure.jl | 0 {libs => lib}/NeuralClosure/src/cnn.jl | 0 {libs => lib}/NeuralClosure/src/create_les_data.jl | 0 {libs => lib}/NeuralClosure/src/filter.jl | 0 {libs => lib}/NeuralClosure/src/fno.jl | 0 {libs => lib}/NeuralClosure/src/training.jl | 0 {libs => lib}/NeuralClosure/test/Project.toml | 0 {libs => lib}/NeuralClosure/test/runtests.jl | 0 {libs => lib}/PaperDC/LICENSE | 0 {libs => lib}/PaperDC/Project.toml | 0 {libs => lib}/PaperDC/README.md | 0 {libs => lib}/PaperDC/postanalysis.jl | 0 {libs => lib}/PaperDC/prioranalysis.jl | 0 {libs => lib}/PaperDC/src/PaperDC.jl | 0 {libs => lib}/PaperDC/src/observe.jl | 0 {libs => lib}/PaperDC/src/rk.jl | 0 {libs => lib}/TensorClosure/Project.toml | 0 {libs => lib}/TensorClosure/README.md | 0 {libs => lib}/TensorClosure/main.jl | 0 25 files changed, 2 insertions(+), 2 deletions(-) rename {libs => lib}/NeuralClosure/LICENSE (100%) rename {libs => lib}/NeuralClosure/Project.toml (100%) rename {libs => lib}/NeuralClosure/README.md (100%) rename {libs => lib}/NeuralClosure/src/NeuralClosure.jl (100%) rename {libs => lib}/NeuralClosure/src/closure.jl (100%) rename {libs => lib}/NeuralClosure/src/cnn.jl (100%) rename {libs => lib}/NeuralClosure/src/create_les_data.jl (100%) rename {libs => lib}/NeuralClosure/src/filter.jl (100%) rename {libs => lib}/NeuralClosure/src/fno.jl (100%) rename {libs => lib}/NeuralClosure/src/training.jl (100%) rename {libs => lib}/NeuralClosure/test/Project.toml (100%) rename {libs => lib}/NeuralClosure/test/runtests.jl (100%) rename {libs => lib}/PaperDC/LICENSE (100%) rename {libs => lib}/PaperDC/Project.toml (100%) rename {libs => lib}/PaperDC/README.md (100%) rename {libs => lib}/PaperDC/postanalysis.jl (100%) rename {libs => lib}/PaperDC/prioranalysis.jl (100%) rename {libs => lib}/PaperDC/src/PaperDC.jl (100%) rename {libs => lib}/PaperDC/src/observe.jl (100%) rename {libs => lib}/PaperDC/src/rk.jl (100%) rename {libs => lib}/TensorClosure/Project.toml (100%) rename {libs => lib}/TensorClosure/README.md (100%) rename {libs => lib}/TensorClosure/main.jl (100%) diff --git a/README.md b/README.md index 04dba8ea1..3036bacc1 100644 --- a/README.md +++ b/README.md @@ -33,7 +33,7 @@ for examples of some typical workflows. More examples can be found in the ## Source code for paper -See [here](./libs/PaperDC) for the source code used in the paper +See [here](./lib/PaperDC) for the source code used in the paper [Discretize first, filter next: learning divergence-consistent closure models for large-eddy simulation](https://arxiv.org/abs/2403.18088). ## Gallery diff --git a/docs/make.jl b/docs/make.jl index 96f5de836..358f5d4c9 100644 --- a/docs/make.jl +++ b/docs/make.jl @@ -6,7 +6,7 @@ using Pkg Pkg.activate(@__DIR__) Pkg.develop([ PackageSpec(; path = joinpath(@__DIR__, "..")), - PackageSpec(; path = joinpath(@__DIR__, "..", "libs", "NeuralClosure")), + PackageSpec(; path = joinpath(@__DIR__, "..", "lib", "NeuralClosure")), PackageSpec(; path = joinpath(@__DIR__, "..", "examples")), ]) Pkg.instantiate() diff --git a/libs/NeuralClosure/LICENSE b/lib/NeuralClosure/LICENSE similarity index 100% rename from libs/NeuralClosure/LICENSE rename to lib/NeuralClosure/LICENSE diff --git a/libs/NeuralClosure/Project.toml b/lib/NeuralClosure/Project.toml similarity index 100% rename from libs/NeuralClosure/Project.toml rename to lib/NeuralClosure/Project.toml diff --git a/libs/NeuralClosure/README.md b/lib/NeuralClosure/README.md similarity index 100% rename from libs/NeuralClosure/README.md rename to lib/NeuralClosure/README.md diff --git a/libs/NeuralClosure/src/NeuralClosure.jl b/lib/NeuralClosure/src/NeuralClosure.jl similarity index 100% rename from libs/NeuralClosure/src/NeuralClosure.jl rename to lib/NeuralClosure/src/NeuralClosure.jl diff --git a/libs/NeuralClosure/src/closure.jl b/lib/NeuralClosure/src/closure.jl similarity index 100% rename from libs/NeuralClosure/src/closure.jl rename to lib/NeuralClosure/src/closure.jl diff --git a/libs/NeuralClosure/src/cnn.jl b/lib/NeuralClosure/src/cnn.jl similarity index 100% rename from libs/NeuralClosure/src/cnn.jl rename to lib/NeuralClosure/src/cnn.jl diff --git a/libs/NeuralClosure/src/create_les_data.jl b/lib/NeuralClosure/src/create_les_data.jl similarity index 100% rename from libs/NeuralClosure/src/create_les_data.jl rename to lib/NeuralClosure/src/create_les_data.jl diff --git a/libs/NeuralClosure/src/filter.jl b/lib/NeuralClosure/src/filter.jl similarity index 100% rename from libs/NeuralClosure/src/filter.jl rename to lib/NeuralClosure/src/filter.jl diff --git a/libs/NeuralClosure/src/fno.jl b/lib/NeuralClosure/src/fno.jl similarity index 100% rename from libs/NeuralClosure/src/fno.jl rename to lib/NeuralClosure/src/fno.jl diff --git a/libs/NeuralClosure/src/training.jl b/lib/NeuralClosure/src/training.jl similarity index 100% rename from libs/NeuralClosure/src/training.jl rename to lib/NeuralClosure/src/training.jl diff --git a/libs/NeuralClosure/test/Project.toml b/lib/NeuralClosure/test/Project.toml similarity index 100% rename from libs/NeuralClosure/test/Project.toml rename to lib/NeuralClosure/test/Project.toml diff --git a/libs/NeuralClosure/test/runtests.jl b/lib/NeuralClosure/test/runtests.jl similarity index 100% rename from libs/NeuralClosure/test/runtests.jl rename to lib/NeuralClosure/test/runtests.jl diff --git a/libs/PaperDC/LICENSE b/lib/PaperDC/LICENSE similarity index 100% rename from libs/PaperDC/LICENSE rename to lib/PaperDC/LICENSE diff --git a/libs/PaperDC/Project.toml b/lib/PaperDC/Project.toml similarity index 100% rename from libs/PaperDC/Project.toml rename to lib/PaperDC/Project.toml diff --git a/libs/PaperDC/README.md b/lib/PaperDC/README.md similarity index 100% rename from libs/PaperDC/README.md rename to lib/PaperDC/README.md diff --git a/libs/PaperDC/postanalysis.jl b/lib/PaperDC/postanalysis.jl similarity index 100% rename from libs/PaperDC/postanalysis.jl rename to lib/PaperDC/postanalysis.jl diff --git a/libs/PaperDC/prioranalysis.jl b/lib/PaperDC/prioranalysis.jl similarity index 100% rename from libs/PaperDC/prioranalysis.jl rename to lib/PaperDC/prioranalysis.jl diff --git a/libs/PaperDC/src/PaperDC.jl b/lib/PaperDC/src/PaperDC.jl similarity index 100% rename from libs/PaperDC/src/PaperDC.jl rename to lib/PaperDC/src/PaperDC.jl diff --git a/libs/PaperDC/src/observe.jl b/lib/PaperDC/src/observe.jl similarity index 100% rename from libs/PaperDC/src/observe.jl rename to lib/PaperDC/src/observe.jl diff --git a/libs/PaperDC/src/rk.jl b/lib/PaperDC/src/rk.jl similarity index 100% rename from libs/PaperDC/src/rk.jl rename to lib/PaperDC/src/rk.jl diff --git a/libs/TensorClosure/Project.toml b/lib/TensorClosure/Project.toml similarity index 100% rename from libs/TensorClosure/Project.toml rename to lib/TensorClosure/Project.toml diff --git a/libs/TensorClosure/README.md b/lib/TensorClosure/README.md similarity index 100% rename from libs/TensorClosure/README.md rename to lib/TensorClosure/README.md diff --git a/libs/TensorClosure/main.jl b/lib/TensorClosure/main.jl similarity index 100% rename from libs/TensorClosure/main.jl rename to lib/TensorClosure/main.jl From d794ca7202a912f651e601efb3ed5c9328066551 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Sun, 5 May 2024 18:42:26 +0200 Subject: [PATCH 357/379] Put CUDA specific solver in extension --- Project.toml | 11 ++- README.md | 2 +- docs/src/getting_started.md | 2 +- examples/Project.toml | 1 - lib/NeuralClosure/src/NeuralClosure.jl | 4 +- src/IncompressibleNavierStokes.jl | 5 -- src/create_initial_conditions.jl | 7 +- src/solvers/pressure/poisson.jl | 44 +--------- src/solvers/pressure/solvers.jl | 106 +++---------------------- 9 files changed, 29 insertions(+), 153 deletions(-) diff --git a/Project.toml b/Project.toml index 1d76fa150..34ee0bc2a 100644 --- a/Project.toml +++ b/Project.toml @@ -5,8 +5,6 @@ version = "0.4.1" [deps] Adapt = "79e6a3ab-5dfb-504d-930d-738a2a938a0e" -CUDA = "052768ef-5323-5732-b1bb-66c8b64840ba" -CUDSS = "45b445bb-4962-46a0-9369-b4df9d0f772e" ChainRulesCore = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4" FFTW = "7a1cc6ca-52ef-59f5-83cd-3a7055c09341" IterativeSolvers = "42fd0dbc-a981-5370-80f2-aaf504508153" @@ -22,6 +20,13 @@ StaticArrays = "90137ffa-7385-5640-81b9-e52037218182" Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2" WriteVTK = "64499a7a-5c06-52f2-abe2-ccb03c286192" +[weakdeps] +CUDA = "052768ef-5323-5732-b1bb-66c8b64840ba" +CUDSS = "45b445bb-4962-46a0-9369-b4df9d0f772e" + +[extensions] +IncompressibleNavierStokesCUDAExt = ["CUDA", "CUDSS"] + [compat] Adapt = "3, 4" CairoMakie = "0.11" @@ -42,7 +47,7 @@ SparseArrays = "1" StaticArrays = "1" Statistics = "1" WriteVTK = "1" -julia = "1.7" +julia = "1.9" [extras] CairoMakie = "13f3f980-e62b-5c42-98c6-ff1f3baf88f0" diff --git a/README.md b/README.md index 3036bacc1..d5d8ff2e1 100644 --- a/README.md +++ b/README.md @@ -24,7 +24,7 @@ into Pkg-mode, and type: ``` which will install the package and all dependencies to your local environment. -Note that IncompressibleNavierStokes requires Julia version `1.7` or above. +Note that IncompressibleNavierStokes requires Julia version `1.9` or above. See the [Documentation](https://agdestein.github.io/IncompressibleNavierStokes.jl/dev/generated/LidDrivenCavity2D/) diff --git a/docs/src/getting_started.md b/docs/src/getting_started.md index eac7cc05d..d005edf7a 100644 --- a/docs/src/getting_started.md +++ b/docs/src/getting_started.md @@ -8,4 +8,4 @@ add IncompressibleNavierStokes ``` which will install the package and all dependencies to your local environment. -Note that IncompressibleNavierStokes requires Julia version `1.7` or above. +Note that IncompressibleNavierStokes requires Julia version `1.9` or above. diff --git a/examples/Project.toml b/examples/Project.toml index 477423491..b5cec717d 100644 --- a/examples/Project.toml +++ b/examples/Project.toml @@ -4,7 +4,6 @@ authors = ["Syver Døving Agdestein "] version = "0.1.0" [deps] -CUDA = "052768ef-5323-5732-b1bb-66c8b64840ba" CairoMakie = "13f3f980-e62b-5c42-98c6-ff1f3baf88f0" GLMakie = "e9467ef8-e4e7-5192-8a1a-b1aee30e663a" IncompressibleNavierStokes = "5e318141-6589-402b-868d-77d7df8c442e" diff --git a/lib/NeuralClosure/src/NeuralClosure.jl b/lib/NeuralClosure/src/NeuralClosure.jl index 9b9084cc6..18ae25a3f 100644 --- a/lib/NeuralClosure/src/NeuralClosure.jl +++ b/lib/NeuralClosure/src/NeuralClosure.jl @@ -3,7 +3,6 @@ Neural closure modelling tools. """ module NeuralClosure -using CUDA using ComponentArrays: ComponentArray using IncompressibleNavierStokes using IncompressibleNavierStokes: Dimension, momentum!, apply_bc_u!, project! @@ -15,6 +14,9 @@ using Random using Tullio using Zygote +# Must be loaded inside for Tullio to work correctly +using CUDA + include("closure.jl") include("cnn.jl") include("fno.jl") diff --git a/src/IncompressibleNavierStokes.jl b/src/IncompressibleNavierStokes.jl index 5fb532347..45ba12a2c 100644 --- a/src/IncompressibleNavierStokes.jl +++ b/src/IncompressibleNavierStokes.jl @@ -20,11 +20,6 @@ using StaticArrays using Statistics using WriteVTK: CollectionFile, paraview_collection, vtk_grid, vtk_save -# Must be loaded inside for Tullio to work correctly -using CUDA -using CUDA.CUSPARSE -# using CUSOLVERRF -using CUDSS # # Easily retrieve value from Val # (::Val{x})() where {x} = x diff --git a/src/create_initial_conditions.jl b/src/create_initial_conditions.jl index 2c770d995..c06fae679 100644 --- a/src/create_initial_conditions.jl +++ b/src/create_initial_conditions.jl @@ -146,14 +146,13 @@ function create_spectrum(; setup, kp, rng = Random.default_rng()) # Remove non-divergence free part: (I - k k^T / k^2) e ke = sum(α -> e[α] .* kkkk[α], 1:D) - CUDA.@allowscalar e0 = getindex.(e, 1) for α = 1:D + e0 = e[1:1] # CUDA doesn't like e[1] @. e[α] -= kkkk[α] * ke / knorm^2 + # Restore k=0 component, which is divergence free anyways + e[1:1] .= e0 end - # Restore k=0 component, which is divergence free anyways - CUDA.@allowscalar setindex!.(e, e0, 1) - # Normalize enorm = sqrt.(sum(α -> e[α] .^ 2, 1:D)) for α = 1:D diff --git a/src/solvers/pressure/poisson.jl b/src/solvers/pressure/poisson.jl index 7759ccfdb..77a1352c7 100644 --- a/src/solvers/pressure/poisson.jl +++ b/src/solvers/pressure/poisson.jl @@ -34,43 +34,6 @@ function poisson!(solver::DirectPressureSolver, p, f) # solver.f .= view(view(f, Ip), :) # copyto!(solver.f, view(view(f, Ip), :)) pp = view(view(p, Ip), :) - if false # p isa CuArray - ldiv!(solver.p, fact, solver.f) - elseif false # p isa CuArray - copyto!(view(solver.f, 1:length(solver.f)-1), view(view(f, Ip), :)) - F = fact - a, b = solver.f, solver.p - # solver.p .= F.Q * (F.U \ (F.L \ (F.P * (F.Rs .* solver.f)))) - # copyto!(pp, view(solver.p, 1:length(solver.p)-1)) - a .*= F.Rs - mul!(b, F.P, a) - ldiv!(a, F.L, b) - ldiv!(b, F.U, a) - mul!(a, F.Q, b) - copyto!(pp, view(a, 1:length(a)-1)) - else - if any(bc -> bc[1] isa PressureBC || bc[2] isa PressureBC, boundary_conditions) - # No extra DOF - viewrange = (:) - else - # With extra DOF - viewrange = 1:length(solver.p)-1 - end - copyto!(view(solver.f, viewrange), Array(view(view(f, Ip), :))) - solver.p .= fact \ solver.f - copyto!(pp, T.(view(solver.p, viewrange))) - end - # @infiltrate - # ldiv!(solver.p, fact, solver.f) - # pp .= solver.p - # copyto!(pp, solver.p) - p -end - -function poisson!(solver::CUDSSPressureSolver, p, f) - (; setup) = solver - (; Ip) = setup.grid - T = eltype(p) if any(bc -> bc[1] isa PressureBC || bc[2] isa PressureBC, boundary_conditions) # No extra DOF viewrange = (:) @@ -78,10 +41,9 @@ function poisson!(solver::CUDSSPressureSolver, p, f) # With extra DOF viewrange = 1:length(solver.p)-1 end - pp = view(view(p, Ip), :) - copyto!(view(solver.f, viewrange), view(view(f, Ip), :)) - cudss("solve", solver.solver, solver.p, solver.f) - copyto!(pp, view(solver.p, viewrange)) + copyto!(view(solver.f, viewrange), Array(view(view(f, Ip), :))) + solver.p .= fact \ solver.f + copyto!(pp, T.(view(solver.p, viewrange))) p end diff --git a/src/solvers/pressure/solvers.jl b/src/solvers/pressure/solvers.jl index 0d3865176..e34ccfa5b 100644 --- a/src/solvers/pressure/solvers.jl +++ b/src/solvers/pressure/solvers.jl @@ -18,106 +18,20 @@ struct DirectPressureSolver{T,S,F,A} <: AbstractPressureSolver{T} function DirectPressureSolver(setup) (; grid, boundary_conditions, ArrayType) = setup (; x, Np) = grid - if false #ArrayType == CuArray - # T = eltype(x[1]) - T = Float64 - f = similar(x[1], T, prod(Np) + 1) - p = similar(x[1], T, prod(Np) + 1) - L = laplacian_mat(setup) - e = ones(T, size(L, 2)) - L = [L e; e' 0] - L = L |> CuSparseMatrixCSR{Float64} - fact = CUSOLVERRF.RFLU(L; symbolic = :RF) - elseif false #ArrayType == CuArray - T = eltype(x[1]) - f = similar(x[1], prod(Np) + 1) - p = similar(x[1], prod(Np) + 1) - L = laplacian_mat(setup) - e = ones(Float64, size(L, 2)) - L = [L e; e' 0] - F = lu(L) - P = sparse(1:length(F.p), F.p, ones(length(F.p))) - Q = sparse(F.q, 1:length(F.q), ones(length(F.q))) - L = F.L |> CuSparseMatrixCSR{T} - U = F.U |> CuSparseMatrixCSR{T} - CUDA.@allowscalar L = L |> LowerTriangular - CUDA.@allowscalar U = U |> UpperTriangular - # CUDA.@allowscalar L = F.L |> CuSparseMatrixCSR{Float64} |> LowerTriangular - # CUDA.@allowscalar U = F.U |> CuSparseMatrixCSR{Float64} |> UpperTriangular - fact = (; - L, - U, - Rs = F.Rs |> CuArray{T}, - P = P |> CuSparseMatrixCSR{T}, - Q = Q |> CuSparseMatrixCSR{T}, - ) - else - # T = eltype(x[1]) - T = Float64 - backend = get_backend(x[1]) - L = laplacian_mat(setup) - if any(bc -> bc[1] isa PressureBC || bc[2] isa PressureBC, boundary_conditions) - f = zeros(T, prod(Np)) - p = zeros(T, prod(Np)) - else - f = zeros(T, prod(Np) + 1) - p = zeros(T, prod(Np) + 1) - e = ones(T, size(L, 2)) - L = [L e; e' 0] - end - # fact = lu(L) - # fact = ldlt(Symmetric(L)) - fact = factorize(L) - end - new{T,typeof(setup),typeof(fact),typeof(f)}(setup, fact, f, p) - end -end - -# This moves all the inner arrays to the GPU when calling -# `cu(::SpectralPressureSolver)` from CUDA.jl -# TODO: CUDA does not support `factorize`, `lu`, etc, but maybe `L` and `U` can be -# converted to `CuArray` after factorization on the CPU -Adapt.adapt_structure(to, s::DirectPressureSolver) = error( - "`DirectPressureSolver` is not yet implemented for CUDA. Consider using `CGPressureSolver`.", -) - -""" - CUDSSPressureSolver() - -Direct pressure solver using a CUDSS LDLt decomposition. -""" -struct CUDSSPressureSolver{T,S,F,A} <: AbstractPressureSolver{T} - setup::S - solver::F - f::A - p::A - function CUDSSPressureSolver(setup) - (; grid, boundary_conditions, ArrayType) = setup - (; x, Np) = grid - T = eltype(x[1]) - @assert x[1] isa CuArray "CUDSSPressureSolver only works for CuArray." - f = fill!(similar(x[1], prod(Np) + 1), 0) - p = fill!(similar(x[1], prod(Np) + 1), 0) + # T = eltype(x[1]) + T = Float64 L = laplacian_mat(setup) if any(bc -> bc[1] isa PressureBC || bc[2] isa PressureBC, boundary_conditions) - # structure = "SPD" # Symmetric positive definite - structure = "G" # General matrix - view = 'F' # Full matrix representation + f = zeros(T, prod(Np)) + p = zeros(T, prod(Np)) else + f = zeros(T, prod(Np) + 1) + p = zeros(T, prod(Np) + 1) e = ones(T, size(L, 2)) L = [L e; e' 0] - structure = "S" # Symmetric (not positive definite) - view = 'L' # Lower triangular representation end - L = CuSparseMatrixCSR(L) - solver = CudssSolver( - L, - structure, - view, # Lower triangular representation - ) - cudss("analysis", solver, p, f) - cudss("factorization", solver, p, f) # Compute factorization - new{T,typeof(setup),typeof(solver),typeof(f)}(setup, solver, f, p) + fact = factorize(L) + new{T,typeof(setup),typeof(fact),typeof(f)}(setup, fact, f, p) end end @@ -171,8 +85,8 @@ function CGMatrixPressureSolver( maxiter = prod(setup.grid.Np), ) L = laplacian_mat(setup) |> setup.device - L = L |> CuSparseMatrixCSR - CGPressureSolver{eltype(L),typeof(L)}(L, abstol, reltol, maxiter) + # L = L |> CuSparseMatrixCSR + CGMatrixPressureSolver{eltype(L),typeof(L)}(L, abstol, reltol, maxiter) end """ From ecf577675493637c6a500f4d213c44048663b78c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Sun, 5 May 2024 21:55:22 +0200 Subject: [PATCH 358/379] refactor: Simplify psolvers --- docs/src/features/gpu.md | 2 +- docs/src/features/precision.md | 4 +- docs/src/features/pressure.md | 12 +- examples/Actuator2D.jl | 2 +- examples/BackwardFacingStep2D.jl | 2 +- examples/DecayingTurbulence2D.jl | 2 +- examples/DecayingTurbulence3D.jl | 2 +- examples/LidDrivenCavity2D.jl | 8 +- examples/MultiActuator.jl | 2 +- examples/PlanarMixing2D.jl | 2 +- examples/PlaneJets2D.jl | 2 +- examples/ShearLayer2D.jl | 2 +- examples/TaylorGreenVortex2D.jl | 2 +- examples/TaylorGreenVortex3D.jl | 2 +- lib/NeuralClosure/src/create_les_data.jl | 8 +- lib/PaperDC/postanalysis.jl | 14 +- lib/PaperDC/prioranalysis.jl | 6 +- lib/TensorClosure/main.jl | 2 +- src/IncompressibleNavierStokes.jl | 14 +- src/create_initial_conditions.jl | 8 +- src/processors/processors.jl | 2 +- src/processors/real_time_plot.jl | 6 +- src/solvers/pressure/poisson.jl | 203 ----------------- src/solvers/pressure/pressure.jl | 37 --- src/solvers/pressure/project.jl | 42 ---- src/solvers/pressure/solvers.jl | 279 ----------------------- src/solvers/solve_unsteady.jl | 4 +- src/utils/save_vtk.jl | 2 +- test/chainrules.jl | 4 +- test/operators.jl | 2 +- test/postprocess2D.jl | 2 +- test/postprocess3D.jl | 2 +- test/pressure_solvers.jl | 35 +-- test/solvers.jl | 2 +- 34 files changed, 68 insertions(+), 652 deletions(-) delete mode 100644 src/solvers/pressure/poisson.jl delete mode 100644 src/solvers/pressure/pressure.jl delete mode 100644 src/solvers/pressure/project.jl delete mode 100644 src/solvers/pressure/solvers.jl diff --git a/docs/src/features/gpu.md b/docs/src/features/gpu.md index ce7e98850..b30d26d77 100644 --- a/docs/src/features/gpu.md +++ b/docs/src/features/gpu.md @@ -8,7 +8,7 @@ Even if a GPU is not available, the operators are multithreaded if Julia is sta Limitations: -- [`DirectPressureSolver`](@ref) is currently used on the CPU with double precision. [`CGPressureSolver`](@ref) works on the GPU. +- [`psolver_direct`](@ref) is currently used on the CPU with double precision. [`psolver_cg`](@ref) works on the GPU. - This has not been tested with other GPU interfaces, such as - [AMDGPU.jl](https://github.com/JuliaGPU/AMDGPU.jl) - [Metal.jl](https://github.com/JuliaGPU/Metal.jl) diff --git a/docs/src/features/precision.md b/docs/src/features/precision.md index dd794ae08..f880f0ff2 100644 --- a/docs/src/features/precision.md +++ b/docs/src/features/precision.md @@ -19,6 +19,6 @@ an example on how to enforce floating point type hygiene. !!! note "Pressure solvers" [`SparseArrays.jl`](https://github.com/JuliaSparse/SparseArrays.jl)s sparse matrix factorizations only support double precision. - [`DirectPressureSolver`](@ref) only works for `Float64`. Consider - using an iterative solver such as [`CGPressureSolver`](@ref) + [`psolver_direct`](@ref) only works for `Float64`. Consider + using an iterative solver such as [`psolver_cg`](@ref) when using single or half precision. diff --git a/docs/src/features/pressure.md b/docs/src/features/pressure.md index 0f178e9d8..a8bb04c8d 100644 --- a/docs/src/features/pressure.md +++ b/docs/src/features/pressure.md @@ -12,13 +12,11 @@ enforces divergence freeness. There are multiple options for solving this system. ```@docs -AbstractPressureSolver -DirectPressureSolver -CUDSSPressureSolver -CGPressureSolver -CGMatrixPressureSolver -SpectralPressureSolver -LowMemorySpectralPressureSolver +psolver_direct +psolver_cg +psolver_cg_matrix +psolver_spectral +psolver_spectral_lowmemory pressure pressure! poisson diff --git a/examples/Actuator2D.jl b/examples/Actuator2D.jl index 66dc83881..01081e777 100644 --- a/examples/Actuator2D.jl +++ b/examples/Actuator2D.jl @@ -53,7 +53,7 @@ bodyforce(dim, x, y, t) = dim() == 1 ? -cₜ * inside(x, y) : 0.0 # Build setup and assemble operators setup = Setup(x, y; Re = 100.0, boundary_conditions, bodyforce); -psolver = DirectPressureSolver(setup); +psolver = psolver_direct(setup); # Initial conditions (extend inflow) u₀ = create_initial_conditions(setup, (dim, x, y) -> dim() == 1 ? 1.0 : 0.0; psolver); diff --git a/examples/BackwardFacingStep2D.jl b/examples/BackwardFacingStep2D.jl index 5a9c2ad2f..a977e1301 100644 --- a/examples/BackwardFacingStep2D.jl +++ b/examples/BackwardFacingStep2D.jl @@ -52,7 +52,7 @@ plotgrid(x, y) # Build setup and assemble operators setup = Setup(x, y; Re, boundary_conditions, ArrayType); -psolver = DirectPressureSolver(setup); +psolver = psolver_direct(setup); # Initial conditions (extend inflow) u₀ = create_initial_conditions(setup, (dim, x, y) -> U(dim, x, y, zero(x)); psolver); diff --git a/examples/DecayingTurbulence2D.jl b/examples/DecayingTurbulence2D.jl index 97f745514..d47bcfd8b 100644 --- a/examples/DecayingTurbulence2D.jl +++ b/examples/DecayingTurbulence2D.jl @@ -36,7 +36,7 @@ setup = Setup(x...; Re, ArrayType); # Since the grid is uniform and identical for x and y, we may use a specialized # spectral pressure solver -psolver = SpectralPressureSolver(setup); +psolver = psolver_spectral(setup); # Create random initial conditions u₀ = random_field(setup, T(0); psolver); diff --git a/examples/DecayingTurbulence3D.jl b/examples/DecayingTurbulence3D.jl index 4f30e115c..403e9118e 100644 --- a/examples/DecayingTurbulence3D.jl +++ b/examples/DecayingTurbulence3D.jl @@ -38,7 +38,7 @@ setup = Setup(x, y, z; Re, ArrayType); # Since the grid is uniform and identical for x, y, and z, we may use a # specialized spectral pressure solver -psolver = SpectralPressureSolver(setup); +psolver = psolver_spectral(setup); # Initial conditions u₀ = random_field(setup; psolver); diff --git a/examples/LidDrivenCavity2D.jl b/examples/LidDrivenCavity2D.jl index 755098dff..a09561b17 100644 --- a/examples/LidDrivenCavity2D.jl +++ b/examples/LidDrivenCavity2D.jl @@ -81,12 +81,12 @@ setup = Setup(x, y; boundary_conditions, Re, ArrayType); # The pressure solver is used to solve the pressure Poisson equation. # Available solvers are # -# - [`DirectPressureSolver`](@ref) (only for CPU with `Float64`) -# - [`CGPressureSolver`](@ref) -# - [`SpectralPressureSolver`](@ref) (only for periodic boundary conditions and +# - [`psolver_direct`](@ref) (only for CPU with `Float64`) +# - [`psolver_cg`](@ref) +# - [`psolver_spectral`](@ref) (only for periodic boundary conditions and # uniform grids) -psolver = DirectPressureSolver(setup); +psolver = psolver_direct(setup); # The initial conditions are provided in function. The value `dim()` determines # the velocity component. diff --git a/examples/MultiActuator.jl b/examples/MultiActuator.jl index cb437830c..5f58e811c 100644 --- a/examples/MultiActuator.jl +++ b/examples/MultiActuator.jl @@ -79,7 +79,7 @@ y = LinRange(-T(2), T(2), 2n + 1) # Build setup and assemble operators setup = Setup(x, y; Re = T(2000), boundary_conditions, bodyforce, ArrayType); -psolver = CUDSSPressureSolver(setup); +psolver = psolver_direct(setup); # Initial conditions (extend inflow) u₀ = create_initial_conditions(setup, (dim, x, y) -> dim() == 1 ? one(x) : zero(x); psolver); diff --git a/examples/PlanarMixing2D.jl b/examples/PlanarMixing2D.jl index 0098dfdf9..841945789 100644 --- a/examples/PlanarMixing2D.jl +++ b/examples/PlanarMixing2D.jl @@ -47,7 +47,7 @@ plotgrid(x, y) # Build setup and assemble operators setup = Setup(x, y; Re, boundary_conditions); -psolver = DirectPressureSolver(setup); +psolver = psolver_direct(setup); # Initial conditions (extend inflow) u₀ = create_initial_conditions(setup, (dim, x, y) -> U(dim, x, y, 0.0); psolver); diff --git a/examples/PlaneJets2D.jl b/examples/PlaneJets2D.jl index fe2c558c0..c1f8ae088 100644 --- a/examples/PlaneJets2D.jl +++ b/examples/PlaneJets2D.jl @@ -88,7 +88,7 @@ setup = Setup(x, y; Re, ArrayType); # Since the grid is uniform and identical for x and y, we may use a specialized # spectral pressure solver -psolver = SpectralPressureSolver(setup) +psolver = psolver_spectral(setup) # Initial conditions u₀ = diff --git a/examples/ShearLayer2D.jl b/examples/ShearLayer2D.jl index 2225290bc..8a5f23ae8 100644 --- a/examples/ShearLayer2D.jl +++ b/examples/ShearLayer2D.jl @@ -38,7 +38,7 @@ plotgrid(x, y) # Build setup and assemble operators setup = Setup(x, y; Re, ArrayType); -psolver = SpectralPressureSolver(setup) +psolver = psolver_spectral(setup) # Initial conditions: We add 1 to u in order to make global momentum # conservation less trivial diff --git a/examples/TaylorGreenVortex2D.jl b/examples/TaylorGreenVortex2D.jl index 3cf8c1bcf..d6b8370e6 100644 --- a/examples/TaylorGreenVortex2D.jl +++ b/examples/TaylorGreenVortex2D.jl @@ -27,7 +27,7 @@ function compute_convergence(; D, nlist, lims, Re, tlims, Δt, uref, ArrayType = @info "Computing error for n = $n" x = ntuple(α -> LinRange(lims..., n + 1), D) setup = Setup(x...; Re, ArrayType) - psolver = SpectralPressureSolver(setup) + psolver = psolver_spectral(setup) u₀ = create_initial_conditions( setup, (dim, x...) -> uref(dim, x..., tlims[1]), diff --git a/examples/TaylorGreenVortex3D.jl b/examples/TaylorGreenVortex3D.jl index 5bed0ea73..c9b212b8f 100644 --- a/examples/TaylorGreenVortex3D.jl +++ b/examples/TaylorGreenVortex3D.jl @@ -34,7 +34,7 @@ setup = Setup(x, y, z; Re, ArrayType); # Since the grid is uniform and identical for x, y, and z, we may use a # specialized spectral pressure solver -psolver = SpectralPressureSolver(setup); +psolver = psolver_spectral(setup); # Initial conditions u₀ = create_initial_conditions( diff --git a/lib/NeuralClosure/src/create_les_data.jl b/lib/NeuralClosure/src/create_les_data.jl index 7be062472..17ef41256 100644 --- a/lib/NeuralClosure/src/create_les_data.jl +++ b/lib/NeuralClosure/src/create_les_data.jl @@ -100,7 +100,7 @@ filtersaver(dns, les, filters, compression, psolver_dns, psolver_les; nupdate = tburn = typeof(Re)(0.1), tsim = typeof(Re)(0.1), Δt = typeof(Re)(1e-4), - PSolver = SpectralPressureSolver, + create_psolver = psolver_spectral, savefreq = 1, ArrayType = Array, icfunc = (setup, psolver) -> random_field(setup, typeof(Re)(0); psolver), @@ -120,7 +120,7 @@ function create_les_data(; tburn = typeof(Re)(0.1), tsim = typeof(Re)(0.1), Δt = typeof(Re)(1e-4), - PSolver = SpectralPressureSolver, + create_psolver = psolver_spectral, savefreq = 1, ArrayType = Array, icfunc = (setup, psolver, rng) -> random_field(setup, typeof(Re)(0); psolver, rng), @@ -152,8 +152,8 @@ function create_les_data(; # Since the grid is uniform and identical for x and y, we may use a specialized # spectral pressure solver - psolver = PSolver(dns) - psolver_les = PSolver.(les) + psolver = create_psolver(dns) + psolver_les = create_psolver.(les) # Number of time steps to save nt = round(Int, tsim / Δt) diff --git a/lib/PaperDC/postanalysis.jl b/lib/PaperDC/postanalysis.jl index 1398c6f4f..0603627c3 100644 --- a/lib/PaperDC/postanalysis.jl +++ b/lib/PaperDC/postanalysis.jl @@ -105,7 +105,7 @@ get_params(nlesscalar) = (; ndns = (n -> (n, n))(4096), # DNS resolution filters = (FaceAverage(), VolumeAverage()), ArrayType, - PSolver = SpectralPressureSolver, + create_psolver = psolver_spectral, icfunc = (setup, psolver) -> random_field(setup, zero(eltype(setup.grid.x[1])); kp = 20, psolver), rng, @@ -333,7 +333,7 @@ let starttime = time() println("iorder = $iorder, ifil = $ifil, ig = $ig") setup = setups_train[ig] - psolver = SpectralPressureSolver(setup) + psolver = psolver_spectral(setup) loss = IncompressibleNavierStokes.create_loss_post(; setup, psolver, @@ -406,7 +406,7 @@ smag = map(CartesianIndices((size(io_train, 2), 2))) do I println("iorder = $iorder, ifil = $ifil, θ = $θ, igrid = $igrid") projectorder = getorder(iorder) setup = setups_train[igrid] - psolver = SpectralPressureSolver(setup) + psolver = psolver_spectral(setup) d = data_train[isample] data = (; u = device.(d.data[igrid, ifil].u[it]), t = d.t[it]) nupdate = 4 @@ -483,7 +483,7 @@ eprior.post |> x -> reshape(x, :, 2) |> x -> round.(x; digits = 2) println("iorder = $iorder, ifil = $ifil, ig = $ig") projectorder = getorder(iorder) setup = setups_test[ig] - psolver = SpectralPressureSolver(setup) + psolver = psolver_spectral(setup) data = (; u = device.(data_test.data[ig, ifil].u), t = data_test.t) nupdate = 2 # No model @@ -684,7 +684,7 @@ kineticenergy = let for iorder = 1:2, ifil = 1:nfilter, ig = 1:ngrid println("iorder = $iorder, ifil = $ifil, ig = $ig") setup = setups_test[ig] - psolver = SpectralPressureSolver(setup) + psolver = psolver_spectral(setup) t = data_test.t u₀ = data_test.data[ig, ifil].u[1] |> device tlims = (t[1], t[end]) @@ -837,7 +837,7 @@ divs = let for iorder = 1:3, ifil = 1:nfilter, ig = 1:ngrid println("iorder = $iorder, ifil = $ifil, ig = $ig") setup = setups_test[ig] - psolver = SpectralPressureSolver(setup) + psolver = psolver_spectral(setup) t = data_test.t u₀ = data_test.data[ig, ifil].u[1] |> device tlims = (t[1], t[end]) @@ -966,7 +966,7 @@ ufinal = let println("iorder = $iorder, ifil = $ifil, igrid = $igrid") t = data_test.t setup = setups_test[igrid] - psolver = SpectralPressureSolver(setup) + psolver = psolver_spectral(setup) u₀ = data_test.data[igrid, ifil].u[1] |> device tlims = (t[1], t[end]) nupdate = 2 diff --git a/lib/PaperDC/prioranalysis.jl b/lib/PaperDC/prioranalysis.jl index 850c0b4b6..32b454230 100644 --- a/lib/PaperDC/prioranalysis.jl +++ b/lib/PaperDC/prioranalysis.jl @@ -61,13 +61,13 @@ filterdefs = [ lims = T(0), T(1) dns = let setup = Setup(ntuple(α -> LinRange(lims..., ndns + 1), D)...; Re, ArrayType) - psolver = SpectralPressureSolver(setup) + psolver = psolver_spectral(setup) (; setup, psolver) end; filters = map(filterdefs) do (Φ, nles) compression = ndns ÷ nles setup = Setup(ntuple(α -> LinRange(lims..., nles + 1), D)...; Re, ArrayType) - psolver = SpectralPressureSolver(setup) + psolver = psolver_spectral(setup) (; setup, Φ, compression, psolver) end; @@ -210,7 +210,7 @@ end # Plot spectra ####################################################### -# To free up memory in 3D (remove SpectralPressureSolver FFT arrays) +# To free up memory in 3D (remove psolver_spectral FFT arrays) dns = (; dns.setup) filters = map(filters) do f (; f.Φ, f.setup, f.compression) diff --git a/lib/TensorClosure/main.jl b/lib/TensorClosure/main.jl index 57df0c5fd..dc79fe2ce 100644 --- a/lib/TensorClosure/main.jl +++ b/lib/TensorClosure/main.jl @@ -13,7 +13,7 @@ n = 1024 lims = T(0), T(1) x = LinRange(lims..., n + 1), LinRange(lims..., n + 1) setup = Setup(x...; Re, ArrayType); -psolver = SpectralPressureSolver(setup); +psolver = psolver_spectral(setup); u₀ = random_field(setup, T(0); psolver); u = u₀ diff --git a/src/IncompressibleNavierStokes.jl b/src/IncompressibleNavierStokes.jl index 45ba12a2c..1c7c764f9 100644 --- a/src/IncompressibleNavierStokes.jl +++ b/src/IncompressibleNavierStokes.jl @@ -37,11 +37,8 @@ include("grid/max_size.jl") # Setup include("setup.jl") -# Pressure solvers -include("solvers/pressure/solvers.jl") -include("solvers/pressure/poisson.jl") -include("solvers/pressure/pressure.jl") -include("solvers/pressure/project.jl") +# Poisson solvers +include("pressure.jl") # Time steppers include("time_steppers/methods.jl") @@ -90,11 +87,8 @@ export Setup export stretched_grid, cosine_grid # Pressure solvers -export DirectPressureSolver, - CUDSSPressureSolver, - CGPressureSolver, - SpectralPressureSolver, - LowMemorySpectralPressureSolver +export psolver_direct, psolver_cg, psolver_cg_matrix, + psolver_spectral, psolver_spectral_lowmemory # Solvers export solve_unsteady, solve_steady_state diff --git a/src/create_initial_conditions.jl b/src/create_initial_conditions.jl index c06fae679..ae3aff49d 100644 --- a/src/create_initial_conditions.jl +++ b/src/create_initial_conditions.jl @@ -3,7 +3,7 @@ setup, initial_velocity, t = 0; - psolver = DirectPressureSolver(setup), + psolver = psolver_direct(setup), doproject = true, ) @@ -15,7 +15,7 @@ function create_initial_conditions( setup, initial_velocity, t = convert(eltype(setup.grid.x[1]), 0); - psolver = DirectPressureSolver(setup), + psolver = psolver_direct(setup), doproject = true, ) (; grid) = setup @@ -174,7 +174,7 @@ end setup, t = 0; A = 1, kp = 10, - psolver = SpectralPressureSolver(setup), + psolver = psolver_spectral(setup), rng = Random.default_rng(), ) @@ -189,7 +189,7 @@ function random_field( t = zero(eltype(setup.grid.x[1])); A = 1, kp = 10, - psolver = SpectralPressureSolver(setup), + psolver = psolver_spectral(setup), rng = Random.default_rng(), ) (; dimension, x, Ip, Ω) = setup.grid diff --git a/src/processors/processors.jl b/src/processors/processors.jl index a045540b5..89584b477 100644 --- a/src/processors/processors.jl +++ b/src/processors/processors.jl @@ -93,7 +93,7 @@ vtk_writer(; if :pressure ∈ fields if isnothing(psolver) @info "Creating new pressure solver for vtk_writer" - psolver = DirectPressureSolver(setup) + psolver = psolver_direct(setup) end F = zero.(u) div = zero(u[1]) diff --git a/src/processors/real_time_plot.jl b/src/processors/real_time_plot.jl index 35b2d92ea..7c649e966 100644 --- a/src/processors/real_time_plot.jl +++ b/src/processors/real_time_plot.jl @@ -111,7 +111,7 @@ function fieldplot( elseif fieldname == :pressure if isnothing(psolver) @info "Creating new pressure solver for fieldplot" - psolver = DirectPressureSolver(setup) + psolver = psolver_direct(setup) end F = zero.(u) div = zero(u[1]) @@ -231,7 +231,7 @@ function fieldplot( elseif fieldname == :pressure if isnothing(psolver) @info "Creating new pressure solver for fieldplot" - psolver = DirectPressureSolver(setup) + psolver = psolver_direct(setup) end F = zero.(u) div = zero(u[1]) @@ -239,7 +239,7 @@ function fieldplot( elseif fieldname == :Dfield if isnothing(psolver) @info "Creating new pressure solver for fieldplot" - psolver = DirectPressureSolver(setup) + psolver = psolver_direct(setup) end F = zero.(u) div = zero(u[1]) diff --git a/src/solvers/pressure/poisson.jl b/src/solvers/pressure/poisson.jl deleted file mode 100644 index 77a1352c7..000000000 --- a/src/solvers/pressure/poisson.jl +++ /dev/null @@ -1,203 +0,0 @@ -""" - poisson(solver, f) - -Solve the Poisson equation for the pressure with right hand side `f` at time `t`. -For periodic and no-slip BC, the sum of `f` should be zero. - -Non-mutating/allocating/out-of-place version. - -See also [`poisson!`](@ref). -""" -poisson(solver, f) = poisson!(solver, zero(f), f) - -# Laplacian is auto-adjoint -ChainRulesCore.rrule(::typeof(poisson), solver, f) = - (poisson(solver, f), φ -> (NoTangent(), NoTangent(), poisson(solver, unthunk(φ)))) - -""" - poisson!(solver, p, f) - -Solve the Poisson equation for the pressure with right hand side `f` at time `t`. -For periodic and no-slip BC, the sum of `f` should be zero. - -Mutating/non-allocating/in-place version. - -See also [`poisson`](@ref). -""" -function poisson! end - -function poisson!(solver::DirectPressureSolver, p, f) - (; setup, fact) = solver - (; grid, boundary_conditions) = setup - (; Ip) = grid - T = eltype(p) - # solver.f .= view(view(f, Ip), :) - # copyto!(solver.f, view(view(f, Ip), :)) - pp = view(view(p, Ip), :) - if any(bc -> bc[1] isa PressureBC || bc[2] isa PressureBC, boundary_conditions) - # No extra DOF - viewrange = (:) - else - # With extra DOF - viewrange = 1:length(solver.p)-1 - end - copyto!(view(solver.f, viewrange), Array(view(view(f, Ip), :))) - solver.p .= fact \ solver.f - copyto!(pp, T.(view(solver.p, viewrange))) - p -end - -function poisson!(solver::CGMatrixPressureSolver, p, f) - (; L, qin, qout, abstol, reltol, maxiter) = solver - copyto!(qin, view(view(f, Ip), :)) - p = view(p, :) - cg!(qout, L, qin; abstol, reltol, maxiter) - copyto!(view(view(p, Ip), :), qout) - p -end - -# Solve Lp = f -# where Lp = Ω * div(pressurgrad(p)) -# -# L is rank-1 deficient, so we add the constraint sum(p) = 0, i.e. solve -# -# [0 1] [0] [0] -# [1 L] [p] = [f] -# -# instead. This way, the matrix is still positive definite. -# For initial guess, we already know the average is zero. -function poisson!(solver::CGPressureSolver, p, f) - (; setup, abstol, reltol, maxiter, r, L, q, preconditioner) = solver - (; grid, workgroupsize) = setup - (; Np, Ip, Ω) = grid - T = typeof(reltol) - - function innerdot(a, b) - @kernel function innerdot!(d, a, b, I0) - I = @index(Global, Cartesian) - I = I + I0 - d[I-I+I0] += a[I] * b[I] - # a[I] = b[I] - end - # d = zero(eltype(a)) - I0 = first(Ip) - I0 -= oneunit(I0) - d = fill!(similar(a, ntuple(Returns(1), length(I0))), 0), - innerdot!(get_backend(a), workgroupsize)(d, a, b, I0; ndrange = Np) - d[] - end - - p .= 0 - - # Initial residual - laplacian!(L, p, setup) - - # Initialize - q .= 0 - r .= f .- L - ρ_prev = one(T) - # residual = norm(r[Ip]) - residual = sqrt(sum(abs2, view(r, Ip))) - # residual = norm(r) - tolerance = max(reltol * residual, abstol) - iteration = 0 - - while iteration < maxiter && residual > tolerance - preconditioner(L, r) - - # ρ = sum(L[Ip] .* r[Ip]) - ρ = dot(view(L, Ip), view(r, Ip)) - # ρ = innerdot(L, r) - # ρ = dot(L, r) - - β = ρ / ρ_prev - q .= L .+ β .* q - - # Periodic/symmetric padding (maybe) - apply_bc_p!(q, T(0), setup) - laplacian!(L, q, setup) - # α = ρ / sum(q[Ip] .* L[Ip]) - α = ρ / dot(view(q, Ip), view(L, Ip)) - # α = ρ / innerdot(q, L) - # α = ρ / dot(q, L) - - p .+= α .* q - r .-= α .* L - - ρ_prev = ρ - # residual = norm(r[Ip]) - residual = sqrt(sum(abs2, view(r, Ip))) - # residual = sqrt(sum(abs2, r)) - # residual = sqrt(innerdot(r, r)) - - iteration += 1 - end - - # @show iteration residual tolerance - - p -end - -function poisson!(solver::SpectralPressureSolver, p, f) - (; setup, plan, Ahat, fhat, phat) = solver - (; Ip) = setup.grid - - f = view(f, Ip) - - fhat .= complex.(f) - - # Fourier transform of right hand side - mul!(phat, plan, fhat) - - # Solve for coefficients in Fourier space - @. fhat = -phat / Ahat - - # Pressure is determined up to constant. We set this to 0 (instead of - # phat[1] / 0 = Inf) - # Note use of singleton range 1:1 instead of scalar index 1 - # (otherwise CUDA gets annoyed) - fhat[1:1] .= 0 - - # Transform back - ldiv!(phat, plan, fhat) - @. p[Ip] = real(phat) - - p -end - -function poisson!(solver::LowMemorySpectralPressureSolver, p, f) - (; setup, ahat, phat) = solver - (; dimension, Ip) = setup.grid - D = dimension() - - f = view(f, Ip) - - phat .= complex.(f) - - # Fourier transform of right hand side - fft!(phat) - - # Solve for coefficients in Fourier space - if D == 2 - ax = ahat - ay = reshape(ahat, 1, :) - @. phat = -phat / (ax + ay) - else - ax = ahat - ay = reshape(ahat, 1, :) - az = reshape(ahat, 1, 1, :) - @. phat = -phat / (ax + ay + az) - end - - # Pressure is determined up to constant. We set this to 0 (instead of - # phat[1] / 0 = Inf) - # Note use of singleton range 1:1 instead of scalar index 1 - # (otherwise CUDA gets annoyed) - phat[1:1] .= 0 - - # Transform back - ifft!(phat) - @. p[Ip] = real(phat) - - p -end diff --git a/src/solvers/pressure/pressure.jl b/src/solvers/pressure/pressure.jl deleted file mode 100644 index 7c03e8f69..000000000 --- a/src/solvers/pressure/pressure.jl +++ /dev/null @@ -1,37 +0,0 @@ -""" - pressure!(p, u, t, setup; psolver, F, div) - -Compute pressure from velocity field. This makes the pressure compatible with the velocity -field, resulting in same order pressure as velocity. -""" -function pressure!(p, u, t, setup; psolver, F, div) - (; grid) = setup - (; dimension, Iu, Ip, Ω) = grid - D = dimension() - momentum!(F, u, t, setup) - apply_bc_u!(F, t, setup; dudt = true) - divergence!(div, F, setup) - @. div *= Ω - poisson!(psolver, p, div) - apply_bc_p!(p, t, setup) - p -end - -""" - pressure(u, t, setup; psolver) - -Do additional pressure solve. This makes the pressure compatible with the velocity -field, resulting in same order pressure as velocity. -""" -function pressure(u, t, setup; psolver) - (; grid) = setup - (; dimension, Iu, Ip, Ω) = grid - D = dimension() - F = momentum(u, t, setup) - F = apply_bc_u(F, t, setup; dudt = true) - div = divergence(F, setup) - div = @. div * Ω - p = poisson(psolver, div) - p = apply_bc_p(p, t, setup) - p -end diff --git a/src/solvers/pressure/project.jl b/src/solvers/pressure/project.jl deleted file mode 100644 index 6c24190fd..000000000 --- a/src/solvers/pressure/project.jl +++ /dev/null @@ -1,42 +0,0 @@ -""" - project(u, setup; psolver) - -Project velocity field onto divergence-free space. -""" -function project(u, setup; psolver) - (; Ω) = setup.grid - T = eltype(u[1]) - - # Divergence of tentative velocity field - div = divergence(u, setup) - div = @. div * Ω - - # Solve the Poisson equation - p = poisson(psolver, div) - - # Apply pressure correction term - p = apply_bc_p(p, T(0), setup) - G = pressuregradient(p, setup) - u .- G -end - -""" - project!(u, setup; psolver, div, p) - -Project velocity field onto divergence-free space. -""" -function project!(u, setup; psolver, div, p) - (; Ω) = setup.grid - T = eltype(u[1]) - - # Divergence of tentative velocity field - divergence!(div, u, setup) - @. div *= Ω - - # Solve the Poisson equation - poisson!(psolver, p, div) - apply_bc_p!(p, T(0), setup) - - # Apply pressure correction term - applypressure!(u, p, setup) -end diff --git a/src/solvers/pressure/solvers.jl b/src/solvers/pressure/solvers.jl deleted file mode 100644 index e34ccfa5b..000000000 --- a/src/solvers/pressure/solvers.jl +++ /dev/null @@ -1,279 +0,0 @@ -""" - AbstractPressureSolver - -Pressure solver for the Poisson equation. -""" -abstract type AbstractPressureSolver{T} end - -""" - DirectPressureSolver() - -Direct pressure solver using a LU decomposition. -""" -struct DirectPressureSolver{T,S,F,A} <: AbstractPressureSolver{T} - setup::S - fact::F - f::A - p::A - function DirectPressureSolver(setup) - (; grid, boundary_conditions, ArrayType) = setup - (; x, Np) = grid - # T = eltype(x[1]) - T = Float64 - L = laplacian_mat(setup) - if any(bc -> bc[1] isa PressureBC || bc[2] isa PressureBC, boundary_conditions) - f = zeros(T, prod(Np)) - p = zeros(T, prod(Np)) - else - f = zeros(T, prod(Np) + 1) - p = zeros(T, prod(Np) + 1) - e = ones(T, size(L, 2)) - L = [L e; e' 0] - end - fact = factorize(L) - new{T,typeof(setup),typeof(fact),typeof(f)}(setup, fact, f, p) - end -end - -# """ -# CGPressureSolver(setup; [abstol], [reltol], [maxiter]) -# -# Conjugate gradients iterative pressure solver. -# """ -# struct CGPressureSolver{T,M<:AbstractMatrix{T}} <: AbstractPressureSolver{T} -# A::M -# abstol::T -# reltol::T -# maxiter::Int -# end - -# function CGPressureSolver( -# setup; -# abstol = 0, -# reltol = √eps(eltype(setup.operators.A)), -# maxiter = size(setup.operators.A, 2), -# ) -# (; A) = setup.operators -# CGPressureSolver{eltype(A),typeof(A)}(A, abstol, reltol, maxiter) -# end - -# # This moves all the inner arrays to the GPU when calling -# # `cu(::SpectralPressureSolver)` from CUDA.jl -# Adapt.adapt_structure(to, s::CGPressureSolver) = CGPressureSolver( -# adapt(to, s.A), -# adapt(to, s.abstol), -# adapt(to, s.reltol), -# adapt(to, s.maxiter), -# ) - -""" - CGMatrixPressureSolver(setup; [abstol], [reltol], [maxiter]) - -Conjugate gradients iterative pressure solver. -""" -struct CGMatrixPressureSolver{T,M} <: AbstractPressureSolver{T} - L::M - abstol::T - reltol::T - maxiter::Int -end - -function CGMatrixPressureSolver( - setup; - abstol = 0, - reltol = sqrt(eps(eltype(setup.grid.x[1]))), - maxiter = prod(setup.grid.Np), -) - L = laplacian_mat(setup) |> setup.device - # L = L |> CuSparseMatrixCSR - CGMatrixPressureSolver{eltype(L),typeof(L)}(L, abstol, reltol, maxiter) -end - -""" - CGPressureSolver(setup; [abstol], [reltol], [maxiter]) - -Conjugate gradients iterative pressure solver. -""" -struct CGPressureSolver{T,S,A,F} <: AbstractPressureSolver{T} - setup::S - abstol::T - reltol::T - maxiter::Int - r::A - L::A - q::A - preconditioner::F -end - -function create_laplace_diag(setup) - (; grid, workgroupsize) = setup - (; dimension, Δ, Δu, N, Np, Ip, Ω) = grid - D = dimension() - δ = Offset{D}() - @kernel function _laplace_diag!(z, p, I0) - I = @index(Global, Cartesian) - I = I + I0 - d = zero(eltype(z)) - for α = 1:length(I) - d -= Ω[I] / Δ[α][I[α]] * (1 / Δu[α][I[α]] + 1 / Δu[α][I[α]-1]) - end - z[I] = -p[I] / d - end - ndrange = Np - I0 = first(Ip) - I0 -= oneunit(I0) - function laplace_diag(z, p) - _laplace_diag!(get_backend(z), workgroupsize)(z, p, I0; ndrange) - # synchronize(get_backend(z)) - end -end - -CGPressureSolver( - setup; - abstol = zero(eltype(setup.grid.x[1])), - reltol = sqrt(eps(eltype(setup.grid.x[1]))), - maxiter = prod(setup.grid.Np), - # preconditioner = copy!, - preconditioner = create_laplace_diag(setup), -) = CGPressureSolver( - setup, - abstol, - reltol, - maxiter, - similar(setup.grid.x[1], setup.grid.N), - similar(setup.grid.x[1], setup.grid.N), - similar(setup.grid.x[1], setup.grid.N), - preconditioner, -) - -struct SpectralPressureSolver{T,A<:AbstractArray{Complex{T}},S,P} <: - AbstractPressureSolver{T} - setup::S - Ahat::A - phat::A - fhat::A - plan::P -end - -""" - SpectralPressureSolver(setup) - -Build spectral pressure solver from setup. -""" -function SpectralPressureSolver(setup) - (; grid, boundary_conditions) = setup - (; dimension, Δ, Np, x) = grid - - D = dimension() - T = eltype(Δ[1]) - - Δx = first.(Array.(Δ)) - - @assert( - all(bc -> bc[1] isa PeriodicBC && bc[2] isa PeriodicBC, boundary_conditions), - "SpectralPressureSolver only implemented for periodic boundary conditions", - ) - - @assert( - all(α -> all(≈(Δx[α]), Δ[α]), 1:D), - "SpectralPressureSolver requires uniform grid along each dimension", - ) - - # Fourier transform of the discretization - # Assuming uniform grid, although Δx[1] and Δx[2] do not need to be the same - - k = ntuple( - d -> reshape( - 0:Np[d]-1, - ntuple(Returns(1), d - 1)..., - :, - ntuple(Returns(1), D - d)..., - ), - D, - ) - - Ahat = fill!(similar(x[1], Complex{T}, Np), 0) - Tπ = T(π) # CUDA doesn't like pi - for d = 1:D - @. Ahat += sin(k[d] * Tπ / Np[d])^2 / Δx[d]^2 - end - - # Scale with Δx*Δy*Δz, since we solve the PDE in integrated form - Ahat .*= 4 * prod(Δx) - - # Pressure is determined up to constant. By setting the constant - # scaling factor to 1, we preserve the average. - # Note use of singleton range 1:1 instead of scalar index 1 - # (otherwise CUDA gets annoyed) - Ahat[1:1] .= 1 - - # Placeholders for intermediate results - phat = zero(Ahat) - fhat = zero(Ahat) - plan = plan_fft(fhat) - - SpectralPressureSolver{T,typeof(Ahat),typeof(setup),typeof(plan)}( - setup, - Ahat, - phat, - fhat, - plan, - ) -end - -struct LowMemorySpectralPressureSolver{ - T, - A<:AbstractArray{T}, - P<:AbstractArray{Complex{T}}, - S, -} <: AbstractPressureSolver{T} - setup::S - ahat::A - phat::P -end - -""" - LowMemorySpectralPressureSolver(setup) - -Build spectral pressure solver from setup. -This one is slower than `SpectralPressureSolver` but occupies less memory. -""" -function LowMemorySpectralPressureSolver(setup) - (; grid, boundary_conditions) = setup - (; dimension, Δ, Np, x) = grid - - D = dimension() - T = eltype(Δ[1]) - - Δx = Array(Δ[1])[1] - - @assert( - all(bc -> bc[1] isa PeriodicBC && bc[2] isa PeriodicBC, boundary_conditions), - "SpectralPressureSolver only implemented for periodic boundary conditions", - ) - - @assert( - all(α -> all(≈(Δx), Δ[α]), 1:D), - "SpectralPressureSolver requires uniform grid along each dimension", - ) - - @assert all(n -> n == Np[1], Np) - - # Fourier transform of the discretization - # Assuming uniform grid, although Δx[1] and Δx[2] do not need to be the same - - k = 0:Np[1]-1 - - Tπ = T(π) # CUDA doesn't like pi - ahat = fill!(similar(x[1], Np[1]), 0) - @. ahat = 4 * Δx^D * sin(k * Tπ / Np[1])^2 / Δx^2 - - # Placeholders for intermediate results - phat = fill!(similar(x[1], Complex{T}, Np), 0) - - LowMemorySpectralPressureSolver{T,typeof(ahat),typeof(phat),typeof(setup)}( - setup, - ahat, - phat, - ) -end diff --git a/src/solvers/solve_unsteady.jl b/src/solvers/solve_unsteady.jl index 88781454d..afcd371b0 100644 --- a/src/solvers/solve_unsteady.jl +++ b/src/solvers/solve_unsteady.jl @@ -4,7 +4,7 @@ u₀, tlims; method = RKMethods.RK44(; T = eltype(u₀[1])), - psolver = DirectPressureSolver(setup), + psolver = psolver_direct(setup), Δt = zero(eltype(u₀[1])), cfl = 1, n_adapt_Δt = 1, @@ -33,7 +33,7 @@ function solve_unsteady( u₀, tlims; method = RKMethods.RK44(; T = eltype(u₀[1])), - psolver = DirectPressureSolver(setup), + psolver = psolver_direct(setup), Δt = zero(eltype(u₀[1])), cfl = 1, n_adapt_Δt = 1, diff --git a/src/utils/save_vtk.jl b/src/utils/save_vtk.jl index c11ade3e6..1db1076c2 100644 --- a/src/utils/save_vtk.jl +++ b/src/utils/save_vtk.jl @@ -12,7 +12,7 @@ function save_vtk( t, filename = "output/solution"; fieldnames = [:velocity], - psolver = DirectPressureSolver(setup), + psolver = psolver_direct(setup), ) parts = split(filename, "/") path = join(parts[1:end-1], "/") diff --git a/test/chainrules.jl b/test/chainrules.jl index 7997b031e..ea5dd4746 100644 --- a/test/chainrules.jl +++ b/test/chainrules.jl @@ -8,7 +8,7 @@ # setup = Setup(x...; Re, # boundary_conditions, # ) -# psolver = DirectPressureSolver(setup) +# psolver = psolver_direct(setup) # u = random_field(setup, T(0); psolver) # randn!.(u) # p = randn!(similar(u[1])) @@ -38,7 +38,7 @@ testchainrules(dim) = @testset "Chain rules $(dim())D" begin stretched_grid(lims..., n, 1.2), cosine_grid(lims..., n), cosine_grid(lims..., n) end setup = Setup(x...; Re) - psolver = DirectPressureSolver(setup) + psolver = psolver_direct(setup) u = random_field(setup, T(0); psolver) randn!.(u) p = randn!(similar(u[1])) diff --git a/test/operators.jl b/test/operators.jl index b32a8a47c..0505e9113 100644 --- a/test/operators.jl +++ b/test/operators.jl @@ -11,7 +11,7 @@ testops(dim) = @testset "Operators $(dim())D" begin stretched_grid(lims..., n, 1.2), cosine_grid(lims..., n), cosine_grid(lims..., n) end setup = Setup(x...; Re) - psolver = DirectPressureSolver(setup) + psolver = psolver_direct(setup) u = random_field(setup, T(0); psolver) (; Iu, Ip, Ω) = setup.grid diff --git a/test/postprocess2D.jl b/test/postprocess2D.jl index d1b26e74f..610fd4b10 100644 --- a/test/postprocess2D.jl +++ b/test/postprocess2D.jl @@ -9,7 +9,7 @@ @test plotgrid(x, y) isa Makie.FigureAxisPlot - psolver = SpectralPressureSolver(setup) + psolver = psolver_spectral(setup) t_start, t_end = tlims = (0.0, 1.0) diff --git a/test/postprocess3D.jl b/test/postprocess3D.jl index 28e6693f1..70f74e6d6 100644 --- a/test/postprocess3D.jl +++ b/test/postprocess3D.jl @@ -12,7 +12,7 @@ @test plotgrid(x, y, z) isa Makie.Figure - psolver = SpectralPressureSolver(setup) + psolver = psolver_spectral(setup) t_start, t_end = tlims = (T(0), T(1)) diff --git a/test/pressure_solvers.jl b/test/pressure_solvers.jl index bc473f345..fb9b934de 100644 --- a/test/pressure_solvers.jl +++ b/test/pressure_solvers.jl @@ -8,12 +8,6 @@ (; xp) = setup.grid D = 2 - # Pressure solvers - direct = DirectPressureSolver(setup) - cg = CGPressureSolver(setup) - spectral = SpectralPressureSolver(setup) - lmspectral = LowMemorySpectralPressureSolver(setup) - initial_pressure(x, y) = 1 / 4 * (cos(2x) + cos(2y)) p_exact = initial_pressure.( @@ -22,30 +16,21 @@ IncompressibleNavierStokes.apply_bc_p!(p_exact, 0.0, setup) lap = IncompressibleNavierStokes.laplacian(p_exact, setup) + # Pressure solvers + direct = psolver_direct(setup) + cg = psolver_cg(setup) + spectral = psolver_spectral(setup) + spectral_lowmemory = solver_spectral_lowmemory(setup) + get_p(psolver) = IncompressibleNavierStokes.apply_bc_p( IncompressibleNavierStokes.poisson(psolver, lap), 0.0, setup, ) - p_direct = get_p(direct) - p_cg = get_p(cg) - p_spectral = get_p(spectral) - p_lmspectral = get_p(lmspectral) - - # Test that in-place and out-of-place versions give same result - get_p_inplace(psolver) = IncompressibleNavierStokes.apply_bc_p!( - IncompressibleNavierStokes.poisson!(psolver, zero(p_exact), lap), - 0.0, - setup, - ) - @test p_direct ≈ get_p_inplace(direct) - @test p_cg ≈ get_p_inplace(cg) - @test p_spectral ≈ get_p_inplace(spectral) - @test p_lmspectral ≈ get_p_inplace(lmspectral) # Test that solvers compute the exact pressure - @test p_direct ≈ p_exact - @test p_cg ≈ p_exact - @test p_spectral ≈ p_exact - @test p_lmspectral ≈ p_exact + @test get_p(p_direct) ≈ p_exact + @test get_p(p_cg) ≈ p_exact + @test get_p(p_spectral) ≈ p_exact + @test get_p(p_spectral_lowmemory) ≈ p_exact end diff --git a/test/solvers.jl b/test/solvers.jl index 2eb2bb80d..94f2587f3 100644 --- a/test/solvers.jl +++ b/test/solvers.jl @@ -7,7 +7,7 @@ y = LinRange(0, 2π, n + 1) setup = Setup(x, y; Re) - psolver = SpectralPressureSolver(setup) + psolver = psolver_spectral(setup) t_start, t_end = tlims = (0.0, 5.0) From 5b5b80a516f2b7621eee3561cd34a6280365e4bb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Sun, 5 May 2024 22:01:48 +0200 Subject: [PATCH 359/379] docs: Remove old line --- docs/src/features/gpu.md | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/src/features/gpu.md b/docs/src/features/gpu.md index b30d26d77..d566a96e6 100644 --- a/docs/src/features/gpu.md +++ b/docs/src/features/gpu.md @@ -8,7 +8,6 @@ Even if a GPU is not available, the operators are multithreaded if Julia is sta Limitations: -- [`psolver_direct`](@ref) is currently used on the CPU with double precision. [`psolver_cg`](@ref) works on the GPU. - This has not been tested with other GPU interfaces, such as - [AMDGPU.jl](https://github.com/JuliaGPU/AMDGPU.jl) - [Metal.jl](https://github.com/JuliaGPU/Metal.jl) From 9a3fe1358f79a9de8babd6c3969067b934465bb2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Sun, 5 May 2024 22:02:58 +0200 Subject: [PATCH 360/379] Add missing file --- src/pressure.jl | 454 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 454 insertions(+) create mode 100644 src/pressure.jl diff --git a/src/pressure.jl b/src/pressure.jl new file mode 100644 index 000000000..1a83d27a6 --- /dev/null +++ b/src/pressure.jl @@ -0,0 +1,454 @@ +# Note: poisson!(...) is a wrapper around psolver!(...) since we need to define +# a shared rrule. This could also be achieved by something like: +# abstract type AbstractPSolver end +# rrule(s::AbstractPSolver)(f) = (s(f), φ -> (NoTangent(), s(φ))) +# (s::AbstractPSolver)(f) = s(zero(...), f) +# struct PSolver < AbstractPSolver ... end +# (::PSolver)(p, f) = # solve Poisson + +""" + poisson(psolver, f) + +Solve the Poisson equation for the pressure with right hand side `f` at time `t`. +For periodic and no-slip BC, the sum of `f` should be zero. + +Non-mutating/allocating/out-of-place version. + +See also [`poisson!`](@ref). +""" +poisson(psolver, f) = poisson!(psolver, zero(f), f) + +# Laplacian is auto-adjoint +ChainRulesCore.rrule(::typeof(poisson), psolver, f) = + (poisson(psolver, f), φ -> (NoTangent(), NoTangent(), poisson(psolver, unthunk(φ)))) + +""" + poisson!(solver, p, f) + +Solve the Poisson equation for the pressure with right hand side `f` at time `t`. +For periodic and no-slip BC, the sum of `f` should be zero. + +Mutating/non-allocating/in-place version. + +See also [`poisson`](@ref). +""" +poisson!(psolver, p, f) = psolver(p, f) + +""" + pressure!(p, u, t, setup; psolver, F, div) + +Compute pressure from velocity field. This makes the pressure compatible with the velocity +field, resulting in same order pressure as velocity. +""" +function pressure!(p, u, t, setup; psolver, F, div) + (; grid) = setup + (; dimension, Iu, Ip, Ω) = grid + D = dimension() + momentum!(F, u, t, setup) + apply_bc_u!(F, t, setup; dudt = true) + divergence!(div, F, setup) + @. div *= Ω + poisson!(psolver, p, div) + apply_bc_p!(p, t, setup) + p +end + +""" + pressure(u, t, setup; psolver) + +Compute pressure from velocity field. This makes the pressure compatible with the velocity +field, resulting in same order pressure as velocity. +""" +function pressure(u, t, setup; psolver) + (; grid) = setup + (; dimension, Iu, Ip, Ω) = grid + D = dimension() + F = momentum(u, t, setup) + F = apply_bc_u(F, t, setup; dudt = true) + div = divergence(F, setup) + div = @. div * Ω + p = poisson(psolver, div) + p = apply_bc_p(p, t, setup) + p +end + +""" + project(u, setup; psolver) + +Project velocity field onto divergence-free space. +""" +function project(u, setup; psolver) + (; Ω) = setup.grid + T = eltype(u[1]) + + # Divergence of tentative velocity field + div = divergence(u, setup) + div = @. div * Ω + + # Solve the Poisson equation + p = poisson(psolver, div) + + # Apply pressure correction term + p = apply_bc_p(p, T(0), setup) + G = pressuregradient(p, setup) + u .- G +end + +""" + project!(u, setup; psolver, div, p) + +Project velocity field onto divergence-free space. +""" +function project!(u, setup; psolver, div, p) + (; Ω) = setup.grid + T = eltype(u[1]) + + # Divergence of tentative velocity field + divergence!(div, u, setup) + @. div *= Ω + + # Solve the Poisson equation + poisson!(psolver, p, div) + apply_bc_p!(p, T(0), setup) + + # Apply pressure correction term + applypressure!(u, p, setup) +end + +""" + poisson_direct(setup) + +Create direct Poisson solver using an appropriate matrix decomposition. +""" +psolver_direct(setup) = psolver_direct(setup.grid.x[1], setup) # Dispatch on array type + +# CPU version +function psolver_direct(::Array, setup) + (; grid, boundary_conditions) = setup + (; x, Np, Ip) = grid + T = Float64 # This is currently required for SuiteSparse + L = laplacian_mat(setup) + isdefinite = + any(bc -> bc[1] isa PressureBC || bc[2] isa PressureBC, boundary_conditions) + if isdefinite + # No extra DOF + ftemp = zeros(T, prod(Np)) + ptemp = zeros(T, prod(Np)) + viewrange = (:) + else + # With extra DOF + ftemp = zeros(T, prod(Np) + 1) + ptemp = zeros(T, prod(Np) + 1) + e = ones(T, size(L, 2)) + L = [L e; e' 0] + viewrange = 1:length(solver.p)-1 + end + fact = factorize(L) + function psolve!(p, f) + copyto!(view(ftemp, viewrange), view(view(f, Ip), :)) + ptemp .= fact \ ftemp + copyto!(view(view(p, Ip), :), eltype(p).(view(ptemp, viewrange))) + p + end +end + +""" + psolver_cg_matrix(setup; kwargs...) + +Conjugate gradients iterative Poisson solver. +The `kwargs` are passed to the `cg!` function +from IterativeSolvers.jl. +""" +function psolver_cg_matrix(setup; kwargs...) + (; x, Np, Ip) = grid + L = laplacian_mat(setup) + isdefinite = + any(bc -> bc[1] isa PressureBC || bc[2] isa PressureBC, boundary_conditions) + if isdefinite + # No extra DOF + ftemp = fill!(similar(x[1], prod(Np)), 0) + ptemp = fill!(similar(x[1], prod(Np)), 0) + viewrange = (:) + else + # With extra DOF + ftemp = fill!(similar(x[1], prod(Np) + 1), 0) + ptemp = fill!(similar(x[1], prod(Np) + 1), 0) + e = fill!(similar(x[1], prod(Np)), 1) + L = [L e; e' 0] + viewrange = 1:length(solver.p)-1 + end + function psolve!(p, f) + copyto!(view(ftemp, viewrange), view(view(f, Ip), :)) + cg!(ptemp, L, ftemp; kwargs...) + copyto!(view(view(p, Ip), :), view(ptemp, viewrange)) + p + end +end + +# Preconditioner +function create_laplace_diag(setup) + (; grid, workgroupsize) = setup + (; dimension, Δ, Δu, N, Np, Ip, Ω) = grid + D = dimension() + δ = Offset{D}() + @kernel function _laplace_diag!(z, p, I0) + I = @index(Global, Cartesian) + I = I + I0 + d = zero(eltype(z)) + for α = 1:length(I) + d -= Ω[I] / Δ[α][I[α]] * (1 / Δu[α][I[α]] + 1 / Δu[α][I[α]-1]) + end + z[I] = -p[I] / d + end + ndrange = Np + I0 = first(Ip) + I0 -= oneunit(I0) + laplace_diag(z, p) = _laplace_diag!(get_backend(z), workgroupsize)(z, p, I0; ndrange) +end + +""" + psolver_cg( + setup; + abstol = zero(eltype(setup.grid.x[1])), + reltol = sqrt(eps(eltype(setup.grid.x[1]))), + maxiter = prod(setup.grid.Np), + preconditioner = create_laplace_diag(setup), + ) + +Conjugate gradients iterative Poisson solver. +""" +function psolver_cg( + setup; + abstol = zero(eltype(setup.grid.x[1])), + reltol = sqrt(eps(eltype(setup.grid.x[1]))), + maxiter = prod(setup.grid.Np), + preconditioner = create_laplace_diag(setup), +) + (; grid, workgroupsize) = setup + (; Np, Ip, Ω) = grid + T = eltype(setup.grid.x[1]) + r = similar(setup.grid.x[1], setup.grid.N) + L = similar(setup.grid.x[1], setup.grid.N) + q = similar(setup.grid.x[1], setup.grid.N) + function psolve!(p, f) + function innerdot(a, b) + @kernel function innerdot!(d, a, b, I0) + I = @index(Global, Cartesian) + I = I + I0 + d[I-I+I0] += a[I] * b[I] + # a[I] = b[I] + end + # d = zero(eltype(a)) + I0 = first(Ip) + I0 -= oneunit(I0) + d = fill!(similar(a, ntuple(Returns(1), length(I0))), 0), + innerdot!(get_backend(a), workgroupsize)(d, a, b, I0; ndrange = Np) + d[] + end + + p .= 0 + + # Initial residual + laplacian!(L, p, setup) + + # Initialize + q .= 0 + r .= f .- L + ρ_prev = one(T) + # residual = norm(r[Ip]) + residual = sqrt(sum(abs2, view(r, Ip))) + # residual = norm(r) + tolerance = max(reltol * residual, abstol) + iteration = 0 + + while iteration < maxiter && residual > tolerance + preconditioner(L, r) + + # ρ = sum(L[Ip] .* r[Ip]) + ρ = dot(view(L, Ip), view(r, Ip)) + # ρ = innerdot(L, r) + # ρ = dot(L, r) + + β = ρ / ρ_prev + q .= L .+ β .* q + + # Periodic/symmetric padding (maybe) + apply_bc_p!(q, T(0), setup) + laplacian!(L, q, setup) + # α = ρ / sum(q[Ip] .* L[Ip]) + α = ρ / dot(view(q, Ip), view(L, Ip)) + # α = ρ / innerdot(q, L) + # α = ρ / dot(q, L) + + p .+= α .* q + r .-= α .* L + + ρ_prev = ρ + # residual = norm(r[Ip]) + residual = sqrt(sum(abs2, view(r, Ip))) + # residual = sqrt(sum(abs2, r)) + # residual = sqrt(innerdot(r, r)) + + iteration += 1 + end + + # @show iteration residual tolerance + + p + end +end + +""" + psolver_spectral(setup) + +Create spectral Poisson solver from setup. +""" +function psolver_spectral(setup) + (; grid, boundary_conditions) = setup + (; dimension, Δ, Np, Ip, x) = grid + + D = dimension() + T = eltype(Δ[1]) + + Δx = first.(Array.(Δ)) + + @assert( + all(bc -> bc[1] isa PeriodicBC && bc[2] isa PeriodicBC, boundary_conditions), + "Spectral psolver only implemented for periodic boundary conditions", + ) + + @assert( + all(α -> all(≈(Δx[α]), Δ[α]), 1:D), + "Spectral psolver requires uniform grid along each dimension", + ) + + # Fourier transform of the discretization + # Assuming uniform grid, although Δx[1] and Δx[2] do not need to be the same + + k = ntuple( + d -> reshape( + 0:Np[d]-1, + ntuple(Returns(1), d - 1)..., + :, + ntuple(Returns(1), D - d)..., + ), + D, + ) + + Ahat = fill!(similar(x[1], Complex{T}, Np), 0) + Tπ = T(π) # CUDA doesn't like pi + for d = 1:D + @. Ahat += sin(k[d] * Tπ / Np[d])^2 / Δx[d]^2 + end + + # Scale with Δx*Δy*Δz, since we solve the PDE in integrated form + Ahat .*= 4 * prod(Δx) + + # Pressure is determined up to constant. By setting the constant + # scaling factor to 1, we preserve the average. + # Note use of singleton range 1:1 instead of scalar index 1 + # (otherwise CUDA gets annoyed) + Ahat[1:1] .= 1 + + # Placeholders for intermediate results + phat = zero(Ahat) + fhat = zero(Ahat) + plan = plan_fft(fhat) + + function psolver(p, f) + f = view(f, Ip) + + fhat .= complex.(f) + + # Fourier transform of right hand side + mul!(phat, plan, fhat) + + # Solve for coefficients in Fourier space + @. fhat = -phat / Ahat + + # Pressure is determined up to constant. We set this to 0 (instead of + # phat[1] / 0 = Inf) + # Note use of singleton range 1:1 instead of scalar index 1 + # (otherwise CUDA gets annoyed) + fhat[1:1] .= 0 + + # Transform back + ldiv!(phat, plan, fhat) + @. p[Ip] = real(phat) + + p + end +end + +""" + psolver_spectral_lowmemory(setup) + +Create spectral Poisson solver from setup. +This one is slower than `psolver_spectral` but occupies less memory. +""" +function psolver_spectral_lowmemory(setup) + (; grid, boundary_conditions) = setup + (; dimension, Δ, Np, Ip, x) = grid + + D = dimension() + T = eltype(Δ[1]) + + Δx = Array(Δ[1])[1] + + @assert( + all(bc -> bc[1] isa PeriodicBC && bc[2] isa PeriodicBC, boundary_conditions), + "Spectral psolver only implemented for periodic boundary conditions", + ) + + @assert( + all(α -> all(≈(Δx), Δ[α]), 1:D), + "Spectral psolver requires uniform grid along each dimension", + ) + + @assert all(n -> n == Np[1], Np) + + # Fourier transform of the discretization + # Assuming uniform grid, although Δx[1] and Δx[2] do not need to be the same + + k = 0:Np[1]-1 + + Tπ = T(π) # CUDA doesn't like pi + ahat = fill!(similar(x[1], Np[1]), 0) + @. ahat = 4 * Δx^D * sin(k * Tπ / Np[1])^2 / Δx^2 + + # Placeholders for intermediate results + phat = fill!(similar(x[1], Complex{T}, Np), 0) + + function psolve!(p, f) + f = view(f, Ip) + + phat .= complex.(f) + + # Fourier transform of right hand side + fft!(phat) + + # Solve for coefficients in Fourier space + if D == 2 + ax = ahat + ay = reshape(ahat, 1, :) + @. phat = -phat / (ax + ay) + else + ax = ahat + ay = reshape(ahat, 1, :) + az = reshape(ahat, 1, 1, :) + @. phat = -phat / (ax + ay + az) + end + + # Pressure is determined up to constant. We set this to 0 (instead of + # phat[1] / 0 = Inf) + # Note use of singleton range 1:1 instead of scalar index 1 + # (otherwise CUDA gets annoyed) + phat[1:1] .= 0 + + # Transform back + ifft!(phat) + @. p[Ip] = real(phat) + + p + end +end From 1a4d6e28f5cbe1a3709ca24399da0f2015677770 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Sun, 5 May 2024 22:03:12 +0200 Subject: [PATCH 361/379] Add missing extension --- ext/IncompressibleNavierStokesCUDAExt.jl | 52 ++++++++++++++++++++++++ 1 file changed, 52 insertions(+) create mode 100644 ext/IncompressibleNavierStokesCUDAExt.jl diff --git a/ext/IncompressibleNavierStokesCUDAExt.jl b/ext/IncompressibleNavierStokesCUDAExt.jl new file mode 100644 index 000000000..568d5f900 --- /dev/null +++ b/ext/IncompressibleNavierStokesCUDAExt.jl @@ -0,0 +1,52 @@ +""" + IncompressibleNavierStokesCUDAExt + +CUDA extension for IncompressibleNavierStokes. +""" +module IncompressibleNavierStokesCUDAExt + +using CUDA +using CUDA.CUSPARSE +using CUDSS +using IncompressibleNavierStokes: PressureBC, laplacian_mat + +# CUDA version, using CUDSS LDLt decomposition. +function poisson_direct(::CuArray, setup) + (; grid, boundary_conditions) = setup + (; x, Np, Ip) = grid + T = eltype(x[1]) + L = laplacian_mat(setup) + isdefinite = + any(bc -> bc[1] isa PressureBC || bc[2] isa PressureBC, boundary_conditions) + if isdefinite + # No extra DOF + ftemp = fill!(similar(x[1], prod(Np)), 0) + ptemp = fill!(similar(x[1], prod(Np)), 0) + viewrange = (:) + # structure = "SPD" # Symmetric positive definite + structure = "G" # General matrix + _view = 'F' # Full matrix representation + else + # With extra DOF + ftemp = fill!(similar(x[1], prod(Np) + 1), 0) + ptemp = fill!(similar(x[1], prod(Np) + 1), 0) + e = fill!(similar(x[1], prod(Np)), 1) + L = [L e; e' 0] + viewrange = 1:length(solver.p)-1 + structure = "S" # Symmetric (not positive definite) + _view = 'L' # Lower triangular representation + end + L = CuSparseMatrixCSR(L) + solver = CudssSolver(L, structure; view = _view) + cudss("analysis", solver, p, f) + cudss("factorization", solver, p, f) # Compute factorization + function psolve!(p, f) + T = eltype(p) + copyto!(view(ftemp, viewrange), view(view(f, Ip), :)) + cudss("solve", solver, ptemp, ftemp) + copyto!(view(view(p, Ip), :), eltype(p).(view(ptemp, viewrange))) + p + end +end + +end From 02cb6c6abb114499eb19e401a6dcd242d0d23a01 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Sun, 5 May 2024 22:04:44 +0200 Subject: [PATCH 362/379] Rename file --- test/{pressure_solvers.jl => psolvers.jl} | 0 test/runtests.jl | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) rename test/{pressure_solvers.jl => psolvers.jl} (100%) diff --git a/test/pressure_solvers.jl b/test/psolvers.jl similarity index 100% rename from test/pressure_solvers.jl rename to test/psolvers.jl diff --git a/test/runtests.jl b/test/runtests.jl index dfad363ce..eb0205df7 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -39,7 +39,7 @@ using Test @testset "IncompressibleNavierStokes" begin include("grid.jl") - include("pressure_solvers.jl") + include("psolvers.jl") include("operators.jl") include("chainrules.jl") # include("models.jl") From f9da7cfb0e5fb89566e02d544dce34bb017c7742 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Sun, 5 May 2024 22:21:56 +0200 Subject: [PATCH 363/379] Add default psolver --- docs/src/features/pressure.md | 1 + examples/Actuator2D.jl | 6 ++---- examples/BackwardFacingStep2D.jl | 7 ++----- examples/DecayingTurbulence2D.jl | 9 ++------- examples/LidDrivenCavity2D.jl | 20 +++++--------------- examples/MultiActuator.jl | 8 ++------ examples/PlaneJets2D.jl | 8 +------- examples/ShearLayer2D.jl | 11 ++--------- lib/TensorClosure/main.jl | 4 +--- src/IncompressibleNavierStokes.jl | 8 ++++++-- src/create_initial_conditions.jl | 8 ++++---- src/pressure.jl | 20 ++++++++++++++++++++ src/processors/processors.jl | 2 +- src/processors/real_time_plot.jl | 6 +++--- src/solvers/solve_unsteady.jl | 4 ++-- src/utils/save_vtk.jl | 2 +- test/chainrules.jl | 5 ++--- test/operators.jl | 3 +-- test/postprocess3D.jl | 2 +- test/solvers.jl | 2 +- 20 files changed, 60 insertions(+), 76 deletions(-) diff --git a/docs/src/features/pressure.md b/docs/src/features/pressure.md index a8bb04c8d..a509c4bdb 100644 --- a/docs/src/features/pressure.md +++ b/docs/src/features/pressure.md @@ -12,6 +12,7 @@ enforces divergence freeness. There are multiple options for solving this system. ```@docs +default_psolver psolver_direct psolver_cg psolver_cg_matrix diff --git a/examples/Actuator2D.jl b/examples/Actuator2D.jl index 01081e777..b93dbf62a 100644 --- a/examples/Actuator2D.jl +++ b/examples/Actuator2D.jl @@ -53,10 +53,9 @@ bodyforce(dim, x, y, t) = dim() == 1 ? -cₜ * inside(x, y) : 0.0 # Build setup and assemble operators setup = Setup(x, y; Re = 100.0, boundary_conditions, bodyforce); -psolver = psolver_direct(setup); # Initial conditions (extend inflow) -u₀ = create_initial_conditions(setup, (dim, x, y) -> dim() == 1 ? 1.0 : 0.0; psolver); +u₀ = create_initial_conditions(setup, (dim, x, y) -> dim() == 1 ? 1.0 : 0.0); u = u₀ # Solve unsteady problem @@ -64,7 +63,6 @@ state, outputs = solve_unsteady( setup, u₀, (0.0, 12.0); - psolver, method = RKMethods.RK44P2(), Δt = 0.05, processors = ( @@ -83,7 +81,7 @@ state, outputs = solve_unsteady( # We may visualize or export the computed fields `(u, p)`. # Export to VTK -save_vtk(setup, state.u, state.t, "$output/solution"; psolver) +save_vtk(setup, state.u, state.t, "$output/solution") # We create a box to visualize the actuator. box = ( diff --git a/examples/BackwardFacingStep2D.jl b/examples/BackwardFacingStep2D.jl index a977e1301..2e809e2ce 100644 --- a/examples/BackwardFacingStep2D.jl +++ b/examples/BackwardFacingStep2D.jl @@ -52,10 +52,8 @@ plotgrid(x, y) # Build setup and assemble operators setup = Setup(x, y; Re, boundary_conditions, ArrayType); -psolver = psolver_direct(setup); - # Initial conditions (extend inflow) -u₀ = create_initial_conditions(setup, (dim, x, y) -> U(dim, x, y, zero(x)); psolver); +u₀ = create_initial_conditions(setup, (dim, x, y) -> U(dim, x, y, zero(x))); # Solve steady state problem ## u, p = solve_steady_state(setup, u₀, p₀); @@ -67,7 +65,6 @@ state, outputs = solve_unsteady( u₀, (T(0), T(7)); Δt = T(0.002), - psolver, processors = ( rtp = realtimeplotter(; setup, @@ -88,7 +85,7 @@ state, outputs = solve_unsteady( # We may visualize or export the computed fields # Export to VTK -save_vtk(setup, state.u, state.t, "$output/solution"; psolver) +save_vtk(setup, state.u, state.t, "$output/solution") # Plot pressure fieldplot(state; setup, fieldname = :pressure) diff --git a/examples/DecayingTurbulence2D.jl b/examples/DecayingTurbulence2D.jl index d47bcfd8b..ba341f818 100644 --- a/examples/DecayingTurbulence2D.jl +++ b/examples/DecayingTurbulence2D.jl @@ -34,12 +34,8 @@ x = LinRange(lims..., n + 1), LinRange(lims..., n + 1) # Build setup and assemble operators setup = Setup(x...; Re, ArrayType); -# Since the grid is uniform and identical for x and y, we may use a specialized -# spectral pressure solver -psolver = psolver_spectral(setup); - # Create random initial conditions -u₀ = random_field(setup, T(0); psolver); +u₀ = random_field(setup, T(0)); # Solve unsteady problem state, outputs = solve_unsteady( @@ -47,7 +43,6 @@ state, outputs = solve_unsteady( u₀, (T(0), T(1)); Δt = T(1e-3), - psolver, processors = ( ## rtp = realtimeplotter(; setup, nupdate = 1), ehist = realtimeplotter(; @@ -75,7 +70,7 @@ outputs.ehist outputs.espec # Export to VTK -save_vtk(setup, state.u, state.t, "$output/solution"; psolver) +save_vtk(setup, state.u, state.t, "$output/solution") # Plot field fieldplot(state; setup) diff --git a/examples/LidDrivenCavity2D.jl b/examples/LidDrivenCavity2D.jl index a09561b17..a187ffa10 100644 --- a/examples/LidDrivenCavity2D.jl +++ b/examples/LidDrivenCavity2D.jl @@ -30,8 +30,8 @@ output = "output/LidDrivenCavity2D" # Note how floating point type hygiene is enforced in the following using `T` # to avoid mixing different precisions. -T = Float64 -## T = Float32 +# T = Float64 +T = Float32 ## T = Float16 # We can also choose to do the computations on a different device. By default, @@ -78,19 +78,9 @@ plotgrid(x, y) # A 3D setup is built if we also provide a vector of z-coordinates. setup = Setup(x, y; boundary_conditions, Re, ArrayType); -# The pressure solver is used to solve the pressure Poisson equation. -# Available solvers are -# -# - [`psolver_direct`](@ref) (only for CPU with `Float64`) -# - [`psolver_cg`](@ref) -# - [`psolver_spectral`](@ref) (only for periodic boundary conditions and -# uniform grids) - -psolver = psolver_direct(setup); - # The initial conditions are provided in function. The value `dim()` determines # the velocity component. -u₀ = create_initial_conditions(setup, (dim, x, y) -> zero(x); psolver); +u₀ = create_initial_conditions(setup, (dim, x, y) -> zero(x)); # ## Solve problems # @@ -121,7 +111,7 @@ processors = ( # By default, a standard fourth order Runge-Kutta method is used. If we don't # provide the time step explicitly, an adaptive time step is used. tlims = (T(0), T(10)) -state, outputs = solve_unsteady(setup, u₀, tlims; Δt = T(1e-3), psolver, processors); +state, outputs = solve_unsteady(setup, u₀, tlims; Δt = T(1e-3), processors); # ## Post-process # @@ -130,7 +120,7 @@ state, outputs = solve_unsteady(setup, u₀, tlims; Δt = T(1e-3), psolver, proc # Export fields to VTK. The file `output/solution.vti` may be opened for # visualization in [ParaView](https://www.paraview.org/). This is particularly # useful for inspecting results from 3D simulations. -save_vtk(setup, state.u, state.t, "$output/solution"; psolver) +save_vtk(setup, state.u, state.t, "$output/solution") # Plot pressure fieldplot(state; setup, fieldname = :pressure) diff --git a/examples/MultiActuator.jl b/examples/MultiActuator.jl index 5f58e811c..98ddf40d4 100644 --- a/examples/MultiActuator.jl +++ b/examples/MultiActuator.jl @@ -79,10 +79,8 @@ y = LinRange(-T(2), T(2), 2n + 1) # Build setup and assemble operators setup = Setup(x, y; Re = T(2000), boundary_conditions, bodyforce, ArrayType); -psolver = psolver_direct(setup); - # Initial conditions (extend inflow) -u₀ = create_initial_conditions(setup, (dim, x, y) -> dim() == 1 ? one(x) : zero(x); psolver); +u₀ = create_initial_conditions(setup, (dim, x, y) -> dim() == 1 ? one(x) : zero(x)); u = u₀ t = T(0) @@ -115,14 +113,12 @@ state, outputs = solve_unsteady( # (T(0), T(1)); method = RKMethods.RK44P2(), Δt = T(0.01), - psolver, processors = ( rtp = realtimeplotter(; setup, # plot = fieldplot, # fieldname = :velocity, # fieldname = :pressure, - psolver, nupdate = 1, ), boxplotter = processor() do state @@ -147,7 +143,7 @@ state, outputs = solve_unsteady( save_vtk(setup, state.u, state.t, "$output/solution") # Plot pressure -fig = fieldplot(state; setup, fieldname = :pressure, psolver) +fig = fieldplot(state; setup, fieldname = :pressure) # lines!(box...; color = :red) lines!.(boxes; color = :red); fig diff --git a/examples/PlaneJets2D.jl b/examples/PlaneJets2D.jl index c1f8ae088..3a0220eeb 100644 --- a/examples/PlaneJets2D.jl +++ b/examples/PlaneJets2D.jl @@ -86,13 +86,8 @@ plotgrid(x, y) setup = Setup(x, y; Re, ArrayType); ## setup = Setup(x, y; Re, boundary_conditions, ArrayType); -# Since the grid is uniform and identical for x and y, we may use a specialized -# spectral pressure solver -psolver = psolver_spectral(setup) - # Initial conditions -u₀ = - create_initial_conditions(setup, (dim, x, y) -> dim() == 1 ? U(x, y) : zero(x); psolver); +u₀ = create_initial_conditions(setup, (dim, x, y) -> dim() == 1 ? U(x, y) : zero(x)); # Real time plot: Streamwise average and spectrum function meanplot(state; setup) @@ -156,7 +151,6 @@ state, outputs = solve_unsteady( (T(0), T(1)); method = RKMethods.RK44P2(), Δt = 0.001, - psolver, processors = ( rtp = realtimeplotter(; setup, diff --git a/examples/ShearLayer2D.jl b/examples/ShearLayer2D.jl index 8a5f23ae8..eed4b8007 100644 --- a/examples/ShearLayer2D.jl +++ b/examples/ShearLayer2D.jl @@ -38,19 +38,13 @@ plotgrid(x, y) # Build setup and assemble operators setup = Setup(x, y; Re, ArrayType); -psolver = psolver_spectral(setup) - # Initial conditions: We add 1 to u in order to make global momentum # conservation less trivial d = T(π / 15) e = T(0.05) U1(y) = y ≤ π ? tanh((y - T(π / 2)) / d) : tanh((T(3π / 2) - y) / d) ## U1(y) = T(1) + (y ≤ π ? tanh((y - T(π / 2)) / d) : tanh((T(3π / 2) - y) / d)) -u₀ = create_initial_conditions( - setup, - (dim, x, y) -> dim() == 1 ? U1(y) : e * sin(x); - psolver, -); +u₀ = create_initial_conditions(setup, (dim, x, y) -> dim() == 1 ? U1(y) : e * sin(x)); # Solve unsteady problem state, outputs = solve_unsteady( @@ -58,7 +52,6 @@ state, outputs = solve_unsteady( u₀, (T(0), T(8)); Δt = T(0.01), - psolver, processors = ( rtp = realtimeplotter(; setup, @@ -81,7 +74,7 @@ state, outputs = solve_unsteady( outputs.rtp # Export to VTK -save_vtk(setup, state.u, state.t, "$output/solution"; psolver) +save_vtk(setup, state.u, state.t, "$output/solution") # Plot pressure fieldplot(state; setup, fieldname = :pressure) diff --git a/lib/TensorClosure/main.jl b/lib/TensorClosure/main.jl index dc79fe2ce..201fef4f6 100644 --- a/lib/TensorClosure/main.jl +++ b/lib/TensorClosure/main.jl @@ -13,8 +13,7 @@ n = 1024 lims = T(0), T(1) x = LinRange(lims..., n + 1), LinRange(lims..., n + 1) setup = Setup(x...; Re, ArrayType); -psolver = psolver_spectral(setup); -u₀ = random_field(setup, T(0); psolver); +u₀ = random_field(setup, T(0)); u = u₀ @@ -38,7 +37,6 @@ state, outputs = solve_unsteady( u₀, (T(0), T(1)); Δt = T(1e-4), - psolver, processors = ( # rtp = realtimeplotter(; setup, nupdate = 1), log = timelogger(; nupdate = 10), diff --git a/src/IncompressibleNavierStokes.jl b/src/IncompressibleNavierStokes.jl index 1c7c764f9..d240cc48c 100644 --- a/src/IncompressibleNavierStokes.jl +++ b/src/IncompressibleNavierStokes.jl @@ -87,8 +87,12 @@ export Setup export stretched_grid, cosine_grid # Pressure solvers -export psolver_direct, psolver_cg, psolver_cg_matrix, - psolver_spectral, psolver_spectral_lowmemory +export default_psolver, + psolver_direct, + psolver_cg, + psolver_cg_matrix, + psolver_spectral, + psolver_spectral_lowmemory # Solvers export solve_unsteady, solve_steady_state diff --git a/src/create_initial_conditions.jl b/src/create_initial_conditions.jl index ae3aff49d..49131e617 100644 --- a/src/create_initial_conditions.jl +++ b/src/create_initial_conditions.jl @@ -3,7 +3,7 @@ setup, initial_velocity, t = 0; - psolver = psolver_direct(setup), + psolver = default_psolver(setup), doproject = true, ) @@ -15,7 +15,7 @@ function create_initial_conditions( setup, initial_velocity, t = convert(eltype(setup.grid.x[1]), 0); - psolver = psolver_direct(setup), + psolver = default_psolver(setup), doproject = true, ) (; grid) = setup @@ -174,7 +174,7 @@ end setup, t = 0; A = 1, kp = 10, - psolver = psolver_spectral(setup), + psolver = default_psolver(setup), rng = Random.default_rng(), ) @@ -189,7 +189,7 @@ function random_field( t = zero(eltype(setup.grid.x[1])); A = 1, kp = 10, - psolver = psolver_spectral(setup), + psolver = default_psolver(setup), rng = Random.default_rng(), ) (; dimension, x, Ip, Ω) = setup.grid diff --git a/src/pressure.jl b/src/pressure.jl index 1a83d27a6..9db83df2c 100644 --- a/src/pressure.jl +++ b/src/pressure.jl @@ -115,6 +115,26 @@ function project!(u, setup; psolver, div, p) applypressure!(u, p, setup) end +""" + default_psolver(setup) + +Get default Poisson solver from setup. +""" +function default_psolver(setup) + (; grid, boundary_conditions) = setup + (; dimension, Δ) = grid + D = dimension() + Δx = first.(Array.(Δ)) + isperiodic = + all(bc -> bc[1] isa PeriodicBC && bc[2] isa PeriodicBC, boundary_conditions) + isuniform = all(α -> all(≈(Δx[α]), Δ[α]), 1:D) + if isperiodic && isuniform + psolver_spectral(setup) + else + psolver_direct(setup) + end +end + """ poisson_direct(setup) diff --git a/src/processors/processors.jl b/src/processors/processors.jl index 89584b477..133339584 100644 --- a/src/processors/processors.jl +++ b/src/processors/processors.jl @@ -93,7 +93,7 @@ vtk_writer(; if :pressure ∈ fields if isnothing(psolver) @info "Creating new pressure solver for vtk_writer" - psolver = psolver_direct(setup) + psolver = default_psolver(setup) end F = zero.(u) div = zero(u[1]) diff --git a/src/processors/real_time_plot.jl b/src/processors/real_time_plot.jl index 7c649e966..9ae77aa2f 100644 --- a/src/processors/real_time_plot.jl +++ b/src/processors/real_time_plot.jl @@ -111,7 +111,7 @@ function fieldplot( elseif fieldname == :pressure if isnothing(psolver) @info "Creating new pressure solver for fieldplot" - psolver = psolver_direct(setup) + psolver = default_psolver(setup) end F = zero.(u) div = zero(u[1]) @@ -231,7 +231,7 @@ function fieldplot( elseif fieldname == :pressure if isnothing(psolver) @info "Creating new pressure solver for fieldplot" - psolver = psolver_direct(setup) + psolver = default_psolver(setup) end F = zero.(u) div = zero(u[1]) @@ -239,7 +239,7 @@ function fieldplot( elseif fieldname == :Dfield if isnothing(psolver) @info "Creating new pressure solver for fieldplot" - psolver = psolver_direct(setup) + psolver = default_psolver(setup) end F = zero.(u) div = zero(u[1]) diff --git a/src/solvers/solve_unsteady.jl b/src/solvers/solve_unsteady.jl index afcd371b0..144084b30 100644 --- a/src/solvers/solve_unsteady.jl +++ b/src/solvers/solve_unsteady.jl @@ -4,7 +4,7 @@ u₀, tlims; method = RKMethods.RK44(; T = eltype(u₀[1])), - psolver = psolver_direct(setup), + psolver = default_psolver(setup), Δt = zero(eltype(u₀[1])), cfl = 1, n_adapt_Δt = 1, @@ -33,7 +33,7 @@ function solve_unsteady( u₀, tlims; method = RKMethods.RK44(; T = eltype(u₀[1])), - psolver = psolver_direct(setup), + psolver = default_psolver(setup), Δt = zero(eltype(u₀[1])), cfl = 1, n_adapt_Δt = 1, diff --git a/src/utils/save_vtk.jl b/src/utils/save_vtk.jl index 1db1076c2..0456e6cc2 100644 --- a/src/utils/save_vtk.jl +++ b/src/utils/save_vtk.jl @@ -12,7 +12,7 @@ function save_vtk( t, filename = "output/solution"; fieldnames = [:velocity], - psolver = psolver_direct(setup), + psolver = default_psolver(setup), ) parts = split(filename, "/") path = join(parts[1:end-1], "/") diff --git a/test/chainrules.jl b/test/chainrules.jl index ea5dd4746..ba16e4cff 100644 --- a/test/chainrules.jl +++ b/test/chainrules.jl @@ -8,8 +8,7 @@ # setup = Setup(x...; Re, # boundary_conditions, # ) -# psolver = psolver_direct(setup) -# u = random_field(setup, T(0); psolver) +# u = random_field(setup, T(0)) # randn!.(u) # p = randn!(similar(u[1])) # test_rrule(apply_bc_u, u, T(0) ⊢ NoTangent(), setup ⊢ NoTangent()) @@ -38,7 +37,7 @@ testchainrules(dim) = @testset "Chain rules $(dim())D" begin stretched_grid(lims..., n, 1.2), cosine_grid(lims..., n), cosine_grid(lims..., n) end setup = Setup(x...; Re) - psolver = psolver_direct(setup) + psolver = default_psolver(setup) u = random_field(setup, T(0); psolver) randn!.(u) p = randn!(similar(u[1])) diff --git a/test/operators.jl b/test/operators.jl index 0505e9113..37295b886 100644 --- a/test/operators.jl +++ b/test/operators.jl @@ -11,8 +11,7 @@ testops(dim) = @testset "Operators $(dim())D" begin stretched_grid(lims..., n, 1.2), cosine_grid(lims..., n), cosine_grid(lims..., n) end setup = Setup(x...; Re) - psolver = psolver_direct(setup) - u = random_field(setup, T(0); psolver) + u = random_field(setup, T(0)) (; Iu, Ip, Ω) = setup.grid @testset "Divergence" begin diff --git a/test/postprocess3D.jl b/test/postprocess3D.jl index 70f74e6d6..eae19a86e 100644 --- a/test/postprocess3D.jl +++ b/test/postprocess3D.jl @@ -12,7 +12,7 @@ @test plotgrid(x, y, z) isa Makie.Figure - psolver = psolver_spectral(setup) + psolver = default_psolver(setup) t_start, t_end = tlims = (T(0), T(1)) diff --git a/test/solvers.jl b/test/solvers.jl index 94f2587f3..91f43ca61 100644 --- a/test/solvers.jl +++ b/test/solvers.jl @@ -7,7 +7,7 @@ y = LinRange(0, 2π, n + 1) setup = Setup(x, y; Re) - psolver = psolver_spectral(setup) + psolver = default_psolver(setup) t_start, t_end = tlims = (0.0, 5.0) From 53216087285196ae54e4bd9bc8e7670f3d958436 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Sun, 5 May 2024 22:39:19 +0200 Subject: [PATCH 364/379] Minor fixes --- ext/IncompressibleNavierStokesCUDAExt.jl | 2 +- src/create_initial_conditions.jl | 4 ++-- src/pressure.jl | 4 ++-- test/psolvers.jl | 10 +++++----- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/ext/IncompressibleNavierStokesCUDAExt.jl b/ext/IncompressibleNavierStokesCUDAExt.jl index 568d5f900..2a4b724bb 100644 --- a/ext/IncompressibleNavierStokesCUDAExt.jl +++ b/ext/IncompressibleNavierStokesCUDAExt.jl @@ -32,7 +32,7 @@ function poisson_direct(::CuArray, setup) ptemp = fill!(similar(x[1], prod(Np) + 1), 0) e = fill!(similar(x[1], prod(Np)), 1) L = [L e; e' 0] - viewrange = 1:length(solver.p)-1 + viewrange = 1:prod(Np) structure = "S" # Symmetric (not positive definite) _view = 'L' # Lower triangular representation end diff --git a/src/create_initial_conditions.jl b/src/create_initial_conditions.jl index 49131e617..ed0a81129 100644 --- a/src/create_initial_conditions.jl +++ b/src/create_initial_conditions.jl @@ -147,10 +147,10 @@ function create_spectrum(; setup, kp, rng = Random.default_rng()) # Remove non-divergence free part: (I - k k^T / k^2) e ke = sum(α -> e[α] .* kkkk[α], 1:D) for α = 1:D - e0 = e[1:1] # CUDA doesn't like e[1] + e0 = e[α][1:1] # CUDA doesn't like e[α][1] @. e[α] -= kkkk[α] * ke / knorm^2 # Restore k=0 component, which is divergence free anyways - e[1:1] .= e0 + e[α][1:1] .= e0 end # Normalize diff --git a/src/pressure.jl b/src/pressure.jl index 9db83df2c..367cac3ab 100644 --- a/src/pressure.jl +++ b/src/pressure.jl @@ -161,7 +161,7 @@ function psolver_direct(::Array, setup) ptemp = zeros(T, prod(Np) + 1) e = ones(T, size(L, 2)) L = [L e; e' 0] - viewrange = 1:length(solver.p)-1 + viewrange = 1:prod(Np) end fact = factorize(L) function psolve!(p, f) @@ -195,7 +195,7 @@ function psolver_cg_matrix(setup; kwargs...) ptemp = fill!(similar(x[1], prod(Np) + 1), 0) e = fill!(similar(x[1], prod(Np)), 1) L = [L e; e' 0] - viewrange = 1:length(solver.p)-1 + viewrange = 1:prod(Np) end function psolve!(p, f) copyto!(view(ftemp, viewrange), view(view(f, Ip), :)) diff --git a/test/psolvers.jl b/test/psolvers.jl index fb9b934de..2391667de 100644 --- a/test/psolvers.jl +++ b/test/psolvers.jl @@ -20,7 +20,7 @@ direct = psolver_direct(setup) cg = psolver_cg(setup) spectral = psolver_spectral(setup) - spectral_lowmemory = solver_spectral_lowmemory(setup) + spectral_lowmemory = psolver_spectral_lowmemory(setup) get_p(psolver) = IncompressibleNavierStokes.apply_bc_p( IncompressibleNavierStokes.poisson(psolver, lap), @@ -29,8 +29,8 @@ ) # Test that solvers compute the exact pressure - @test get_p(p_direct) ≈ p_exact - @test get_p(p_cg) ≈ p_exact - @test get_p(p_spectral) ≈ p_exact - @test get_p(p_spectral_lowmemory) ≈ p_exact + @test get_p(direct) ≈ p_exact + @test get_p(cg) ≈ p_exact + @test get_p(spectral) ≈ p_exact + @test get_p(spectral_lowmemory) ≈ p_exact end From 0a8ee436118256a69e1a9392a59e2437ceef24fe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Sun, 5 May 2024 22:42:38 +0200 Subject: [PATCH 365/379] Merge files --- src/IncompressibleNavierStokes.jl | 4 -- src/{grid => }/grid.jl | 76 +++++++++++++++++++++++++++++++ src/grid/cosine_grid.jl | 18 -------- src/grid/dimension.jl | 19 -------- src/grid/max_size.jl | 10 ---- src/grid/stretched_grid.jl | 25 ---------- 6 files changed, 76 insertions(+), 76 deletions(-) rename src/{grid => }/grid.jl (73%) delete mode 100644 src/grid/cosine_grid.jl delete mode 100644 src/grid/dimension.jl delete mode 100644 src/grid/max_size.jl delete mode 100644 src/grid/stretched_grid.jl diff --git a/src/IncompressibleNavierStokes.jl b/src/IncompressibleNavierStokes.jl index d240cc48c..cdfb3921c 100644 --- a/src/IncompressibleNavierStokes.jl +++ b/src/IncompressibleNavierStokes.jl @@ -28,11 +28,7 @@ using WriteVTK: CollectionFile, paraview_collection, vtk_grid, vtk_save include("boundary_conditions.jl") # Grid -include("grid/dimension.jl") include("grid/grid.jl") -include("grid/stretched_grid.jl") -include("grid/cosine_grid.jl") -include("grid/max_size.jl") # Setup include("setup.jl") diff --git a/src/grid/grid.jl b/src/grid.jl similarity index 73% rename from src/grid/grid.jl rename to src/grid.jl index 13bd41597..f47e796ee 100644 --- a/src/grid/grid.jl +++ b/src/grid.jl @@ -1,3 +1,79 @@ +""" + Dimension(N) + +Represent an `N`-dimensional space. +Returns `N` when called. + +```example +julia> d = Dimension(3) +Dimension{3}() + +julia> d() +3 +``` +""" +struct Dimension{N} end + +Dimension(N) = Dimension{N}() + +(::Dimension{N})() where {N} = N + +""" + max_size(grid) + +Get size of the largest grid element. +""" +function max_size(grid) + (; Δ) = grid + m = maximum.(Δ) + sqrt(sum(m .^ 2)) +end + +""" + cosine_grid(a, b, N) + +Create a nonuniform grid of `N + 1` points from `a` to `b` using a cosine +profile, i.e. + +```math +x_i = a + \\frac{1}{2} \\left( 1 - \\cos \\left( \\pi \\frac{i}{n} \\right) \\right) +(b - a), \\quad i = 0, \\dots, N +``` + +See also [`stretched_grid`](@ref). +""" +function cosine_grid(a, b, N) + T = typeof(a) + i = T.(0:N) + @. a + (b - a) * (1 - cospi(i / N)) / 2 +end + +""" + stretched_grid(a, b, N, s = 1) + +Create a nonuniform grid of `N + 1` points from `a` to `b` with a stretch +factor of `s`. If `s = 1`, return a uniform spacing from `a` to `b`. Otherwise, +return a vector ``x \\in \\mathbb{R}^{N + 1}`` such that ``x_n = a + \\sum_{i = +1}^n s^{i - 1} h`` for ``n = 0, \\dots , N``. Setting ``x_N = b`` then gives +``h = (b - a) \\frac{1 - s}{1 - s^N}``, resulting in + +```math +x_n = a + (b - a) \\frac{1 - s^n}{1 - s^N}, \\quad n = 0, \\dots, N. +``` + +Note that `stretched_grid(a, b, N, s)[n]` corresponds to ``x_{n - 1}``. + +See also [`cosine_grid`](@ref). +""" +function stretched_grid(a, b, N, s = 1) + s > 0 || error("The stretch factor must be positive") + if s ≈ 1 + LinRange(a, b, N + 1) + else + map(i -> a + (b - a) * (1 - s^i) / (1 - s^N), 0:N) + end +end + """ Grid(x, boundary_conditions) diff --git a/src/grid/cosine_grid.jl b/src/grid/cosine_grid.jl deleted file mode 100644 index 7f2fd4b97..000000000 --- a/src/grid/cosine_grid.jl +++ /dev/null @@ -1,18 +0,0 @@ -""" - cosine_grid(a, b, N) - -Create a nonuniform grid of `N + 1` points from `a` to `b` using a cosine -profile, i.e. - -```math -x_i = a + \\frac{1}{2} \\left( 1 - \\cos \\left( \\pi \\frac{i}{n} \\right) \\right) -(b - a), \\quad i = 0, \\dots, N -``` - -See also [`stretched_grid`](@ref). -""" -function cosine_grid(a, b, N) - T = typeof(a) - i = T.(0:N) - @. a + (b - a) * (1 - cospi(i / N)) / 2 -end diff --git a/src/grid/dimension.jl b/src/grid/dimension.jl deleted file mode 100644 index 3cc8e8b81..000000000 --- a/src/grid/dimension.jl +++ /dev/null @@ -1,19 +0,0 @@ -""" - Dimension(N) - -Represent an `N`-dimensional space. -Returns `N` when called. - -```example -julia> d = Dimension(3) -Dimension{3}() - -julia> d() -3 -``` -""" -struct Dimension{N} end - -Dimension(N) = Dimension{N}() - -(::Dimension{N})() where {N} = N diff --git a/src/grid/max_size.jl b/src/grid/max_size.jl deleted file mode 100644 index 23c05bce5..000000000 --- a/src/grid/max_size.jl +++ /dev/null @@ -1,10 +0,0 @@ -""" - max_size(grid) - -Get size of the largest grid element. -""" -function max_size(grid) - (; Δ) = grid - m = maximum.(Δ) - sqrt(sum(m .^ 2)) -end diff --git a/src/grid/stretched_grid.jl b/src/grid/stretched_grid.jl deleted file mode 100644 index c122d46ed..000000000 --- a/src/grid/stretched_grid.jl +++ /dev/null @@ -1,25 +0,0 @@ -""" - stretched_grid(a, b, N, s = 1) - -Create a nonuniform grid of `N + 1` points from `a` to `b` with a stretch -factor of `s`. If `s = 1`, return a uniform spacing from `a` to `b`. Otherwise, -return a vector ``x \\in \\mathbb{R}^{N + 1}`` such that ``x_n = a + \\sum_{i = -1}^n s^{i - 1} h`` for ``n = 0, \\dots , N``. Setting ``x_N = b`` then gives -``h = (b - a) \\frac{1 - s}{1 - s^N}``, resulting in - -```math -x_n = a + (b - a) \\frac{1 - s^n}{1 - s^N}, \\quad n = 0, \\dots, N. -``` - -Note that `stretched_grid(a, b, N, s)[n]` corresponds to ``x_{n - 1}``. - -See also [`cosine_grid`](@ref). -""" -function stretched_grid(a, b, N, s = 1) - s > 0 || error("The stretch factor must be positive") - if s ≈ 1 - LinRange(a, b, N + 1) - else - map(i -> a + (b - a) * (1 - s^i) / (1 - s^N), 0:N) - end -end From 426357a0126ca07a9c7e6eec1f32d9a07536909a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Sun, 5 May 2024 22:49:38 +0200 Subject: [PATCH 366/379] Merge files --- src/IncompressibleNavierStokes.jl | 18 +- .../real_time_plot.jl => processors.jl} | 188 ++++++++++++++++++ src/processors/animator.jl | 37 ---- src/processors/processors.jl | 149 -------------- 4 files changed, 192 insertions(+), 200 deletions(-) rename src/{processors/real_time_plot.jl => processors.jl} (69%) delete mode 100644 src/processors/animator.jl delete mode 100644 src/processors/processors.jl diff --git a/src/IncompressibleNavierStokes.jl b/src/IncompressibleNavierStokes.jl index cdfb3921c..323724710 100644 --- a/src/IncompressibleNavierStokes.jl +++ b/src/IncompressibleNavierStokes.jl @@ -24,17 +24,12 @@ using WriteVTK: CollectionFile, paraview_collection, vtk_grid, vtk_save # # Easily retrieve value from Val # (::Val{x})() where {x} = x -# Boundary conditions +# General stuff include("boundary_conditions.jl") - -# Grid -include("grid/grid.jl") - -# Setup +include("grid.jl") include("setup.jl") - -# Poisson solvers include("pressure.jl") +include("operators.jl") # Time steppers include("time_steppers/methods.jl") @@ -49,12 +44,7 @@ include("time_steppers/tableaux.jl") include("create_initial_conditions.jl") # Processors -include("processors/processors.jl") -include("processors/real_time_plot.jl") -include("processors/animator.jl") - -# Discrete operators -include("operators.jl") +include("processors.jl") # Solvers include("solvers/get_timestep.jl") diff --git a/src/processors/real_time_plot.jl b/src/processors.jl similarity index 69% rename from src/processors/real_time_plot.jl rename to src/processors.jl index 9ae77aa2f..a06988a3d 100644 --- a/src/processors/real_time_plot.jl +++ b/src/processors.jl @@ -1,3 +1,191 @@ +""" + processor(initialize, finalize = (initialized, state) -> initialized) + +Process results from time stepping. Before time stepping, the `initialize` +function is called on an observable of the time stepper `state`, returning +`initialized`. The observable is updated every time step. + +After timestepping, the `finalize` function is called on `initialized` and the +final `state`. + +See the following example: + +```example +function initialize(state) + s = 0 + println("Let's sum up the time steps") + on(state) do (; n, t) + println("The summand is \$n, the time is \$t") + s = s + n + end + s +end + +finalize(i, state) = println("The final sum (at time t=\$(state.t)) is \$s") +p = processor(initialize, finalize) +``` + +When solved for 6 time steps from t=0 to t=2 the displayed output is + +``` +Let's sum up the time steps +The summand is 0, the time is 0.0 +The summand is 1, the time is 0.4 +The summand is 2, the time is 0.8 +The summand is 3, the time is 1.2 +The summand is 4, the time is 1.6 +The summand is 5, the time is 2.0 +The final sum (at time t=2.0) is 15 +``` +""" +processor(initialize, finalize = (initialized, state) -> initialized) = + (; initialize, finalize) + +""" + timelogger(; nupdate = 1) + +Create processor that logs time step information. +""" +timelogger(; nupdate = 1) = + processor() do state + on(state) do (; t, n) + n % nupdate == 0 || return + @printf "Iteration %d\tt = %g\n" n t + end + nothing + end + +""" + vtk_writer(; + setup, + nupdate = 1, + dir = "output", + filename = "solution", + fields = (:velocity, :pressure, :vorticity), + ) + +Create processor that writes the solution every `nupdate` time steps to a VTK +file. The resulting Paraview data collection file is stored in +`"\$dir/\$filename.pvd"`. +""" +vtk_writer(; + setup, + nupdate = 1, + dir = "output", + filename = "solution", + fields = (:velocity, :pressure, :vorticity), + psolver = nothing, +) = + processor((pvd, state) -> vtk_save(pvd)) do state + (; grid) = setup + (; dimension, xp) = grid + D = dimension() + ispath(dir) || mkpath(dir) + pvd = paraview_collection(joinpath(dir, filename)) + xparr = Array.(xp) + (; u) = state[] + if :velocity ∈ fields + up = interpolate_u_p(u, setup) + uparr = Array.(up) + # ParaView prefers 3D vectors. Add zero z-component. + D == 2 && (uparr = (uparr..., zero(up[1]))) + end + if :pressure ∈ fields + if isnothing(psolver) + @info "Creating new pressure solver for vtk_writer" + psolver = default_psolver(setup) + end + F = zero.(u) + div = zero(u[1]) + p = zero(u[1]) + parr = adapt(Array, p) + end + if :vorticity ∈ fields + ω = vorticity(u, setup) + ωp = interpolate_ω_p(ω, setup) + ωparr = adapt(Array, ωp) + end + on(state) do (; u, t, n) + n % nupdate == 0 || return + tformat = replace(string(t), "." => "p") + vtk_grid("$(dir)/$(filename)_t=$tformat", xparr...) do vtk + if :velocity ∈ fields + interpolate_u_p!(up, u, setup) + copyto!.(uparr, up) + vtk["velocity"] = uparr + end + if :pressure ∈ fields + pressure!(p, u, setup; psolver, F, div) + vtk["pressure"] = copyto!(parr, p) + end + if :vorticity ∈ fields + vorticity!(ω, u, setup) + interpolate_ω_p!(ωp, ω, setup) + D == 2 ? copyto!(ωparr, ωp) : copyto!.(ωparr, ωp) + vtk["vorticity"] = ωparr + end + pvd[t] = vtk + end + end + pvd + end + +""" + fieldsaver(; setup, nupdate = 1) + +Create processor that stores the solution and time every `nupdate` time step. +""" +fieldsaver(; setup, nupdate = 1) = + processor() do state + T = eltype(setup.grid.x[1]) + (; u) = state[] + fields = (; u = fill(Array.(u), 0), t = zeros(T, 0)) + on(state) do (; u, p, t, n) + n % nupdate == 0 || return + push!(fields.u, Array.(u)) + push!(fields.t, t) + end + fields + end + +""" + animator(; setup, path, plot = fieldplot, nupdate = 1, kwargs...) + +Animate a plot of the solution every `update` iteration. +The animation is saved to `path`, which should have one +of the following extensions: + +- ".mkv" +- ".mp4" +- ".webm" +- ".gif" + +The plot is determined by a `plotter` processor. +Additional `kwargs` are passed to `plot`. +""" +animator(; + setup, + path, + plot = fieldplot, + nupdate = 1, + framerate = 24, + visible = true, + kwargs..., +) = + processor((stream, state) -> save(path, stream)) do outerstate + ispath(dirname(path)) || mkpath(dirname(path)) + state = Observable(outerstate[]) + fig = plot(state; setup, kwargs...) + visible && display(fig) + stream = VideoStream(fig; framerate, visible) + on(outerstate) do outerstate + outerstate.n % nupdate == 0 || return + state[] = outerstate + recordframe!(stream) + end + stream + end + """ realtimeplotter(; setup, diff --git a/src/processors/animator.jl b/src/processors/animator.jl deleted file mode 100644 index c4c754141..000000000 --- a/src/processors/animator.jl +++ /dev/null @@ -1,37 +0,0 @@ -""" - animator(; setup, path, plot = fieldplot, nupdate = 1, kwargs...) - -Animate a plot of the solution every `update` iteration. -The animation is saved to `path`, which should have one -of the following extensions: - -- ".mkv" -- ".mp4" -- ".webm" -- ".gif" - -The plot is determined by a `plotter` processor. -Additional `kwargs` are passed to `plot`. -""" -animator(; - setup, - path, - plot = fieldplot, - nupdate = 1, - framerate = 24, - visible = true, - kwargs..., -) = - processor((stream, state) -> save(path, stream)) do outerstate - ispath(dirname(path)) || mkpath(dirname(path)) - state = Observable(outerstate[]) - fig = plot(state; setup, kwargs...) - visible && display(fig) - stream = VideoStream(fig; framerate, visible) - on(outerstate) do outerstate - outerstate.n % nupdate == 0 || return - state[] = outerstate - recordframe!(stream) - end - stream - end diff --git a/src/processors/processors.jl b/src/processors/processors.jl deleted file mode 100644 index 133339584..000000000 --- a/src/processors/processors.jl +++ /dev/null @@ -1,149 +0,0 @@ -""" - processor(initialize, finalize = (initialized, state) -> initialized) - -Process results from time stepping. Before time stepping, the `initialize` -function is called on an observable of the time stepper `state`, returning -`initialized`. The observable is updated every time step. - -After timestepping, the `finalize` function is called on `initialized` and the -final `state`. - -See the following example: - -```example -function initialize(state) - s = 0 - println("Let's sum up the time steps") - on(state) do (; n, t) - println("The summand is \$n, the time is \$t") - s = s + n - end - s -end - -finalize(i, state) = println("The final sum (at time t=\$(state.t)) is \$s") -p = processor(initialize, finalize) -``` - -When solved for 6 time steps from t=0 to t=2 the displayed output is - -``` -Let's sum up the time steps -The summand is 0, the time is 0.0 -The summand is 1, the time is 0.4 -The summand is 2, the time is 0.8 -The summand is 3, the time is 1.2 -The summand is 4, the time is 1.6 -The summand is 5, the time is 2.0 -The final sum (at time t=2.0) is 15 -``` -""" -processor(initialize, finalize = (initialized, state) -> initialized) = - (; initialize, finalize) - -""" - timelogger(; nupdate = 1) - -Create processor that logs time step information. -""" -timelogger(; nupdate = 1) = - processor() do state - on(state) do (; t, n) - n % nupdate == 0 || return - @printf "Iteration %d\tt = %g\n" n t - end - nothing - end - -""" - vtk_writer(; - setup, - nupdate = 1, - dir = "output", - filename = "solution", - fields = (:velocity, :pressure, :vorticity), - ) - -Create processor that writes the solution every `nupdate` time steps to a VTK -file. The resulting Paraview data collection file is stored in -`"\$dir/\$filename.pvd"`. -""" -vtk_writer(; - setup, - nupdate = 1, - dir = "output", - filename = "solution", - fields = (:velocity, :pressure, :vorticity), - psolver = nothing, -) = - processor((pvd, state) -> vtk_save(pvd)) do state - (; grid) = setup - (; dimension, xp) = grid - D = dimension() - ispath(dir) || mkpath(dir) - pvd = paraview_collection(joinpath(dir, filename)) - xparr = Array.(xp) - (; u) = state[] - if :velocity ∈ fields - up = interpolate_u_p(u, setup) - uparr = Array.(up) - # ParaView prefers 3D vectors. Add zero z-component. - D == 2 && (uparr = (uparr..., zero(up[1]))) - end - if :pressure ∈ fields - if isnothing(psolver) - @info "Creating new pressure solver for vtk_writer" - psolver = default_psolver(setup) - end - F = zero.(u) - div = zero(u[1]) - p = zero(u[1]) - parr = adapt(Array, p) - end - if :vorticity ∈ fields - ω = vorticity(u, setup) - ωp = interpolate_ω_p(ω, setup) - ωparr = adapt(Array, ωp) - end - on(state) do (; u, t, n) - n % nupdate == 0 || return - tformat = replace(string(t), "." => "p") - vtk_grid("$(dir)/$(filename)_t=$tformat", xparr...) do vtk - if :velocity ∈ fields - interpolate_u_p!(up, u, setup) - copyto!.(uparr, up) - vtk["velocity"] = uparr - end - if :pressure ∈ fields - pressure!(p, u, setup; psolver, F, div) - vtk["pressure"] = copyto!(parr, p) - end - if :vorticity ∈ fields - vorticity!(ω, u, setup) - interpolate_ω_p!(ωp, ω, setup) - D == 2 ? copyto!(ωparr, ωp) : copyto!.(ωparr, ωp) - vtk["vorticity"] = ωparr - end - pvd[t] = vtk - end - end - pvd - end - -""" - fieldsaver(; setup, nupdate = 1) - -Create processor that stores the solution and time every `nupdate` time step. -""" -fieldsaver(; setup, nupdate = 1) = - processor() do state - T = eltype(setup.grid.x[1]) - (; u) = state[] - fields = (; u = fill(Array.(u), 0), t = zeros(T, 0)) - on(state) do (; u, p, t, n) - n % nupdate == 0 || return - push!(fields.u, Array.(u)) - push!(fields.t, t) - end - fields - end From d5d930e75747e7c2203951e5914d4b43090bd87e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Sun, 5 May 2024 22:52:24 +0200 Subject: [PATCH 367/379] Add info --- test/chainrules.jl | 2 ++ test/operators.jl | 2 ++ 2 files changed, 4 insertions(+) diff --git a/test/chainrules.jl b/test/chainrules.jl index ba16e4cff..eff75485b 100644 --- a/test/chainrules.jl +++ b/test/chainrules.jl @@ -17,6 +17,8 @@ # Test chain rule correctness by comparing with finite differences testchainrules(dim) = @testset "Chain rules $(dim())D" begin + @info "Testing chain rules in $(dim())D" + # Setup D = dim() T = Float64 diff --git a/test/operators.jl b/test/operators.jl index 37295b886..ac4c5c28d 100644 --- a/test/operators.jl +++ b/test/operators.jl @@ -1,4 +1,6 @@ testops(dim) = @testset "Operators $(dim())D" begin + @info "Testing operators in $(dim())D" + # Setup D = dim() T = Float64 From 195b6e04c3a032d5818741c8c3ceed47f64789cb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Mon, 6 May 2024 21:12:41 +0200 Subject: [PATCH 368/379] Add temperature equation --- README.md | 7 +- examples/RayleighBenard2D.jl | 59 +++++++++ examples/RayleighTaylor2D.jl | 67 ++++++++++ examples/RayleighTaylor3D.jl | 96 +++++++++++++++ ext/IncompressibleNavierStokesCUDAExt.jl | 14 ++- src/IncompressibleNavierStokes.jl | 2 +- src/boundary_conditions.jl | 62 ++++++++++ src/operators.jl | 114 +++++++++++++++++- src/processors.jl | 27 ++++- src/setup.jl | 47 ++++++++ src/solvers/solve_unsteady.jl | 44 ++++--- .../step_explicit_runge_kutta.jl | 33 +++-- src/time_steppers/time_stepper_caches.jl | 13 +- 13 files changed, 536 insertions(+), 49 deletions(-) create mode 100644 examples/RayleighBenard2D.jl create mode 100644 examples/RayleighTaylor2D.jl create mode 100644 examples/RayleighTaylor3D.jl diff --git a/README.md b/README.md index d5d8ff2e1..89e17501d 100644 --- a/README.md +++ b/README.md @@ -50,10 +50,15 @@ snapshot files using the `save_vtk` function, or the full time series using the | ![](assets/examples/Actuator3D.png) | ![](assets/examples/BackwardFacingStep3D.png) | ![](assets/examples/DecayingTurbulence3D.png) | ![](assets/examples/TaylorGreenVortex3D.png) | | [Actuator (3D)](examples/Actuator3D.jl) | [Backward Facing Step (3D)](examples/BackwardFacingStep3D.jl) | [Decaying Turbulence (3D)](examples/DecayingTurbulence3D.jl) | [Taylor-Green Vortex (3D)](examples/TaylorGreenVortex3D.jl) | +IncompressibleNavierStokes also supports adding a temperature equation. + +| ![](assets/examples/RB2D.mp4) | ![](assets/examples/RT3D.mp4) | +|:-:|:-:| +| [Rayleigh-Bénard (2D)](examples/RayleighBenard2D.jl) | [Rayleigh-Taylor (3D)](examples/RayleighTaylor3D.jl) | ## Demo -The following example code using a negative body force on a small rectangle +The following example code using a negative body force on a small rectangle with an unsteady inflow. It simulates a wind turbine (actuator) under varying wind conditions. diff --git a/examples/RayleighBenard2D.jl b/examples/RayleighBenard2D.jl new file mode 100644 index 000000000..d095d363a --- /dev/null +++ b/examples/RayleighBenard2D.jl @@ -0,0 +1,59 @@ +#md using CairoMakie +using GLMakie #!md +using CairoMakie +using IncompressibleNavierStokes + +using CUDA, CUDSS +ArrayType = CuArray + +temperature = temperature_equation(; + Pr = T(0.71), + Ra = T(1e7), + Ge = T(0.1), + dodissipation = true, + boundary_conditions = ((SymmetricBC(), SymmetricBC()), (DirichletBC(1.0), DirichletBC(0.0))), + gdir = 2, + nondim_type = 1, +) + +n = 64 +x = LinRange(T(0), T(1), n + 1) +y = LinRange(T(0), T(1), n + 1) +setup = Setup( + x, + y; + boundary_conditions = ((DirichletBC(), DirichletBC()), (DirichletBC(), DirichletBC())), + Re = 1 / temperature.α1, + temperature, + ArrayType, +); +ustart = create_initial_conditions(setup, (dim, x, y) -> zero(x)); +(; xp) = setup.grid; +tempstart = @. max(-sin($(T(3π)) * xp[1]) / 100, 0) + (1 - xp[2]'); + +# with_theme(theme_black()) do +with_theme(;) do +state, outputs = solve_unsteady(; + setup, + ustart, + tempstart, + tlims = (0.0, 100.0), + Δt = 1e-2, + processors = (; + # rtp = realtimeplotter(; + anim = animator(; + path = "examples/output/RB2D.mp4", + setup, + nupdate = 100, + fieldname = :temperature, + colorrange = (0.0, 1.0), + # displayupdates = true, + size = (600, 500), + ), + log = timelogger(; nupdate = 20), + ), +); +end + +state.temp +IncompressibleNavierStokes.apply_bc_temp!(state.temp, T(0), setup) diff --git a/examples/RayleighTaylor2D.jl b/examples/RayleighTaylor2D.jl new file mode 100644 index 000000000..b7e8371b1 --- /dev/null +++ b/examples/RayleighTaylor2D.jl @@ -0,0 +1,67 @@ +#md using CairoMakie +using GLMakie #!md +using CairoMakie +using IncompressibleNavierStokes + +using CUDA, CUDSS +T = Float32 +ArrayType = CuArray + +temperature = temperature_equation(; + Pr = T(0.71), + Ra = T(1e6), + Ge = T(1.0), + # Ge = T(0.1), + dodissipation = true, + boundary_conditions = ((SymmetricBC(), SymmetricBC()), (SymmetricBC(), SymmetricBC())), + gdir = 2, + nondim_type = 1, +) + +n = 128 +x = LinRange(T(0), T(1), n + 1) +y = LinRange(T(0), T(2), 2n + 1) +setup = Setup( + x, + y; + boundary_conditions = ((DirichletBC(), DirichletBC()), (DirichletBC(), DirichletBC())), + Re = 1 / temperature.α1, + temperature, + ArrayType, +); +ustart = create_initial_conditions(setup, (dim, x, y) -> zero(x)); +(; xp) = setup.grid; +tempstart = @. $(T(1)) * (1.0 + 0.05 * sin($(T(2π)) * xp[1]) > xp[2]'); +tempstart = @. $(T(1)) * (1.0 + 0.05 * sin($(T(π)) * xp[1]) > xp[2]'); +# @. tempstart += 0.3 * randn() + +state, outputs = solve_unsteady(; + setup, + # ustart, + # tempstart, + ustart = state.u, + tempstart = state.temp, + tlims = (0.0, 40.0), + Δt = 5e-3, + processors = (; + rtp = realtimeplotter(; + # anim = animator(; + path = "examples/output/RT2D.mp4", + setup, + nupdate = 200, + fieldname = :temperature, + displayupdates = true, + size = (300, 500), + ), + log = timelogger(; nupdate = 50), + ), +); + +state.u .|> extrema +state.temp |> extrema +state.u[1] +state.u[2] +state.temp + +fieldplot(state; setup, fieldname = :temperature) +save("toto.png", current_figure()) diff --git a/examples/RayleighTaylor3D.jl b/examples/RayleighTaylor3D.jl new file mode 100644 index 000000000..a425c3f1b --- /dev/null +++ b/examples/RayleighTaylor3D.jl @@ -0,0 +1,96 @@ +#md using CairoMakie +using GLMakie #!md +using IncompressibleNavierStokes + +using CUDA, CUDSS +T = Float32 +ArrayType = CuArray + +outdir = joinpath(@__DIR__, "output") + +temperature = temperature_equation(; + Pr = T(0.71), + Ra = T(1e6), + # Ge = T(0.1), + Ge = T(1.0), + dodissipation = true, + boundary_conditions = ( + (SymmetricBC(), SymmetricBC(), SymmetricBC()), + (SymmetricBC(), SymmetricBC(), SymmetricBC()), + (SymmetricBC(), SymmetricBC(), SymmetricBC()), + ), + gdir = 3, + nondim_type = 1, +) + +n = 80 +x = LinRange(T(0), T(1), n + 1) +y = LinRange(T(0), T(1), n + 1) +z = LinRange(T(0), T(2), 2n + 1) +setup = Setup( + x, + y, + z; + boundary_conditions = ( + (DirichletBC(), DirichletBC(), DirichletBC()), + (DirichletBC(), DirichletBC(), DirichletBC()), + (DirichletBC(), DirichletBC(), DirichletBC()), + ), + Re = 1 / temperature.α1, + temperature, + ArrayType, +); + +psolver = psolver_direct(setup) + +ustart = create_initial_conditions(setup, (dim, x, y, z) -> zero(x); psolver); +(; xp) = setup.grid; +xx = xp[1]; +xy = reshape(xp[2], 1, :); +xz = reshape(xp[3], 1, 1, :); +tempstart = @. $(T(1)) * (1.0 + 0.05 * sin($(T(1.05 * π)) * xx) * sin($(T(π)) * xy) > xz); + +fieldplot( + (; u = ustart, temp = tempstart, t = T(0)); + # state; + setup, + levels = LinRange{T}(0.8, 1, 5), + # levels = LinRange(-T(5), T(1), 10), + # fieldname = :eig2field, + fieldname = :temperature, + size = (400, 600), +) + +save("$outdir/RT3D_initial.png", current_figure()) + +state, outputs = solve_unsteady(; + setup, + ustart, + tempstart, + tlims = (0.0, 40.0), + # Δt = 5e-3, + Δt = 1e-2, + psolver, + processors = (; + # rtp = realtimeplotter(; + anim = animator(; + # path = "$outdir/RT3D_temp.mp4", + path = "$outdir/RT3D_l2.mp4", + setup, + nupdate = 20, + fieldname = :eig2field, + # fieldname = :temperature, + levels = LinRange(-T(5), T(1), 10), + # levels = LinRange{T}(0, 1, 10), + # displayupdates = true, + size = (400, 600), + ), + log = timelogger(; nupdate = 10), + ), +); + +field = IncompressibleNavierStokes.eig2field(state.u, setup)[setup.grid.Ip] +# hist(vec(Array(log(max(eps(T), field))) +hist(vec(Array(log.(max.(eps(T), .-field))))) + +fieldplot(state; setup, fieldname = :temperature) diff --git a/ext/IncompressibleNavierStokesCUDAExt.jl b/ext/IncompressibleNavierStokesCUDAExt.jl index 2a4b724bb..38f91edb1 100644 --- a/ext/IncompressibleNavierStokesCUDAExt.jl +++ b/ext/IncompressibleNavierStokesCUDAExt.jl @@ -8,10 +8,12 @@ module IncompressibleNavierStokesCUDAExt using CUDA using CUDA.CUSPARSE using CUDSS +using IncompressibleNavierStokes using IncompressibleNavierStokes: PressureBC, laplacian_mat +using SparseArrays # CUDA version, using CUDSS LDLt decomposition. -function poisson_direct(::CuArray, setup) +function IncompressibleNavierStokes.psolver_direct(::CuArray, setup) (; grid, boundary_conditions) = setup (; x, Np, Ip) = grid T = eltype(x[1]) @@ -30,16 +32,18 @@ function poisson_direct(::CuArray, setup) # With extra DOF ftemp = fill!(similar(x[1], prod(Np) + 1), 0) ptemp = fill!(similar(x[1], prod(Np) + 1), 0) - e = fill!(similar(x[1], prod(Np)), 1) + # e = fill!(similar(x[1], prod(Np)), 1) + e = ones(T, prod(Np)) + L = SparseMatrixCSC(L) L = [L e; e' 0] viewrange = 1:prod(Np) structure = "S" # Symmetric (not positive definite) _view = 'L' # Lower triangular representation end L = CuSparseMatrixCSR(L) - solver = CudssSolver(L, structure; view = _view) - cudss("analysis", solver, p, f) - cudss("factorization", solver, p, f) # Compute factorization + solver = CudssSolver(L, structure, _view) + cudss("analysis", solver, ptemp, ftemp) + cudss("factorization", solver, ptemp, ftemp) # Compute factorization function psolve!(p, f) T = eltype(p) copyto!(view(ftemp, viewrange), view(view(f, Ip), :)) diff --git a/src/IncompressibleNavierStokes.jl b/src/IncompressibleNavierStokes.jl index 323724710..849751054 100644 --- a/src/IncompressibleNavierStokes.jl +++ b/src/IncompressibleNavierStokes.jl @@ -67,7 +67,7 @@ export fieldplot, energy_history_plot, energy_spectrum_plot export animator # Setup -export Setup +export Setup, temperature_equation # 1D grids export stretched_grid, cosine_grid diff --git a/src/boundary_conditions.jl b/src/boundary_conditions.jl index 42e06850f..890f46693 100644 --- a/src/boundary_conditions.jl +++ b/src/boundary_conditions.jl @@ -25,6 +25,7 @@ struct DirichletBC{F,G} <: AbstractBC end DirichletBC() = DirichletBC(nothing, nothing) +DirichletBC(u) = DirichletBC(u, nothing) """ SymmetricBC() @@ -104,9 +105,11 @@ offset_p(::PressureBC, isright) = 1 + !isright function apply_bc_u! end function apply_bc_p! end +function apply_bc_temp! end apply_bc_u(u, t, setup; kwargs...) = apply_bc_u!(copy.(u), t, setup; kwargs...) apply_bc_p(p, t, setup; kwargs...) = apply_bc_p!(copy(p), t, setup; kwargs...) +apply_bc_temp(temp, t, setup; kwargs...) = apply_bc_temp!(copy(temp), t, setup; kwargs...) ChainRulesCore.rrule(::typeof(apply_bc_u), u, t, setup; kwargs...) = ( apply_bc_u(u, t, setup; kwargs...), @@ -213,6 +216,46 @@ function apply_bc_p_pullback!(φbar, t, setup; kwargs...) φbar end +function apply_bc_temp!(temp, t, setup; kwargs...) + (; temperature, grid) = setup + (; boundary_conditions) = temperature + (; dimension) = grid + D = dimension() + for β = 1:D + apply_bc_temp!(boundary_conditions[β][1], temp, β, t, setup; isright = false) + apply_bc_temp!(boundary_conditions[β][2], temp, β, t, setup; isright = true) + end + temp +end + +function apply_bc_temp_pullback!(φbar, t, setup; kwargs...) + (; temperature, grid) = setup + (; boundary_conditions) = temperature + (; dimension) = grid + D = dimension() + for β = 1:D + apply_bc_temp_pullback!( + boundary_conditions[β][1], + φbar, + β, + t, + setup; + isright = false, + kwargs..., + ) + apply_bc_temp_pullback!( + boundary_conditions[β][2], + φbar, + β, + t, + setup; + isright = true, + kwargs..., + ) + end + φbar +end + function apply_bc_u!(::PeriodicBC, u, β, t, setup; isright, kwargs...) (; grid, workgroupsize) = setup (; dimension, N) = grid @@ -309,6 +352,9 @@ function apply_bc_p_pullback!(::PeriodicBC, φbar, β, t, setup; isright, kwargs φbar end +apply_bc_temp!(bc::PeriodicBC, temp, β, t, setup; isright, kwargs...) = + apply_bc_p!(bc, temp, β, t, setup; isright, kwargs...) + function apply_bc_u!(bc::DirichletBC, u, β, t, setup; isright, dudt = false, kwargs...) (; dimension, x, xp, N) = setup.grid D = dimension() @@ -361,6 +407,19 @@ end # apply_bc_p_pullback!(::DirichletBC, φbar, β, t, setup; isright, kwargs...) = # @not_implemented("DirichletBC pullback not yet implemented.") +function apply_bc_temp!(bc::DirichletBC, temp, β, t, setup; isright, kwargs...) + (; dimension, N) = setup.grid + D = dimension() + e = Offset{D}() + I = if isright + CartesianIndices(ntuple(γ -> γ == β ? (N[γ]:N[γ]) : (1:N[γ]), D)) + else + CartesianIndices(ntuple(γ -> γ == β ? (1:1) : (1:N[γ]), D)) + end + temp[I] .= bc.u + temp +end + function apply_bc_u!(::SymmetricBC, u, β, t, setup; isright, kwargs...) (; dimension, N) = setup.grid D = dimension() @@ -399,6 +458,9 @@ end # apply_bc_p_pullback!(::SymmetricBC, φbar, β, t, setup; isright, kwargs...) = # @not_implemented("SymmetricBC pullback not yet implemented.") +apply_bc_temp!(bc::SymmetricBC, temp, β, t, setup; isright, kwargs...) = + apply_bc_p!(bc, temp, β, t, setup; isright, kwargs...) + function apply_bc_u!(bc::PressureBC, u, β, t, setup; isright, kwargs...) (; grid, workgroupsize) = setup (; dimension, N, Nu, Iu) = grid diff --git a/src/operators.jl b/src/operators.jl index f4684eb07..89395ca6c 100644 --- a/src/operators.jl +++ b/src/operators.jl @@ -681,6 +681,80 @@ end # (NoTangent(), convectiondiffusion_adjoint!(similar.(u), φ, setup), NoTangent()), # ) +function convection_diffusion_temp!(c, u, temp, setup) + (; grid, workgroupsize, temperature) = setup + (; dimension, Δ, Δu, Np, Ip) = grid + (; α4) = temperature + D = dimension() + e = Offset{D}() + @kernel function conv!(c, u, temp, ::Val{βrange}, I0) where {βrange} + I = @index(Global, Cartesian) + I = I + I0 + cI = zero(eltype(c)) + KernelAbstractions.Extras.LoopInfo.@unroll for β in βrange + # TODO: Add interpolation weights + ∂T∂x1 = (temp[I] - temp[I-e(β)]) / Δu[β][I[β]-1] + ∂T∂x2 = (temp[I+e(β)] - temp[I]) / Δu[β][I[β]] + uT1 = u[β][I-e(β)] * (temp[I] + temp[I-e(β)]) / 2 + uT2 = u[β][I] * (temp[I+e(β)] + temp[I]) / 2 + cI += (-(uT2 - uT1) + α4 * (∂T∂x2 - ∂T∂x1)) / Δ[β][I[β]] + end + c[I] = cI + end + I0 = first(Ip) + I0 -= oneunit(I0) + conv!(get_backend(c), workgroupsize)(c, u, temp, Val(1:D), I0; ndrange = Np) + c +end + +# function dissipation!(c, u, setup) +# (; grid, workgroupsize, temperature) = setup +# (; dimension, Δ, Np, Ip) = grid +# D = dimension() +# e = Offset{D}() +# @inline ∂2(u, α, β, I) = ((u[α][I+e(β)] - u[α][I]) / Δ[β][I])^2 / 2 +# @inline Φ(u, α, β, I) = -∂2(u, α, β, I) - ∂2(u, α, β, I+e(β)) +# @kernel function diss!(d, u, ::Val{βrange}, I0) where {βrange} +# I = @index(Global, Cartesian) +# I = I + I0 +# cI = zero(eltype(c)) +# KernelAbstractions.Extras.LoopInfo.@unroll for β in βrange +# cI += Φ(u, β, β, I) / Δ[β][I[β]] +# end +# c[I] += cI +# end +# end + +function dissipation!(diss, diff, u, setup) + (; grid, workgroupsize, Re, temperature) = setup + (; dimension, Δ, Np, Ip) = grid + (; α1, γ) = temperature + D = dimension() + e = Offset{D}() + fill!.(diff, 0) + diffusion!(diff, u, setup) + @kernel function interpolate!(diss, diff, u, I0, ::Val{βrange}) where {βrange} + I = @index(Global, Cartesian) + I += I0 + d = zero(eltype(diss)) + KernelAbstractions.Extras.LoopInfo.@unroll for β in βrange + d += Re * α1 / γ * (u[β][I] * diff[β][I] + u[β][I-e(β)] * diff[β][I-e(β)]) / 2 + end + diss[I] += d + end + I0 = first(Ip) + I0 -= oneunit(I0) + interpolate!(get_backend(diss), workgroupsize)( + diss, + diff, + u, + I0, + Val(1:D); + ndrange = Np, + ) + diss +end + """ bodyforce!(F, u, t, setup) @@ -721,13 +795,36 @@ ChainRulesCore.rrule(::typeof(bodyforce), u, t, setup) = (bodyforce(u, t, setup), φ -> (NoTangent(), ZeroTangent(), NoTangent(), NoTangent())) """ - momentum!(F, u, t, setup) + gravity!(F, temp, setup) + +Compute gravity term (add to existing `F`). +""" +function gravity!(F, temp, setup) + (; grid, workgroupsize, temperature) = setup + (; dimension, Δ, Δu, Nu, Iu) = grid + (; gdir, α2) = temperature + D = dimension() + e = Offset{D}() + @kernel function g!(F, temp, ::Val{gdir}, I0) where {gdir} + I = @index(Global, Cartesian) + I = I + I0 + # TODO: Add interpolation weights + F[gdir][I] += α2 * (temp[I+e(gdir)] + temp[I]) / 2 + end + I0 = first(Iu[gdir]) + I0 -= oneunit(I0) + g!(get_backend(F[1]), workgroupsize)(F, temp, Val(gdir), I0; ndrange = Nu[gdir]) + F +end + +""" + momentum!(F, u, temp, t, setup) Right hand side of momentum equations, excluding pressure gradient. Put the result in ``F``. """ -function momentum!(F, u, t, setup) - (; grid, closure_model) = setup +function momentum!(F, u, temp, t, setup) + (; grid, closure_model, temperature) = setup (; dimension) = grid D = dimension() for α = 1:D @@ -737,6 +834,7 @@ function momentum!(F, u, t, setup) # convection!(F, u, setup) convectiondiffusion!(F, u, setup) bodyforce!(F, u, t, setup) + isnothing(temp) || gravity!(F, temp, setup) F end @@ -749,11 +847,11 @@ end # (tupleadd(u...), φ -> (NoTangent(), map(u -> φ, u)...)) """ - momentum(u, t, setup) + momentum(u, temp, t, setup) Right hand side of momentum equations, excluding pressure gradient. """ -function momentum(u, t, setup) +function momentum(u, temp, t, setup) (; grid, closure_model) = setup (; dimension) = grid D = dimension() @@ -765,10 +863,14 @@ function momentum(u, t, setup) # end F = @. d + c + f # F = tupleadd(d, c, f) + if !isnothing(temp) + g = gravity(temp, setup) + F = @. F + g + end F end -# ChainRulesCore.rrule(::typeof(momentum), u, t, setup) = ( +# ChainRulesCore.rrule(::typeof(momentum), u, temp, t, setup) = ( # (error(); momentum(u, t, setup)), # φ -> ( # NoTangent(), diff --git a/src/processors.jl b/src/processors.jl index a06988a3d..aaff21a2d 100644 --- a/src/processors.jl +++ b/src/processors.jl @@ -284,7 +284,7 @@ function fieldplot( xf = Array.(getindex.(setup.grid.xp, Ip.indices)) - (; u, t) = state[] + (; u, temp, t) = state[] _f = if fieldname in (1, 2) up = interpolate_u_p(u, setup) up[fieldname] @@ -310,6 +310,8 @@ function fieldplot( elseif fieldname == :V2 B, V = tensorbasis(u, setup) V[2] + elseif fieldname == :temperature + temp end _f = Array(_f)[Ip] field = lift(state) do (; u, t) @@ -334,6 +336,8 @@ function fieldplot( elseif fieldname == :V2 tensorbasis!(B, V, u, setup) -V[2] + elseif fieldname == :temperature + temp end # Array(f)[Ip] copyto!(_f, view(f, Ip)) @@ -412,7 +416,13 @@ function fieldplot( (; xlims, x, xp, Ip) = grid xf = Array.(getindex.(setup.grid.xp, Ip.indices)) + dxf = diff.(xf) + if all(α -> all(≈(dxf[α][1]), dxf[α]), 1:3) + xf = ntuple(α -> LinRange(xf[α][1], xf[α][end], length(xf[α])), 3) + end + (; u) = state[] + T = eltype(xf[1]) if fieldname == :velocity elseif fieldname == :vorticity elseif fieldname == :streamfunction @@ -444,11 +454,12 @@ function fieldplot( idx = parse(Int, string(fieldname)[2:end]) tb = tensorbasis(u, setup) tb[sym][idx] + elseif fieldname == :temperature else error("Unknown fieldname") end - field = lift(state) do (; u, t) + field = lift(state) do (; u, temp, t) f = if fieldname == :velocity up = interpolate_u_p(u, setup) map((u, v, w) -> √sum(u^2 + v^2 + w^2), up...) @@ -479,10 +490,19 @@ function fieldplot( union(Symbol.(["B$i" for i = 1:11]), Symbol.(["V$i" for i = 1:5])) tensorbasis!(tb..., u, setup) tb[sym][idx] + elseif fieldname == :temperature + temp end Array(f)[Ip] end + # color = lift(state) do (; temp) + # Array(view(temp, Ip)) + # end + # colorrange = lift(state) do (; temp) + # extrema(view(temp, Ip)) + # end + # lims = @lift get_lims($field) lims = isnothing(levels) ? lift(get_lims, field) : extrema(levels) @@ -498,6 +518,8 @@ function fieldplot( xf..., field; levels, + # color, + # colorrange, colorrange = lims, # colorrange = extrema(levels), alpha, @@ -506,7 +528,6 @@ function fieldplot( # lowclip = :red, kwargs..., ) - docolorbar && Colorbar(fig[1, 2], hm) fig end diff --git a/src/setup.jl b/src/setup.jl index 2f9c87f11..f299c7166 100644 --- a/src/setup.jl +++ b/src/setup.jl @@ -8,6 +8,7 @@ closure_model = nothing, ArrayType = Array, workgroupsize = 64, + temperature = nothing, ) Create setup. @@ -22,6 +23,7 @@ function Setup( projectorder = :last, ArrayType = Array, workgroupsize = 64, + temperature = nothing, ) setup = (; grid = Grid(x, boundary_conditions; ArrayType), @@ -34,6 +36,7 @@ function Setup( ArrayType, T = eltype(x[1]), workgroupsize, + temperature, ) if !isnothing(bodyforce) && issteadybodyforce (; dimension, x, N) = setup.grid @@ -45,3 +48,47 @@ function Setup( end setup end + +""" + temperature_equation(; + Pr, + Ra, + Ge, + dodissipation = true, + boundary_conditions, + gdir = 2, + ) + +Temperature equation setup. +""" +function temperature_equation(; + Pr, + Ra, + Ge, + dodissipation = true, + boundary_conditions, + gdir = 2, + nondim_type = 1, +) + if nondim_type == 1 + # free fall velocity, uref = sqrt(beta*g*Delta T*H) + α1 = sqrt(Pr / Ra) + α2 = eltype(Pr)(1) + α3 = Ge * sqrt(Pr / Ra) + α4 = 1 / sqrt(Pr * Ra) + elseif nondim_type == 2 + # uref = kappa/H (based on heat conduction time scale) + α1 = Pr + α2 = Pr * Ra + α3 = Ge / Ra + α4 = 1 + elseif nondim_type == 3 + # uref = sqrt(c*Delta T) + α1 = sqrt(Pr * Ge / Ra) + α2 = Ge + α3 = sqrt(Pr * Ge / Ra) + α4 = sqrt(Ge / (Pr * Ra)) + end + γ = α1 / α3 + (; α1, α2, α3, α4, γ, dodissipation, boundary_conditions, gdir) +end diff --git a/src/solvers/solve_unsteady.jl b/src/solvers/solve_unsteady.jl index 144084b30..1f1aabfed 100644 --- a/src/solvers/solve_unsteady.jl +++ b/src/solvers/solve_unsteady.jl @@ -1,8 +1,9 @@ """ - solve_unsteady( + solve_unsteady(; setup, - u₀, - tlims; + tlims + ustart, + tempstart = nothing, method = RKMethods.RK44(; T = eltype(u₀[1])), psolver = default_psolver(setup), Δt = zero(eltype(u₀[1])), @@ -28,43 +29,46 @@ the host using `Array(u)` in the processor. Return `(; u, t), outputs`, where `outputs` is a named tuple with the outputs of `processors` with the same field names. """ -function solve_unsteady( +function solve_unsteady(; setup, - u₀, - tlims; - method = RKMethods.RK44(; T = eltype(u₀[1])), + tlims, + ustart, + tempstart = nothing, + method = RKMethods.RK44(; T = eltype(ustart[1])), psolver = default_psolver(setup), - Δt = zero(eltype(u₀[1])), + Δt = zero(eltype(ustart[1])), cfl = 1, n_adapt_Δt = 1, docopy = true, processors = (;), θ = nothing, ) - docopy && (u₀ = copy.(u₀)) + docopy && (ustart = copy.(ustart)) + docopy && !isnothing(tempstart) && (tempstart = copy(tempstart)) - t_start, t_end = tlims + tstart, tend = tlims isadaptive = isnothing(Δt) # Cache arrays for intermediate computations - cache = ode_method_cache(method, setup, u₀) + cache = ode_method_cache(method, setup, ustart, tempstart) # Time stepper - stepper = create_stepper(method; setup, psolver, u = u₀, t = t_start) + stepper = + create_stepper(method; setup, psolver, u = ustart, temp = tempstart, t = tstart) # Initialize processors for iteration results state = Observable(get_state(stepper)) initialized = (; (k => v.initialize(state) for (k, v) in pairs(processors))...) if isadaptive - while stepper.t < t_end + while stepper.t < tend if stepper.n % n_adapt_Δt == 0 # Change timestep based on operators Δt = get_timestep(stepper, cfl) end # Make sure not to step past `t_end` - Δt = min(Δt, t_end - stepper.t) + Δt = min(Δt, tend - stepper.t) # Perform a single time step with the time integration method stepper = timestep!(method, stepper, Δt; θ, cache) @@ -73,8 +77,8 @@ function solve_unsteady( state[] = get_state(stepper) end else - nstep = round(Int, (t_end - t_start) / Δt) - Δt = (t_end - t_start) / nstep + nstep = round(Int, (tend - tstart) / Δt) + Δt = (tend - tstart) / nstep for it = 1:nstep # Perform a single time step with the time integration method stepper = timestep!(method, stepper, Δt; θ, cache) @@ -85,7 +89,7 @@ function solve_unsteady( end # Final state - (; u, t) = stepper + (; u, temp, t) = stepper # Processor outputs outputs = (; @@ -93,10 +97,10 @@ function solve_unsteady( ) # Return state and outputs - (; u, t), outputs + (; u, temp, t), outputs end function get_state(stepper) - (; u, t, n) = stepper - (; u, t, n) + (; u, temp, t, n) = stepper + (; u, temp, t, n) end diff --git a/src/time_steppers/step_explicit_runge_kutta.jl b/src/time_steppers/step_explicit_runge_kutta.jl index bf2f77751..0c5459305 100644 --- a/src/time_steppers/step_explicit_runge_kutta.jl +++ b/src/time_steppers/step_explicit_runge_kutta.jl @@ -1,12 +1,12 @@ -create_stepper(::ExplicitRungeKuttaMethod; setup, psolver, u, t, n = 0) = - (; setup, psolver, u, t, n) +create_stepper(::ExplicitRungeKuttaMethod; setup, psolver, u, temp, t, n = 0) = + (; setup, psolver, u, temp, t, n) function timestep!(method::ExplicitRungeKuttaMethod, stepper, Δt; θ = nothing, cache) - (; setup, psolver, u, t, n) = stepper - (; grid, closure_model) = setup + (; setup, psolver, u, temp, t, n) = stepper + (; grid, closure_model, temperature) = setup (; dimension, Iu) = grid (; A, b, c) = method - (; u₀, ku, div, p) = cache + (; u₀, ku, div, p, temp₀, ktemp, diff) = cache D = dimension() nstage = length(b) m = closure_model @@ -14,11 +14,18 @@ function timestep!(method::ExplicitRungeKuttaMethod, stepper, Δt; θ = nothing, # Update current solution t₀ = t copyto!.(u₀, u) + isnothing(temp) || copyto!(temp₀, temp) for i = 1:nstage # Compute force at current stage i apply_bc_u!(u, t, setup) - momentum!(ku[i], u, t, setup) + isnothing(temp) || apply_bc_temp!(temp, t, setup) + momentum!(ku[i], u, temp, t, setup) + if !isnothing(temp) + ktemp[i] .= 0 + convection_diffusion_temp!(ktemp[i], u, temp, setup) + temperature.dodissipation && dissipation!(ktemp[i], diff, u, setup) + end # Add closure term isnothing(m) || map((k, m) -> k .+= m, ku[i], m(u, θ)) @@ -34,6 +41,9 @@ function timestep!(method::ExplicitRungeKuttaMethod, stepper, Δt; θ = nothing, # @. u[α][Iu[α]] += Δt * A[i, j] * ku[j][α][Iu[α]] end end + for j = 1:i + @. temp .= temp₀ + Δt * A[i, j] * ktemp[j] + end # Project stage u directly # Make velocity divergence free at time t @@ -45,13 +55,14 @@ function timestep!(method::ExplicitRungeKuttaMethod, stepper, Δt; θ = nothing, # since we divide by an infinitely thin (eps(T)) volume width in the # diffusion term apply_bc_u!(u, t, setup) + isnothing(temp) || apply_bc_temp!(temp, t, setup) - create_stepper(method; setup, psolver, u, t, n = n + 1) + create_stepper(method; setup, psolver, u, temp, t, n = n + 1) end function timestep(method::ExplicitRungeKuttaMethod, stepper, Δt; θ = nothing) - (; setup, psolver, u, t, n) = stepper - (; grid, closure_model) = setup + (; setup, psolver, u, temp, t, n) = stepper + (; grid, closure_model, temperature) = setup (; dimension) = grid (; A, b, c) = method D = dimension() @@ -66,7 +77,7 @@ function timestep(method::ExplicitRungeKuttaMethod, stepper, Δt; θ = nothing) for i = 1:nstage # Compute force at current stage i u = apply_bc_u(u, t, setup) - F = momentum(u, t, setup) + F = momentum(u, temp, t, setup) # Add closure term isnothing(m) || (F = F .+ m(u, θ)) @@ -95,5 +106,5 @@ function timestep(method::ExplicitRungeKuttaMethod, stepper, Δt; θ = nothing) # diffusion term u = apply_bc_u(u, t, setup) - create_stepper(method; setup, psolver, u, t, n = n + 1) + create_stepper(method; setup, psolver, u, temp, t, n = n + 1) end diff --git a/src/time_steppers/time_stepper_caches.jl b/src/time_steppers/time_stepper_caches.jl index b9ee9e741..6c17cfa8d 100644 --- a/src/time_steppers/time_stepper_caches.jl +++ b/src/time_steppers/time_stepper_caches.jl @@ -32,13 +32,22 @@ function ode_method_cache(::OneLegMethod{T}, setup, u) where {T} (; unew, pnew, F, div, Δp) end -function ode_method_cache(method::ExplicitRungeKuttaMethod{T}, setup, u) where {T} +function ode_method_cache(method::ExplicitRungeKuttaMethod, setup, u, temp) u₀ = zero.(u) ns = nstage(method) ku = [zero.(u) for i = 1:ns] div = zero(u[1]) p = zero(u[1]) - (; u₀, ku, div, p) + if isnothing(temp) + temp₀ = nothing + ktemp = nothing + diff = nothing + else + temp₀ = copy(temp) + ktemp = [copy(temp) for i = 1:ns] + diff = zero.(u) + end + (; u₀, ku, div, p, temp₀, ktemp, diff) end function ode_method_cache(method::ImplicitRungeKuttaMethod{T}, setup, V, p) where {T} From dea771c032d031dea90352698fd84e6a79c21908 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= <40632532+agdestein@users.noreply.github.com> Date: Mon, 6 May 2024 21:19:31 +0200 Subject: [PATCH 369/379] Update README.md --- README.md | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/README.md b/README.md index 89e17501d..e23d07503 100644 --- a/README.md +++ b/README.md @@ -52,9 +52,7 @@ snapshot files using the `save_vtk` function, or the full time series using the IncompressibleNavierStokes also supports adding a temperature equation. -| ![](assets/examples/RB2D.mp4) | ![](assets/examples/RT3D.mp4) | -|:-:|:-:| -| [Rayleigh-Bénard (2D)](examples/RayleighBenard2D.jl) | [Rayleigh-Taylor (3D)](examples/RayleighTaylor3D.jl) | +https://github.com/agdestein/IncompressibleNavierStokes.jl/assets/40632532/e4894d97-d890-4d4a-a43a-8eb2772d5130 ## Demo From e5f305a988f498f04a8bdb5f5a29d3e154bdd65a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Mon, 6 May 2024 21:30:18 +0200 Subject: [PATCH 370/379] Add some missing temps --- examples/RayleighBenard2D.jl | 16 +++++++------- examples/RayleighTaylor2D.jl | 6 ++++-- lib/NeuralClosure/src/create_les_data.jl | 6 ++++-- lib/PaperDC/prioranalysis.jl | 4 ++-- lib/PaperDC/src/observe.jl | 4 ++-- lib/PaperDC/src/rk.jl | 16 +++++++------- src/operators.jl | 4 ++-- src/pressure.jl | 4 ++-- .../step_explicit_runge_kutta.jl | 21 +++++++++++++++++-- test/operators.jl | 2 +- 10 files changed, 51 insertions(+), 32 deletions(-) diff --git a/examples/RayleighBenard2D.jl b/examples/RayleighBenard2D.jl index d095d363a..5bcb6452f 100644 --- a/examples/RayleighBenard2D.jl +++ b/examples/RayleighBenard2D.jl @@ -1,17 +1,21 @@ #md using CairoMakie using GLMakie #!md -using CairoMakie using IncompressibleNavierStokes using CUDA, CUDSS ArrayType = CuArray +outdir = joinpath(@__DIR__, "output") + temperature = temperature_equation(; Pr = T(0.71), Ra = T(1e7), Ge = T(0.1), dodissipation = true, - boundary_conditions = ((SymmetricBC(), SymmetricBC()), (DirichletBC(1.0), DirichletBC(0.0))), + boundary_conditions = ( + (SymmetricBC(), SymmetricBC()), + (DirichletBC(1.0), DirichletBC(0.0)), + ), gdir = 2, nondim_type = 1, ) @@ -31,8 +35,6 @@ ustart = create_initial_conditions(setup, (dim, x, y) -> zero(x)); (; xp) = setup.grid; tempstart = @. max(-sin($(T(3π)) * xp[1]) / 100, 0) + (1 - xp[2]'); -# with_theme(theme_black()) do -with_theme(;) do state, outputs = solve_unsteady(; setup, ustart, @@ -42,7 +44,7 @@ state, outputs = solve_unsteady(; processors = (; # rtp = realtimeplotter(; anim = animator(; - path = "examples/output/RB2D.mp4", + path = "$outdir/RB2D.mp4", setup, nupdate = 100, fieldname = :temperature, @@ -53,7 +55,3 @@ state, outputs = solve_unsteady(; log = timelogger(; nupdate = 20), ), ); -end - -state.temp -IncompressibleNavierStokes.apply_bc_temp!(state.temp, T(0), setup) diff --git a/examples/RayleighTaylor2D.jl b/examples/RayleighTaylor2D.jl index b7e8371b1..5d294ed61 100644 --- a/examples/RayleighTaylor2D.jl +++ b/examples/RayleighTaylor2D.jl @@ -7,6 +7,8 @@ using CUDA, CUDSS T = Float32 ArrayType = CuArray +outdir = joinpath(@__DIR__, "output") + temperature = temperature_equation(; Pr = T(0.71), Ra = T(1e6), @@ -46,12 +48,12 @@ state, outputs = solve_unsteady(; processors = (; rtp = realtimeplotter(; # anim = animator(; - path = "examples/output/RT2D.mp4", + path = "$outdir/RT2D.mp4", setup, nupdate = 200, fieldname = :temperature, displayupdates = true, - size = (300, 500), + size = (400, 600), ), log = timelogger(; nupdate = 50), ), diff --git a/lib/NeuralClosure/src/create_les_data.jl b/lib/NeuralClosure/src/create_les_data.jl index 17ef41256..93125a3e4 100644 --- a/lib/NeuralClosure/src/create_les_data.jl +++ b/lib/NeuralClosure/src/create_les_data.jl @@ -40,11 +40,12 @@ function lesdatagen(dnsobs, Φ, les, compression, psolver) FΦ = zero.(Φu) c = zero.(Φu) results = (; u = fill(Array.(dnsobs[].u), 0), c = fill(Array.(dnsobs[].u), 0)) + temp = nothing on(dnsobs) do (; u, F, t) Φ(Φu, u, les, compression) apply_bc_u!(Φu, t, les) Φ(ΦF, F, les, compression) - momentum!(FΦ, Φu, t, les) + momentum!(FΦ, Φu, temp, t, les) apply_bc_u!(FΦ, t, les; dudt = true) project!(FΦ, les; psolver, div, p) for α = 1:length(u) @@ -77,9 +78,10 @@ filtersaver(dns, les, filters, compression, psolver_dns, psolver_les; nupdate = i = 1:length(les), Φ in filters ] results = (; data, t = zeros(T, 0), comptime) + temp = nothing on(state) do (; u, t, n) n % nupdate == 0 || return - momentum!(F, u, t, dns) + momentum!(F, u, temp, t, dns) apply_bc_u!(F, t, dns; dudt = true) project!(F, dns; psolver = psolver_dns, div, p) push!(results.t, t) diff --git a/lib/PaperDC/prioranalysis.jl b/lib/PaperDC/prioranalysis.jl index 32b454230..e83e66ec7 100644 --- a/lib/PaperDC/prioranalysis.jl +++ b/lib/PaperDC/prioranalysis.jl @@ -99,11 +99,11 @@ D == 2 && with_theme(; fontsize = 25) do apply_bc_u!(state.u, T(0), dns.setup) v = fil.Φ(state.u, fil.setup, fil.compression) apply_bc_u!(v, T(0), fil.setup) - Fv = momentum(v, T(0), fil.setup) + Fv = momentum(v, nothing, T(0), fil.setup) apply_bc_u!(Fv, T(0), fil.setup) PFv = project(Fv, fil.setup; psolver = fil.psolver) apply_bc_u!(PFv, T(0), fil.setup) - F = momentum(state.u, T(0), dns.setup) + F = momentum(state.u, nothing, T(0), dns.setup) apply_bc_u!(F, T(0), dns.setup) PF = project(F, dns.setup; dns.psolver) apply_bc_u!(PF, T(0), dns.setup) diff --git a/lib/PaperDC/src/observe.jl b/lib/PaperDC/src/observe.jl index 8fa53f59e..118037b69 100644 --- a/lib/PaperDC/src/observe.jl +++ b/lib/PaperDC/src/observe.jl @@ -26,7 +26,7 @@ function observe_v(dnsobs, Φ, les, compression, psolver) Φ(v, u, les, compression) apply_bc_u!(v, t, les) Φ(ΦPF, PF, les, compression) - momentum!(PFΦ, v, t, les) + momentum!(PFΦ, v, nothing, t, les) apply_bc_u!(PFΦ, t, les; dudt = true) project!(PFΦ, les; psolver, div, p) foreach(α -> c[α] .= ΦPF[α] .- PFΦ[α], 1:D) @@ -69,7 +69,7 @@ observe_u(dns, psolver_dns, filters; nupdate = 1) = on(state) do (; u, t, n) n % nupdate == 0 || return apply_bc_u!(u, t, dns) - momentum!(PF, u, t, dns) + momentum!(PF, u, nothing, t, dns) apply_bc_u!(PF, t, dns; dudt = true) project!(PF, dns; psolver = psolver_dns, div, p) dnsobs[] = (; u, PF, t) diff --git a/lib/PaperDC/src/rk.jl b/lib/PaperDC/src/rk.jl index 72e9059b4..a2a944574 100644 --- a/lib/PaperDC/src/rk.jl +++ b/lib/PaperDC/src/rk.jl @@ -16,11 +16,11 @@ end ode_method_cache(method::RKProject, setup, u) = ode_method_cache(method.rk, setup, u) -create_stepper(method::RKProject; setup, psolver, u, t, n = 0) = - create_stepper(method.rk; setup, psolver, u, t, n) +create_stepper(method::RKProject; setup, psolver, u, temp, t, n = 0) = + create_stepper(method.rk; setup, psolver, u, temp, t, n) function timestep!(method::RKProject, stepper, Δt; θ = nothing, cache) - (; setup, psolver, u, t, n) = stepper + (; setup, psolver, u, temp, t, n) = stepper (; grid, closure_model) = setup (; dimension, Iu) = grid (; rk, projectorder) = method @@ -38,7 +38,7 @@ function timestep!(method::RKProject, stepper, Δt; θ = nothing, cache) for i = 1:nstage # Compute force at current stage i apply_bc_u!(u, t, setup) - momentum!(ku[i], u, t, setup) + momentum!(ku[i], u, temp, t, setup) # Project F first if projectorder == :first @@ -80,11 +80,11 @@ function timestep!(method::RKProject, stepper, Δt; θ = nothing, cache) # diffusion term apply_bc_u!(u, t, setup) - create_stepper(method; setup, psolver, u, t, n = n + 1) + create_stepper(method; setup, psolver, u, temp, t, n = n + 1) end function timestep(method::RKProject, stepper, Δt; θ = nothing) - (; setup, psolver, u, t, n) = stepper + (; setup, psolver, u, temp, t, n) = stepper (; grid, closure_model) = setup (; dimension) = grid (; rk, projectorder) = method @@ -102,7 +102,7 @@ function timestep(method::RKProject, stepper, Δt; θ = nothing) for i = 1:nstage # Compute force at current stage i u = apply_bc_u(u, t, setup) - F = momentum(u, t, setup) + F = momentum(u, temp, t, setup) # Project F first if projectorder == :first @@ -145,5 +145,5 @@ function timestep(method::RKProject, stepper, Δt; θ = nothing) # diffusion term u = apply_bc_u(u, t, setup) - create_stepper(method; setup, psolver, u, t, n = n + 1) + create_stepper(method; setup, psolver, u, temp, t, n = n + 1) end diff --git a/src/operators.jl b/src/operators.jl index 89395ca6c..2dc9f3aa6 100644 --- a/src/operators.jl +++ b/src/operators.jl @@ -871,10 +871,10 @@ function momentum(u, temp, t, setup) end # ChainRulesCore.rrule(::typeof(momentum), u, temp, t, setup) = ( -# (error(); momentum(u, t, setup)), +# (error(); momentum(u, temp, t, setup)), # φ -> ( # NoTangent(), -# momentum_pullback!(zero.(φ), φ, u, t, setup), +# momentum_pullback!(zero.(φ), φ, u, temp, t, setup), # NoTangent(), # NoTangent(), # ), diff --git a/src/pressure.jl b/src/pressure.jl index 367cac3ab..59cb3ac92 100644 --- a/src/pressure.jl +++ b/src/pressure.jl @@ -44,7 +44,7 @@ function pressure!(p, u, t, setup; psolver, F, div) (; grid) = setup (; dimension, Iu, Ip, Ω) = grid D = dimension() - momentum!(F, u, t, setup) + momentum!(F, u, temp, t, setup) apply_bc_u!(F, t, setup; dudt = true) divergence!(div, F, setup) @. div *= Ω @@ -63,7 +63,7 @@ function pressure(u, t, setup; psolver) (; grid) = setup (; dimension, Iu, Ip, Ω) = grid D = dimension() - F = momentum(u, t, setup) + F = momentum(u, temp, t, setup) F = apply_bc_u(F, t, setup; dudt = true) div = divergence(F, setup) div = @. div * Ω diff --git a/src/time_steppers/step_explicit_runge_kutta.jl b/src/time_steppers/step_explicit_runge_kutta.jl index 0c5459305..6aa68fafc 100644 --- a/src/time_steppers/step_explicit_runge_kutta.jl +++ b/src/time_steppers/step_explicit_runge_kutta.jl @@ -41,8 +41,11 @@ function timestep!(method::ExplicitRungeKuttaMethod, stepper, Δt; θ = nothing, # @. u[α][Iu[α]] += Δt * A[i, j] * ku[j][α][Iu[α]] end end - for j = 1:i - @. temp .= temp₀ + Δt * A[i, j] * ktemp[j] + if !isnothing(temp) + temp .= temp₀ + for j = 1:i + @. temp += Δt * A[i, j] * ktemp[j] + end end # Project stage u directly @@ -73,17 +76,24 @@ function timestep(method::ExplicitRungeKuttaMethod, stepper, Δt; θ = nothing) t₀ = t u₀ = u ku = () + ktemp = () for i = 1:nstage # Compute force at current stage i u = apply_bc_u(u, t, setup) + isnothing(temp) || (temp = apply_bc_temp(temp, t, setup)) F = momentum(u, temp, t, setup) + if !isnothing(temp) + Ftemp = convection_diffusion_temp(u, temp, setup) + temperature.dodissipation && (Ftemp += dissipation(u, setup)) + end # Add closure term isnothing(m) || (F = F .+ m(u, θ)) # Store right-hand side of stage i ku = (ku..., F) + isnothing(temp) || (ktemp = (ktemp..., Ftemp)) # Intermediate time step t = t₀ + c[i] * Δt @@ -94,6 +104,12 @@ function timestep(method::ExplicitRungeKuttaMethod, stepper, Δt; θ = nothing) u = @. u + Δt * A[i, j] * ku[j] # u = tupleadd(u, @.(Δt * A[i, j] * ku[j])) end + if !isnothing(temp) + temp = temp₀ + for j = 1:i + temp = @. temp + Δt * A[i, j] * ktemp[j] + end + end # Project stage u directly # Make velocity divergence free at time t @@ -105,6 +121,7 @@ function timestep(method::ExplicitRungeKuttaMethod, stepper, Δt; θ = nothing) # since we divide by an infinitely thin (eps(T)) volume width in the # diffusion term u = apply_bc_u(u, t, setup) + isnothing(temp) || (temp = apply_bc_temp(temp, t, setup)) create_stepper(method; setup, psolver, u, temp, t, n = n + 1) end diff --git a/test/operators.jl b/test/operators.jl index ac4c5c28d..d644a455f 100644 --- a/test/operators.jl +++ b/test/operators.jl @@ -117,7 +117,7 @@ testops(dim) = @testset "Operators $(dim())D" begin end @testset "Momentum" begin - m = momentum(u, T(1), setup) + m = momentum(u, temp, T(1), setup) @test m isa Tuple @test m[1] isa Array{T} @test all(all.(!isnan, m)) From 0fcc1fa399dc9d87e7d78a39ff43b821e9bbdb41 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Mon, 6 May 2024 21:47:18 +0200 Subject: [PATCH 371/379] docs: Document temperature --- docs/make.jl | 1 + docs/src/features/temperature.md | 26 ++++++++++++++++++++++++++ src/operators.jl | 12 ++++++++++++ src/setup.jl | 5 +++++ 4 files changed, 44 insertions(+) create mode 100644 docs/src/features/temperature.md diff --git a/docs/make.jl b/docs/make.jl index 358f5d4c9..b961d031d 100644 --- a/docs/make.jl +++ b/docs/make.jl @@ -81,6 +81,7 @@ makedocs(; "Floating point precision" => "features/precision.md", "GPU Support" => "features/gpu.md", "Operators" => "features/operators.md", + "Temperature equation" => "features/temperature.md", "Large eddy simulation" => "features/les.md", "Neural closure models" => "features/closure.md", ], diff --git a/docs/src/features/temperature.md b/docs/src/features/temperature.md new file mode 100644 index 000000000..88d2a8e75 --- /dev/null +++ b/docs/src/features/temperature.md @@ -0,0 +1,26 @@ +# Temperature equation + +IncompressibleNavierStokes.jl supports adding a temperature equation, which is +coupled back to the momentum equation through a gravity term. + +To enable the temperature equation, you need to set the `temperature` keyword +in setup: + +```julia +setup = Setup( + args...; + kwargs..., + temperature = temperature_equation(; kwargs...), +) +``` + +```@docs +temperature_equation +``` + +Some operators are available for the temperature equation: +```@docs +gravity! +convection_diffusion_temp! +dissipation! +``` diff --git a/src/operators.jl b/src/operators.jl index 2dc9f3aa6..59be232ed 100644 --- a/src/operators.jl +++ b/src/operators.jl @@ -681,6 +681,12 @@ end # (NoTangent(), convectiondiffusion_adjoint!(similar.(u), φ, setup), NoTangent()), # ) +""" + convection_diffusion_temp!(c, u, temp, setup) + +Compute convection-diffusion term for the temperature equation. +Add result to `c`. +""" function convection_diffusion_temp!(c, u, temp, setup) (; grid, workgroupsize, temperature) = setup (; dimension, Δ, Δu, Np, Ip) = grid @@ -725,6 +731,12 @@ end # end # end +""" + dissipation!(diss, diff, u, setup) + +Compute dissipation term for the temperature equation. +Add result to `diss`. +""" function dissipation!(diss, diff, u, setup) (; grid, workgroupsize, Re, temperature) = setup (; dimension, Δ, Np, Ip) = grid diff --git a/src/setup.jl b/src/setup.jl index f299c7166..53b323af9 100644 --- a/src/setup.jl +++ b/src/setup.jl @@ -60,6 +60,11 @@ end ) Temperature equation setup. + +The equation is parameterized by three dimensionless numbers (Prandtl number, +Rayleigh numbe, and Gebhart number), and requires separate boundary conditions +for the `temperature` field. The `gdir` keyword specifies the direction gravity, +while `non_dim_type` specifies the type of non-dimensionalization. """ function temperature_equation(; Pr, From 42290e4c52f2171404039d9051885599ff1de855 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Mon, 6 May 2024 21:54:28 +0200 Subject: [PATCH 372/379] Add doc script --- docs/liveserve.jl | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 docs/liveserve.jl diff --git a/docs/liveserve.jl b/docs/liveserve.jl new file mode 100644 index 000000000..0c93059bc --- /dev/null +++ b/docs/liveserve.jl @@ -0,0 +1,9 @@ +push!(LOAD_PATH, "@live-server") + +using LiveServer + +servedocs(; + foldername = @__DIR__, + literate = joinpath(@__DIR__, "..", "examples"), + skip_dir = joinpath(@__DIR__, "src", "generated"), +) From ce4fb3739d1850e8e050ef4a5ee3eb7409430a37 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Mon, 6 May 2024 21:58:52 +0200 Subject: [PATCH 373/379] docs: Minor changes --- docs/src/features/temperature.md | 7 ++++++- src/setup.jl | 2 +- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/docs/src/features/temperature.md b/docs/src/features/temperature.md index 88d2a8e75..53efc0f45 100644 --- a/docs/src/features/temperature.md +++ b/docs/src/features/temperature.md @@ -1,7 +1,12 @@ +```@meta +CurrentModule = IncompressibleNavierStokes +``` + # Temperature equation IncompressibleNavierStokes.jl supports adding a temperature equation, which is -coupled back to the momentum equation through a gravity term. +coupled back to the momentum equation through a gravity term +[Sanderse2023](@cite). To enable the temperature equation, you need to set the `temperature` keyword in setup: diff --git a/src/setup.jl b/src/setup.jl index 53b323af9..542c72950 100644 --- a/src/setup.jl +++ b/src/setup.jl @@ -62,7 +62,7 @@ end Temperature equation setup. The equation is parameterized by three dimensionless numbers (Prandtl number, -Rayleigh numbe, and Gebhart number), and requires separate boundary conditions +Rayleigh number, and Gebhart number), and requires separate boundary conditions for the `temperature` field. The `gdir` keyword specifies the direction gravity, while `non_dim_type` specifies the type of non-dimensionalization. """ From 350b4d5a71f6126870e2aa7d093bd6e1bf662f10 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Mon, 6 May 2024 22:10:03 +0200 Subject: [PATCH 374/379] Fix: Update signatures --- README.md | 7 +++--- examples/Actuator2D.jl | 9 +++---- examples/Actuator3D.jl | 8 +++--- examples/BackwardFacingStep2D.jl | 8 +++--- examples/BackwardFacingStep3D.jl | 8 +++--- examples/DecayingTurbulence2D.jl | 8 +++--- examples/DecayingTurbulence3D.jl | 8 +++--- examples/LidDrivenCavity2D.jl | 4 +-- examples/LidDrivenCavity3D.jl | 8 +++--- examples/MultiActuator.jl | 9 +++---- examples/PlanarMixing2D.jl | 8 +++--- examples/PlaneJets2D.jl | 8 +++--- examples/Project.toml | 2 ++ examples/ShearLayer2D.jl | 8 +++--- examples/TaylorGreenVortex2D.jl | 4 +-- examples/TaylorGreenVortex3D.jl | 8 +++--- lib/PaperDC/postanalysis.jl | 42 ++++++++++++++++---------------- lib/PaperDC/prioranalysis.jl | 8 +++--- src/solvers/solve_unsteady.jl | 2 +- 19 files changed, 83 insertions(+), 84 deletions(-) diff --git a/README.md b/README.md index e23d07503..c3258efe2 100644 --- a/README.md +++ b/README.md @@ -105,12 +105,11 @@ bodyforce(dim, x, y, t) = dim() == 1 && inside(x, y) ? -1.82 : 0.0 setup = Setup(x, y; Re = 100.0, boundary_conditions, bodyforce); # Initial conditions (extend inflow) -u₀ = create_initial_conditions(setup, (dim, x, y) -> dim() == 1 ? 1.0 : 0.0); +ustart = create_initial_conditions(setup, (dim, x, y) -> dim() == 1 ? 1.0 : 0.0); # Solve unsteady Navier-Stokes equations -solve_unsteady( - setup, u₀, (0.0, 12.0); - Δt = 0.05, +solve_unsteady(; + setup, ustart, tlims = (0.0, 12.0), Δt = 0.05, processors = ( anim = animator(; setup, path = "vorticity.mp4", nupdate = 4), log = timelogger(), diff --git a/examples/Actuator2D.jl b/examples/Actuator2D.jl index b93dbf62a..62f4dc5ba 100644 --- a/examples/Actuator2D.jl +++ b/examples/Actuator2D.jl @@ -55,14 +55,13 @@ bodyforce(dim, x, y, t) = dim() == 1 ? -cₜ * inside(x, y) : 0.0 setup = Setup(x, y; Re = 100.0, boundary_conditions, bodyforce); # Initial conditions (extend inflow) -u₀ = create_initial_conditions(setup, (dim, x, y) -> dim() == 1 ? 1.0 : 0.0); -u = u₀ +ustart = create_initial_conditions(setup, (dim, x, y) -> dim() == 1 ? 1.0 : 0.0); # Solve unsteady problem -state, outputs = solve_unsteady( +state, outputs = solve_unsteady(; setup, - u₀, - (0.0, 12.0); + ustart, + tlims, (0.0, 12.0), method = RKMethods.RK44P2(), Δt = 0.05, processors = ( diff --git a/examples/Actuator3D.jl b/examples/Actuator3D.jl index 941259c34..01792dd38 100644 --- a/examples/Actuator3D.jl +++ b/examples/Actuator3D.jl @@ -72,13 +72,13 @@ bodyforce(dim, x, y, z) = dim() == 1 ? -cₜ * inside(x, y, z) : zero(x) setup = Setup(x, y, z; Re, boundary_conditions, bodyforce, ArrayType); # Initial conditions (extend inflow) -u₀ = create_initial_conditions(setup, (dim, x, y, z) -> dim() == 1 ? one(x) : zero(x)); +ustart = create_initial_conditions(setup, (dim, x, y, z) -> dim() == 1 ? one(x) : zero(x)); # Solve unsteady problem -(; u, t), outputs = solve_unsteady( +(; u, t), outputs = solve_unsteady(; setup, - u₀, - (T(0), T(3)); + ustart, + tlims = (T(0), T(3)), method = RKMethods.RK44P2(), Δt = T(0.05), processors = ( diff --git a/examples/BackwardFacingStep2D.jl b/examples/BackwardFacingStep2D.jl index 2e809e2ce..a93023193 100644 --- a/examples/BackwardFacingStep2D.jl +++ b/examples/BackwardFacingStep2D.jl @@ -53,17 +53,17 @@ plotgrid(x, y) setup = Setup(x, y; Re, boundary_conditions, ArrayType); # Initial conditions (extend inflow) -u₀ = create_initial_conditions(setup, (dim, x, y) -> U(dim, x, y, zero(x))); +ustart = create_initial_conditions(setup, (dim, x, y) -> U(dim, x, y, zero(x))); # Solve steady state problem ## u, p = solve_steady_state(setup, u₀, p₀); nothing # Solve unsteady problem -state, outputs = solve_unsteady( +state, outputs = solve_unsteady(; setup, - u₀, - (T(0), T(7)); + ustart, + tlims = (T(0), T(7)), Δt = T(0.002), processors = ( rtp = realtimeplotter(; diff --git a/examples/BackwardFacingStep3D.jl b/examples/BackwardFacingStep3D.jl index e57741c95..72ba438d8 100644 --- a/examples/BackwardFacingStep3D.jl +++ b/examples/BackwardFacingStep3D.jl @@ -55,17 +55,17 @@ boundary_conditions = ( setup = Setup(x, y, z; Re, boundary_conditions, ArrayType); # Initial conditions (extend inflow) -u₀ = create_initial_conditions(setup, (dim, x, y, z) -> U(dim, x, y, z, zero(x))); +ustart = create_initial_conditions(setup, (dim, x, y, z) -> U(dim, x, y, z, zero(x))); # Solve steady state problem ## u, p = solve_steady_state(setup, u₀, p₀); nothing # Solve unsteady problem -state, outputs = solve_unsteady( +state, outputs = solve_unsteady(; setup, - u₀, - (T(0), T(7)); + ustart, + tlims = (T(0), T(7)), Δt = T(0.01), processors = ( rtp = realtimeplotter(; diff --git a/examples/DecayingTurbulence2D.jl b/examples/DecayingTurbulence2D.jl index ba341f818..e3d5beb9f 100644 --- a/examples/DecayingTurbulence2D.jl +++ b/examples/DecayingTurbulence2D.jl @@ -35,13 +35,13 @@ x = LinRange(lims..., n + 1), LinRange(lims..., n + 1) setup = Setup(x...; Re, ArrayType); # Create random initial conditions -u₀ = random_field(setup, T(0)); +ustart = random_field(setup, T(0)); # Solve unsteady problem -state, outputs = solve_unsteady( +state, outputs = solve_unsteady(; setup, - u₀, - (T(0), T(1)); + ustart, + tlims = (T(0), T(1)), Δt = T(1e-3), processors = ( ## rtp = realtimeplotter(; setup, nupdate = 1), diff --git a/examples/DecayingTurbulence3D.jl b/examples/DecayingTurbulence3D.jl index 403e9118e..684177111 100644 --- a/examples/DecayingTurbulence3D.jl +++ b/examples/DecayingTurbulence3D.jl @@ -41,13 +41,13 @@ setup = Setup(x, y, z; Re, ArrayType); psolver = psolver_spectral(setup); # Initial conditions -u₀ = random_field(setup; psolver); +ustart = random_field(setup; psolver); # Solve unsteady problem -(; u, t), outputs = solve_unsteady( +(; u, t), outputs = solve_unsteady(; setup, - u₀, - (T(0), T(1)); + ustart, + tlims = (T(0), T(1)), Δt = T(1e-3), psolver, processors = ( diff --git a/examples/LidDrivenCavity2D.jl b/examples/LidDrivenCavity2D.jl index a187ffa10..fa61deef2 100644 --- a/examples/LidDrivenCavity2D.jl +++ b/examples/LidDrivenCavity2D.jl @@ -80,7 +80,7 @@ setup = Setup(x, y; boundary_conditions, Re, ArrayType); # The initial conditions are provided in function. The value `dim()` determines # the velocity component. -u₀ = create_initial_conditions(setup, (dim, x, y) -> zero(x)); +ustart = create_initial_conditions(setup, (dim, x, y) -> zero(x)); # ## Solve problems # @@ -111,7 +111,7 @@ processors = ( # By default, a standard fourth order Runge-Kutta method is used. If we don't # provide the time step explicitly, an adaptive time step is used. tlims = (T(0), T(10)) -state, outputs = solve_unsteady(setup, u₀, tlims; Δt = T(1e-3), processors); +state, outputs = solve_unsteady(; setup, ustart, tlims; Δt = T(1e-3), processors); # ## Post-process # diff --git a/examples/LidDrivenCavity3D.jl b/examples/LidDrivenCavity3D.jl index d49a66d5e..981118614 100644 --- a/examples/LidDrivenCavity3D.jl +++ b/examples/LidDrivenCavity3D.jl @@ -49,13 +49,13 @@ boundary_conditions = ( setup = Setup(x, y, z; Re, boundary_conditions, ArrayType); # Initial conditions -u₀ = create_initial_conditions(setup, (dim, x, y, z) -> zero(x)) +ustart = create_initial_conditions(setup, (dim, x, y, z) -> zero(x)) # Solve unsteady problem -(; u, t), outputs = solve_unsteady( +(; u, t), outputs = solve_unsteady(; setup, - u₀, - (T(0), T(0.2)); + ustart, + tlims = (T(0), T(0.2)), Δt = T(1e-3), processors = ( ## rtp = realtimeplotter(; setup, plot = fieldplot, nupdate = 50), diff --git a/examples/MultiActuator.jl b/examples/MultiActuator.jl index 98ddf40d4..99ea04cf7 100644 --- a/examples/MultiActuator.jl +++ b/examples/MultiActuator.jl @@ -80,8 +80,7 @@ y = LinRange(-T(2), T(2), 2n + 1) setup = Setup(x, y; Re = T(2000), boundary_conditions, bodyforce, ArrayType); # Initial conditions (extend inflow) -u₀ = create_initial_conditions(setup, (dim, x, y) -> dim() == 1 ? one(x) : zero(x)); -u = u₀ +ustart = create_initial_conditions(setup, (dim, x, y) -> dim() == 1 ? one(x) : zero(x)); t = T(0) # # We create a box to visualize the actuator. @@ -106,10 +105,10 @@ end box = boxes[1] # Solve unsteady problem -state, outputs = solve_unsteady( +state, outputs = solve_unsteady(; setup, - u₀, - (T(0), 4 * T(12)); + ustart, + tlims = (T(0), 4 * T(12)), # (T(0), T(1)); method = RKMethods.RK44P2(), Δt = T(0.01), diff --git a/examples/PlanarMixing2D.jl b/examples/PlanarMixing2D.jl index 841945789..97d3ef8f7 100644 --- a/examples/PlanarMixing2D.jl +++ b/examples/PlanarMixing2D.jl @@ -50,13 +50,13 @@ setup = Setup(x, y; Re, boundary_conditions); psolver = psolver_direct(setup); # Initial conditions (extend inflow) -u₀ = create_initial_conditions(setup, (dim, x, y) -> U(dim, x, y, 0.0); psolver); +ustart = create_initial_conditions(setup, (dim, x, y) -> U(dim, x, y, 0.0); psolver); # Solve unsteady problem -state, outputs = solve_unsteady( +state, outputs = solve_unsteady(; setup, - u₀, - (0.0, 100.0); + ustart, + tlims = (0.0, 100.0), psolver, method = RKMethods.RK44P2(), Δt = 0.1, diff --git a/examples/PlaneJets2D.jl b/examples/PlaneJets2D.jl index 3a0220eeb..2d01473c8 100644 --- a/examples/PlaneJets2D.jl +++ b/examples/PlaneJets2D.jl @@ -87,7 +87,7 @@ setup = Setup(x, y; Re, ArrayType); ## setup = Setup(x, y; Re, boundary_conditions, ArrayType); # Initial conditions -u₀ = create_initial_conditions(setup, (dim, x, y) -> dim() == 1 ? U(x, y) : zero(x)); +ustart = create_initial_conditions(setup, (dim, x, y) -> dim() == 1 ? U(x, y) : zero(x)); # Real time plot: Streamwise average and spectrum function meanplot(state; setup) @@ -145,10 +145,10 @@ function meanplot(state; setup) end # Solve unsteady problem -state, outputs = solve_unsteady( +state, outputs = solve_unsteady(; setup, - u₀, - (T(0), T(1)); + ustart, + tlims = (T(0), T(1)), method = RKMethods.RK44P2(), Δt = 0.001, processors = ( diff --git a/examples/Project.toml b/examples/Project.toml index b5cec717d..6afb14999 100644 --- a/examples/Project.toml +++ b/examples/Project.toml @@ -4,6 +4,8 @@ authors = ["Syver Døving Agdestein "] version = "0.1.0" [deps] +CUDA = "052768ef-5323-5732-b1bb-66c8b64840ba" +CUDSS = "45b445bb-4962-46a0-9369-b4df9d0f772e" CairoMakie = "13f3f980-e62b-5c42-98c6-ff1f3baf88f0" GLMakie = "e9467ef8-e4e7-5192-8a1a-b1aee30e663a" IncompressibleNavierStokes = "5e318141-6589-402b-868d-77d7df8c442e" diff --git a/examples/ShearLayer2D.jl b/examples/ShearLayer2D.jl index eed4b8007..e0870590e 100644 --- a/examples/ShearLayer2D.jl +++ b/examples/ShearLayer2D.jl @@ -44,13 +44,13 @@ d = T(π / 15) e = T(0.05) U1(y) = y ≤ π ? tanh((y - T(π / 2)) / d) : tanh((T(3π / 2) - y) / d) ## U1(y) = T(1) + (y ≤ π ? tanh((y - T(π / 2)) / d) : tanh((T(3π / 2) - y) / d)) -u₀ = create_initial_conditions(setup, (dim, x, y) -> dim() == 1 ? U1(y) : e * sin(x)); +ustart = create_initial_conditions(setup, (dim, x, y) -> dim() == 1 ? U1(y) : e * sin(x)); # Solve unsteady problem -state, outputs = solve_unsteady( +state, outputs = solve_unsteady(; setup, - u₀, - (T(0), T(8)); + ustart, + tlims = (T(0), T(8)), Δt = T(0.01), processors = ( rtp = realtimeplotter(; diff --git a/examples/TaylorGreenVortex2D.jl b/examples/TaylorGreenVortex2D.jl index d6b8370e6..4e4898771 100644 --- a/examples/TaylorGreenVortex2D.jl +++ b/examples/TaylorGreenVortex2D.jl @@ -28,7 +28,7 @@ function compute_convergence(; D, nlist, lims, Re, tlims, Δt, uref, ArrayType = x = ntuple(α -> LinRange(lims..., n + 1), D) setup = Setup(x...; Re, ArrayType) psolver = psolver_spectral(setup) - u₀ = create_initial_conditions( + ustart = create_initial_conditions( setup, (dim, x...) -> uref(dim, x..., tlims[1]), tlims[1]; @@ -41,7 +41,7 @@ function compute_convergence(; D, nlist, lims, Re, tlims, Δt, uref, ArrayType = psolver, doproject = false, ) - (; u, t), outputs = solve_unsteady(setup, u₀, tlims; Δt, psolver) + (; u, t), outputs = solve_unsteady(; setup, ustart, tlims, Δt, psolver) (; Ip) = setup.grid a, b = T(0), T(0) for α = 1:D diff --git a/examples/TaylorGreenVortex3D.jl b/examples/TaylorGreenVortex3D.jl index c9b212b8f..05ffd41e5 100644 --- a/examples/TaylorGreenVortex3D.jl +++ b/examples/TaylorGreenVortex3D.jl @@ -37,7 +37,7 @@ setup = Setup(x, y, z; Re, ArrayType); psolver = psolver_spectral(setup); # Initial conditions -u₀ = create_initial_conditions( +ustart = create_initial_conditions( setup, (dim, x, y, z) -> dim() == 1 ? sinpi(2x) * cospi(2y) * sinpi(2z) / 2 : @@ -46,10 +46,10 @@ u₀ = create_initial_conditions( ); # Solve unsteady problem -(; u, t), outputs = solve_unsteady( +(; u, t), outputs = solve_unsteady(; setup, - u₀, - (T(0), T(1.0)); + ustart, + tlims = (T(0), T(1.0)), Δt = T(1e-3), processors = ( ## rtp = realtimeplotter(; setup, plot = fieldplot, nupdate = 10), diff --git a/lib/PaperDC/postanalysis.jl b/lib/PaperDC/postanalysis.jl index 0603627c3..35bc97254 100644 --- a/lib/PaperDC/postanalysis.jl +++ b/lib/PaperDC/postanalysis.jl @@ -686,11 +686,11 @@ kineticenergy = let setup = setups_test[ig] psolver = psolver_spectral(setup) t = data_test.t - u₀ = data_test.data[ig, ifil].u[1] |> device + ustart = data_test.data[ig, ifil].u[1] |> device tlims = (t[1], t[end]) nupdate = 2 Δt = (t[2] - t[1]) / nupdate - T = eltype(u₀[1]) + T = eltype(ustart[1]) ewriter = processor() do state ehist = zeros(T, 0) on(state) do (; u, n) @@ -710,31 +710,31 @@ kineticenergy = let data_test.data[ig, ifil].u, ) ke_nomodel[ig, ifil] = - solve_unsteady(setup, u₀, tlims; Δt, processors, psolver)[2].ewriter + solve_unsteady(; setup, ustart, tlims, Δt, processors, psolver)[2].ewriter end ke_smag[ig, ifil, iorder] = - solve_unsteady( + solve_unsteady(; (; setup..., projectorder = getorder(iorder), closure_model = smagorinsky_closure(setup), ), - u₀, - tlims; + ustart, + tlims, Δt, processors, psolver, θ = θ_smag[ifil, iorder], )[2].ewriter ke_cnn_prior[ig, ifil, iorder] = - solve_unsteady( + solve_unsteady(; (; setup..., projectorder = getorder(iorder), closure_model = wrappedclosure(closure, setup), ), - u₀, - tlims; + ustart, + tlims, Δt, processors, psolver, @@ -747,8 +747,8 @@ kineticenergy = let projectorder = getorder(iorder), closure_model = wrappedclosure(closure, setup), ), - u₀, - tlims; + ustart, + tlims, Δt, processors, psolver, @@ -839,11 +839,11 @@ divs = let setup = setups_test[ig] psolver = psolver_spectral(setup) t = data_test.t - u₀ = data_test.data[ig, ifil].u[1] |> device + ustart = data_test.data[ig, ifil].u[1] |> device tlims = (t[1], t[end]) nupdate = 2 Δt = (t[2] - t[1]) / nupdate - T = eltype(u₀[1]) + T = eltype(ustart[1]) dwriter = processor() do state div = fill!(similar(setup.grid.x[1], setup.grid.N), 0) dhist = zeros(T, 0) @@ -870,10 +870,10 @@ divs = let end end s(closure_model, θ) = - solve_unsteady( + solve_unsteady(; (; setup..., closure_model), - u₀, - tlims; + ustart, + tlims, method = RKProject(RK44(; T), getorder(iorder)), Δt, processors = (; dwriter), @@ -967,16 +967,16 @@ ufinal = let t = data_test.t setup = setups_test[igrid] psolver = psolver_spectral(setup) - u₀ = data_test.data[igrid, ifil].u[1] |> device + ustart = data_test.data[igrid, ifil].u[1] |> device tlims = (t[1], t[end]) nupdate = 2 Δt = (t[2] - t[1]) / nupdate - T = eltype(u₀[1]) + T = eltype(ustart[1]) s(closure_model, θ) = - solve_unsteady( + solve_unsteady(; (; setup..., closure_model), - u₀, - tlims; + ustart, + tlims, method = RKProject(RK44(; T), getorder(iorder)), Δt, psolver, diff --git a/lib/PaperDC/prioranalysis.jl b/lib/PaperDC/prioranalysis.jl index e83e66ec7..16998f5f4 100644 --- a/lib/PaperDC/prioranalysis.jl +++ b/lib/PaperDC/prioranalysis.jl @@ -73,14 +73,14 @@ end; # Create random initial conditions rng = Random.seed!(Random.default_rng(), 12345) -u₀ = random_field(dns.setup, T(0); kp, dns.psolver, rng); +ustart = random_field(dns.setup, T(0); kp, dns.psolver, rng); clean() # Solve unsteady problem -@time state, outputs = solve_unsteady( +@time state, outputs = solve_unsteady(; dns.setup, - u₀, - (T(0), T(1e-1)); + ustart, + tlims = (T(0), T(1e-1)), Δt, docopy = true, # leave initial conditions unchanged, false to free up memory dns.psolver, diff --git a/src/solvers/solve_unsteady.jl b/src/solvers/solve_unsteady.jl index 1f1aabfed..a95445cdf 100644 --- a/src/solvers/solve_unsteady.jl +++ b/src/solvers/solve_unsteady.jl @@ -1,7 +1,7 @@ """ solve_unsteady(; setup, - tlims + tlims, ustart, tempstart = nothing, method = RKMethods.RK44(; T = eltype(u₀[1])), From a3664ceaf0a60cbdd1522f8b2205240ae54d3246 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Mon, 6 May 2024 22:15:08 +0200 Subject: [PATCH 375/379] Minor fixes --- README.md | 2 +- examples/Actuator2D.jl | 2 +- lib/PaperDC/postanalysis.jl | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index c3258efe2..813106429 100644 --- a/README.md +++ b/README.md @@ -109,7 +109,7 @@ ustart = create_initial_conditions(setup, (dim, x, y) -> dim() == 1 ? 1.0 : 0.0) # Solve unsteady Navier-Stokes equations solve_unsteady(; - setup, ustart, tlims = (0.0, 12.0), Δt = 0.05, + setup, ustart, tlims = (0.0, 48.0), Δt = 0.05, processors = ( anim = animator(; setup, path = "vorticity.mp4", nupdate = 4), log = timelogger(), diff --git a/examples/Actuator2D.jl b/examples/Actuator2D.jl index 62f4dc5ba..a1470b914 100644 --- a/examples/Actuator2D.jl +++ b/examples/Actuator2D.jl @@ -61,7 +61,7 @@ ustart = create_initial_conditions(setup, (dim, x, y) -> dim() == 1 ? 1.0 : 0.0) state, outputs = solve_unsteady(; setup, ustart, - tlims, (0.0, 12.0), + tlims = (0.0, 12.0), method = RKMethods.RK44P2(), Δt = 0.05, processors = ( diff --git a/lib/PaperDC/postanalysis.jl b/lib/PaperDC/postanalysis.jl index 35bc97254..2e018d5c4 100644 --- a/lib/PaperDC/postanalysis.jl +++ b/lib/PaperDC/postanalysis.jl @@ -741,7 +741,7 @@ kineticenergy = let θ = θ_cnn_prior[ig, ifil], )[2].ewriter ke_cnn_post[ig, ifil, iorder] = - solve_unsteady( + solve_unsteady(; (; setup..., projectorder = getorder(iorder), From 2907b3c0e2ad03a0cbf72d440207a0c2f25bc2db Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Mon, 6 May 2024 22:20:56 +0200 Subject: [PATCH 376/379] fix(docs): dois --- docs/references.bib | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/references.bib b/docs/references.bib index fe18b85a4..2a45b8834 100644 --- a/docs/references.bib +++ b/docs/references.bib @@ -160,7 +160,7 @@ @book{Sagaut2006 } @article{Sanderse2012, author = {B. Sanderse and B. Koren}, - doi = {https://doi.org/10.1016/j.jcp.2011.11.028}, + doi = {10.1016/j.jcp.2011.11.028}, issn = {0021-9991}, journal = {Journal of Computational Physics}, keywords = {Differential–algebraic equations, Incompressible Navier–Stokes equations, Temporal accuracy, Time integration, Runge–Kutta method, Moving meshes}, @@ -173,7 +173,7 @@ @article{Sanderse2012 } @article{Sanderse2013, author = {B. Sanderse}, - doi = {https://doi.org/10.1016/j.jcp.2012.07.039}, + doi = {10.1016/j.jcp.2012.07.039}, issn = {0021-9991}, journal = {Journal of Computational Physics}, keywords = {Energy conservation, Time reversibility, Runge–Kutta method, Additive Runge–Kutta method, Incompressible Navier–Stokes equations, Differential–algebraic equations, Algebraic stability, stability, Stiffness}, @@ -185,7 +185,7 @@ @article{Sanderse2013 } @article{Sanderse2014, author = {B. Sanderse and R.W.C.P. Verstappen and B. Koren}, - doi = {https://doi.org/10.1016/j.jcp.2013.10.002}, + doi = {10.1016/j.jcp.2013.10.002}, issn = {0021-9991}, journal = {Journal of Computational Physics}, keywords = {Incompressible Navier–Stokes equations, Symmetry preservation, Energy conservation, Boundary conditions, Fourth order accuracy}, From 0dab32f3fef872dba0d87219693c9ed6cecc59f0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Mon, 6 May 2024 22:42:41 +0200 Subject: [PATCH 377/379] Fix signatures --- src/pressure.jl | 8 ++++---- src/processors.jl | 10 +++++----- src/time_steppers/step_one_leg.jl | 4 ++-- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/src/pressure.jl b/src/pressure.jl index 59cb3ac92..50fc406bb 100644 --- a/src/pressure.jl +++ b/src/pressure.jl @@ -35,12 +35,12 @@ See also [`poisson`](@ref). poisson!(psolver, p, f) = psolver(p, f) """ - pressure!(p, u, t, setup; psolver, F, div) + pressure!(p, u, temp, t, setup; psolver, F, div) Compute pressure from velocity field. This makes the pressure compatible with the velocity field, resulting in same order pressure as velocity. """ -function pressure!(p, u, t, setup; psolver, F, div) +function pressure!(p, u, temp, t, setup; psolver, F, div) (; grid) = setup (; dimension, Iu, Ip, Ω) = grid D = dimension() @@ -54,12 +54,12 @@ function pressure!(p, u, t, setup; psolver, F, div) end """ - pressure(u, t, setup; psolver) + pressure(u, temp, t, setup; psolver) Compute pressure from velocity field. This makes the pressure compatible with the velocity field, resulting in same order pressure as velocity. """ -function pressure(u, t, setup; psolver) +function pressure(u, temp, t, setup; psolver) (; grid) = setup (; dimension, Iu, Ip, Ω) = grid D = dimension() diff --git a/src/processors.jl b/src/processors.jl index aaff21a2d..124913d76 100644 --- a/src/processors.jl +++ b/src/processors.jl @@ -115,7 +115,7 @@ vtk_writer(; vtk["velocity"] = uparr end if :pressure ∈ fields - pressure!(p, u, setup; psolver, F, div) + pressure!(p, u, temp, t, setup; psolver, F, div) vtk["pressure"] = copyto!(parr, p) end if :vorticity ∈ fields @@ -314,7 +314,7 @@ function fieldplot( temp end _f = Array(_f)[Ip] - field = lift(state) do (; u, t) + field = lift(state) do (; u, temp, t) f = if fieldname in (1, 2) interpolate_u_p!(up, u, setup) up[fieldname] @@ -329,7 +329,7 @@ function fieldplot( elseif fieldname == :streamfunction get_streamfunction!(setup, ψ, u, t) elseif fieldname == :pressure - pressure!(p, u, t, setup; psolver, F, div) + pressure!(p, u, temp, t, setup; psolver, F, div) elseif fieldname == :V1 tensorbasis!(B, V, u, setup) V[1] @@ -469,9 +469,9 @@ function fieldplot( elseif fieldname == :streamfunction get_streamfunction(setup, u, t) elseif fieldname == :pressure - pressure!(p, u, t, setup; psolver, F, div) + pressure!(p, u, temp, t, setup; psolver, F, div) elseif fieldname == :Dfield - pressure!(p, u, t, setup; psolver, F, div) + pressure!(p, u, temp, t, setup; psolver, F, div) Dfield!(d, G, p, setup) din = view(d, Ip) @. din = log(max(logtol, din)) diff --git a/src/time_steppers/step_one_leg.jl b/src/time_steppers/step_one_leg.jl index e84331a1c..3ed344897 100644 --- a/src/time_steppers/step_one_leg.jl +++ b/src/time_steppers/step_one_leg.jl @@ -102,7 +102,7 @@ function timestep!(method::OneLegMethod, stepper, Δt; θ = nothing, cache) if n == 0 stepper_startup = create_stepper(method_startup; setup, psolver, u, t) (; u, t, n) = timestep(method_startup, stepper_startup, Δt) - pressure!(p, u, t, setup; psolver, F, div) + pressure!(p, u, temp, t, setup; psolver, F, div) return create_stepper(method; setup, psolver, u, p, t, n, uold, pold, told) end @@ -151,7 +151,7 @@ function timestep!(method::OneLegMethod, stepper, Δt; θ = nothing, cache) # Alternatively, do an additional Poisson solve if p_add_solve - pressure!(pnew, unew, t + Δt, setup; psolver, F, div) + pressure!(pnew, unew, tempnew, t + Δt, setup; psolver, F, div) end n += 1 From 80654d5140f2b5cff90939e34bf8b9b7d26bad22 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Mon, 6 May 2024 23:26:47 +0200 Subject: [PATCH 378/379] Update docs --- docs/src/assets/grid.png | Bin 79108 -> 68810 bytes docs/src/assets/grid.svg | 1126 ------------------------------ docs/src/equations/ns.md | 121 ++-- docs/src/equations/spatial.md | 544 +++------------ docs/src/features/closure.md | 21 +- docs/src/features/gpu.md | 20 +- docs/src/features/temperature.md | 2 + 7 files changed, 192 insertions(+), 1642 deletions(-) delete mode 100644 docs/src/assets/grid.svg diff --git a/docs/src/assets/grid.png b/docs/src/assets/grid.png index ed11cd750eeb85354b40d8694a021e4f1abfa039..a40dbc86514fd1c48e370d90f1bd0880cf70efa9 100644 GIT binary patch literal 68810 zcmeFYbyQVd_dj~*PU!}vyCkGT5Jb9^I=~^N58a(o5`wfyiKKKlf)aw1G$J6~4R_)5 zymjw)j61$}+~0riVKDYSd(So3T64`c*No4JRC}g?gGqr2fk1GcD9UL-AV|az2%;=H zDtO}CBBBK@BQU)u7EhkALD0b$K#711K>&BeNZ|JGds#%#2muKK9znvuKOwlkqq+O1 zKtTC(jRE(Rf8Qs8dzQaxKvM`W0wM$lTrFIdgZ}4F5!~Hz z{>N3}iHaH>9~UnV7r!uQ&%-Am$|ESs^N@~@UzAT+l#d@!Bjo&RM0cE~3_o!KE+N*k zvT9FcW$7H99W1P0!XOZj4BrHC#V#qT9%D^gc{W@f{LT1W(P?>kDqG?xHTrSPkD_l3 zF6afU6*UZ~P$q+dX`{8#NG%Pi4N#=_kh-Tko{Y*XT^Vaw5o~riNzTuC&$b|ic%@nE z=u(Zqe>`QrXZo;?0o9m5N(hxeTFxScm;DJVoXYrgZ~vI9w^}s&(&ebwGa5l6L3jvz zjLC>OYyK0?&}BDNCsBLZW!dR9m*Q2wYtM?%kbEji7WqT|4UuJ@O1+n6?_cSE9*-!D zcvn)O*&2VWx7ysJ&#tR~m~|DJ+uYjR?V5gH(G(#%o=!%Zn3ARIag0tT5tj9gDtr&G z$uH6Xv*BTTKW`kSKz|wS6`r5FEin^wbG{lPdb99Q9}E603WZ?bxrAFrFUF!jYS~dr zn+BSN#FO%xPwNCDZ8!!7bpkNF`dNgqEbQqd!_TUkEJOjP{1*jOJdRjLWnL|Ur%o3T z^P&&qIF63!F~qFiAG?~oXS^>@pH{cg!u_2KQIyu{few0DYw5xDRGx~OIoNSQ%^gf( zTpo6gcMAp)lk{+en%TnObfz#%YkP5q-KI7MI%{)r23zS6B zm#vwIIfJAGrkIB)U|L+L#1UfR2edWbXp;VTO6?;dkA(ETBS+ln*j0X>p+aE8$d za0zhnaLRjFyYVqdVA6>>n_Gx#$UXj>1^6b;UKMCv*Z5v7A|mkH^AiY0sW6HT(mqLVcZ%p7YA2oGnl*^ z%pQLK-&vTO{j>lwT0a$!oz6<>M1F7dA69|2I}o>|Nkcdo$P_E5Mw~ z8t~!cGlfFUV5XeBrovFbLYS8mDlBNu`Ou6{@S(7vkO=gl>A$g1b+!gt34QtRqq<{d z4p@l@LU|tYKp%1n@bL3-3h?rpa+>n<2y>d6J$z_x3KfQ#TL}GOWo{<=*umKj3YOE_ z4r&SGcC@$r^Wsk6qS9(l#2NUwc>eW7?Ijd$0UC%iJhirW_4wBdEo(cNCLDSvO|!6xmy-dFc~0N=$)Pd27jJ|xroX-!=P{n zXDtVZm*Ncnw7UO%tpc`_ITQ|+gTi5e=|AnT)<5hpI}e{I&tJp8W2fR^Zf)WD|3-bc zdFaFdJy>!@YZuVJ=buM^?I=x{(_i2I`ufuP&sL(N`?D!Tp=N(g!3F9D`)j=cufJZI zSwZbBVZeI)t*-xgZv8)~0-le?|OX?fS2{{#O$CUlIRTyZ(O@7v{ghDVRO*g4{u{q^YN_2LdfrQ)LA? z$n9Os#n0?>2rd|oiux`P2-dy3e*{Q+1{rvW27jU=kG772N=!wXkMMjM0-=LEk(1W) znAw{1^e|Y3a^LKFr%YSskG6UlSL_FMu=hUvprV&ZhK>^-4N}k zbzr;GV*M6QukxD78j%wRznvbsfK7%y|C5?K@#Xfn7z^Vek=^T+n8y^hj}>bb=YmLf9TeN2n!{#?T_|EI4E|MTO2H2TN&f2R8X z(0G6!4UL|kTnG78ToO+gwEty_t8}-Tw45ArJ}zzV2Dcvp((OJiHi-aNq`0@^Z`E6( z;2*Wp6sYVW>xiRzC~|V}O5~{6wll1dY#+h|%WM<~vgAGn3l}MS_~X}vk#kL>`#szY z2eU2_b_=)|IafoE=(UoMWd(YsG(jh%5^7YUA+c>&%6HBA-pcM>L1u)nFWnQA&^BOATV?bT;7Aiuf7}8U!AR-JMBk} zVeAehk8E!0OKV-B_O6h-#^yhN?sySx+`Y){*I^XP?q<+9%%vAwhme?20B482Jmk4iYk&VDv}y&sJYY0iiAte~FOJ zwL&>;VlaD8{2b)SMQ4v55rsO)AYXh}vjBi+hI!%d5swbELl|{|7M^ zkwPrdZf^aqgm?cvB$P~8h1&Jas5r^xNe8bPYVhB};0w$fk1xF3Rk$|3CI(b5D(VlE zyjjN6TsXqQWSnCEB2zh(a$1jX85oO@%B`O(@>oO=aO}UQqEn&QNMu77@N>R}=RUzB zy`v+#27gH^6_u?@I#Jd7b7}&D%v&j5=jo~3_Y%Y?^RAnB1CK9=JgPJ+Wl0Y)r(gV0 zfzV;rH*&exRbk3|N%e(O2#a3L?S~ycg+pKj)A2jz_erIw3=EWyghu z)0lGqEjZ?#_sXMZHRT90CXPR9mrUpx4Wdck?O+)PylWsgaX?iu%^^G6P#wKgz16~J z%fKSZ+5ccHf8HmFyX06=_GjI#W>BKrZfqe^=06@Pjv?pEU45*>_d9qp?!O;>r?u+v zA@hgPH`!Onx!KnoZ3km&IaX%_x@xVi>>*{_l9Ct-sYH8HUy#p^FY(R!hciuuQ~H0B4% ztGt>sbNWs6jn;>UiUXG_fIPCjiv~+HT8ejXEIaRz-I$ezrPh9x>hBgmgXHZvXLXGt zpL##ylNBeK=Fqr2?TYFVCW>MrVv&&zT>e6@YJ46wC)*+Qm)%Kqtmyw**`L8nG_XvC zMWLFb_#WTuNF6SAqz1)9SyRL3)dD+rm7D|fm)_O_9zR9s>F*CBeal_;rDN9S&oYCS z;YK<_BYdx=*TMAF&2i;djBf>l+~<0vqy<9j-#r9YLrr$@Ys#D(We!N@@~6pdCF^3^ zlUT3YKktlroFL+gxljFQyBPOD(08T|aAF#4N`7ZEMTBqr$*&3lM^A3s(MO^IA2wgs|s zD|}y*i~|D6D{nSc9k@A2otFK3?!FagGm|;mI-O1S)NHgq_)&wdpKrXsDk7^96-g|F z_&sC66#Jml0aiYwEAXW%9$3Ht{M1Q|`^Y|<6S@1F#tOskt-)1492EAlyvbPpGBs6r zB+4l{8*djvmJbcT1$Lr`j$D6rklFWAzXSt@F?7hFTXoRkV*@S1A6OMnoNn?GOm92p z1d4?8m*%VQgf*1>PQ113UGTYgEqR6z(uJ^4Gui3ig(G%isjQx(ty@hK!F8mNpe-dQ zC*T?rh=4725F!4uxP=-?^14`@C@R8=!b}xKB@#X%qskII=1U%uhsvjqg&&}-t42LO zs<$eqorbGTP|Cs3nBP(-rJ_Vw6C{3SFtv4~~nPb>qtu--$g= z9g+QStH-7erIXMSBe+-}vxM{?4tLc#-9NhUviklVecqLH+yp=E#4UuzpGrq;D2f5> z<;J~d#|8U)W7Hl?E%+MbSdZ`&_VH9RdIKklThz8C$@}86Hf2D!DuaWS8zl$4YswyT zY!Z#Fm{@Pc8nU+^yR&p5DaobAoSkd~J3CPO^7Ah1Qro%3Sy>ZqJYn##SnyOVbtwz* zb#kz1S#HjYahuPK(Dh|y(7ioIo5J6n4XtRBa)9;c-o;!o*TwX!i_+FTTXHR?y>K~B ze#Y^3GZeqQ>>clx1jULpW1e!*=t5kF@fBjLZF%VYNc?T1k#2a@s%sdfC?M z{f2Tez7)AiPa@|%x%VheE@9|^;fdUsu}>eaUNO;?(42tHn(b|Y%HBsqEDV#qhP8N) z+iN`Ewxj(OGM^Lu5E^gn&)1jG9#eG58{tk>$yZa94(&=0^fvAWaL=Vg=+&;oT;N(b zqI|;S&)}ZdYi)fDSpq;e5g{3kh;f?Kr^BdX6maeKss}&?6fGI?cnDN%d)pxlZC8v} zLbDJBm&2mnO&pxNL)wa%xIaS}y8D8F+S@2?bTWK=sjJ`3{l4Q5yQ1q((wwtvMOyKw zqsV*lrcn9wROEM$e9(S>))pLLB^ueXkNTxU+M70t(AMYDp_!odc$2{I!dfoN8#nTD zH{5@vH*t5P;TMI6PHDx0s`^%~e>Xl$?*73uIn8Ks;(O+A2*YOxJ$`?c5IZmx@;i{h zGxCv4^5>IGp79PtlJBi>=y6ra@#^-=Uh!QC&Fg(iHH|QO5xppWD#gw&`|IZu zuqrRVB2Ed!dp@c|WU4`$)6_%C!&a*I26D)A8~IXSx~aJ2Utzxz~x z{T}6zUASt2WJDZo>6KHASeYcUqn%L|*1gdhBTm}&(ezx5mWqNqMoz%DXWj!zM%F(o@o0KLErp}(`;oKA#F=7=G&CimAif3 z`qsv&f-}(HuLP;^^4nSqmF!@zMB5n60WyNH)+05an|C-?C2rHQ5zYvO3^_|3z(RSO z5jWevr_;-bTeYB057LfCEaDm?P)!paO%Ag5^_Yob#QF5g=uzG1D|SmS&u;{ypI{*} zcYgMgVO^O?Poc(9qbTxj1DcJe2smTV#rM0zpVg;?Apm=3A&kvWQFA+ZZAgKNhe*Jy zW$-fpQ>uvXtrbzBS9ox633{nMQ9CfZpE4ret#+ShB4=jH?)!16K}NI#-|FI`M9nHG zS5<5gBT(bnmvzlimMl3D8Juy+{X@O0 zE3o0zV*3$RGf}%!2x*_`cub9>@b1;`KlYgX7@S0yVD|%mp~2&OC6l9^_H4~&1)ih9tLbXdM_fUak!{# zlou7fm%C(tpz$7sRO>Yf2T+!=wdLGfjEoox<>d+Ju=}z2qKziDx7X=5V}2@P0F5Sq z1e?M4N*Fj>n6@##?4JgdlIEPyfZV-3r^US&+itv){I6CxGT8np;lKS;VyL~y=~I77rA;QyHqhb1qOX{Z%fqm z3l8(|=UKD2nxU1-<*m08(f0bUyxRNt0u5t|U3z<3^MYZ=txM2ff+!X1Th(IQ1`~4a z%Rt*D85VmXJY)hD@-h)21i@9@@uWDKM3)@F*ttGaJ+ZN9#_Hi=u>duJXfx{1SFak% zUS-HGWsG>ET#HInH;~96N_u}J-A_Vc4V>5!0JEz%4O`0?u$dkc1N-Z(E^6gvFw04H z%zdiCyvF9U_=4Altnd@dy~<L2~sD3DC-8D3bI zZP4v1_s_SVZBkGj@3g@4yFBr_T6s-vwX3p{lI(`pBZt(}Z?eCbA}R&BJ+|TY;Udny zR;1S9#>{Zh$E)d~hj65C$jeLhP2hY#+Zm%4o=;$-V9Q6wtzjC`+wM;JP7D^Xly(0m zfhGMzrj@X&tCAdCqT9%yI#^X4#fk!`hnHK&0~)j{eZ|4f#acUbzvX);5~T8Ex%e7r zcrqV8hPreh(m;Uycw^tmixk&79_%>V?!Pb-ppA9Cwq4v zdXv(}n?8=QSGiouo)#GGKWhoGho&ko{UG z&Z!Cu1r0wB5rtILO*w90T1@DuKHtS47-jSn-`tk}-A@1lWJhjeiQktSD|e9)%-q|X z;AVPQ&FS{q4WzS7i-d)8twCoQdFzuVbrxOGIDw9MUVtv|0AeeOp?ydREk0t$vs`aut8y@uC zbitD-Ey-IE_vt^v!&grCu(w3FFARz^)C~oK>9WJ{St+zbSHKz95w5VwrFEg0JL}*U z-S6~CvDElW08xBC`^2nA%BqRX%auS}zfAA^+CYbcM!qMib#P#McG8YV>1 zf}{I2lNc8Y`>)H~Vnl?vC^Y+Gd0~siMn>5A_Y8}P2A=Anp6W0l#QHDAnuFln^y!GQkMo#qwQT#Dg8afhQ3_{)z$Q zvt(2yFci%0i|HEn%;VhIOziCz>x>=ccV*uKdu=Q`P8(7Ags0NmXL=CoGckvW7#Pz1 z@{u{PykwDnw;eH}tAvgSNImy4xf8YJSOLxM)8#o?bpzh6*9%{Ef;BPZUNe<_bicpO zUK%y0V788HS>eZes!A4Z{p7(I2}dL^w^}39BAn%XGQXOHPhbDwn5!_6HkOGo{^?tu z($O|qL^a~}ZMB0#m_+GQK?iL9t%_0$ zOM_RMnm(0MD>J{XVm{vl*)hZ7NK!RhG;De;+WJ*F2e~YRxQ;#?A-wFTzeHwbyx?Qn zF^%Iyr58NPiE0MPKwQp z-YiMtF;_UE3iSf-H=_1!t>I#h74gR_Y|7e#h%ag(5NwS6Z-g33dXT&@WbDJz0S4zj zMM}#@z7{QH4YdSSM}lTz+wf7d2iq?0Uu?rPL-~fPgq`hHrj7Ust*d|XiXNsz-<+KD zu}*R$)F1Je8keZMSG&sIuQZnp&hm<;ZeFBq!z+g=c>#D3bp^Hc z_9dy24|m9;@{irX5Q6jLgeJP{Q^0R$aerro_Kj8na2nPxl^k5D+#{Zx$?S~F-hAx; z;063P)}SeDGbLroZ1Y2oEuscu`8Ihsx7$%y;=31Ub($T_4y7I1hg*B3DrCT-G&z>u z7y0e3$U3UE-ECPnp37wAzLLGAw4~7X`>}9JPZ6&j;Z*%BS9AfUUP`y}ZutJ}P+%U+ zX&o_pO6%44Xhg_X^shF_g;y!B_w!)ZWp2wt4&@4=3r5^G9+sGreyv!+yzdnCALwu$ ztxVb%eO`EJ=Rn@A=|ECWS}42tT4;RnH_8dBZse!P>=^ZwnH;3-mKxkw+HTwi3dX3n zRUeU--l!kxeKh=e=k+a(5Gi83T%hgXcR1-1K795JF`dcrG>pyQ?T=49$(4JrNgFFq zJxZc z59HY*z7Wz{6;gX@$tiZCgg5_)kXn09LrDu2es!{aY}NmLuy|hJW;@leXBf)r1l$h^ zPcysLzJu#`>VUr7w^}>jNTRiskTNBCZSY`>w5f7UuXNp87a8tjp={{4+K}Z)BI!|! zm5H>GcU*7UN_#q|_qxra)te`Xd48SJFvXy#+0Ks9zEKpNe(UXzKI7eF*blWYj4VwPN1;6j1Wr0set2kbsELPV|tfsEd9|sz8r@c(5Vrf?>|JZq~JAIt^ z^|{Bvu~D;OK1VzuLU|ZS&OrTMw*>x@dJr9+y(8sbQIBg{57BG#3b1~6<<7BVlacfGwti*9uvDO?(>8;8!T>F1&Poqgc?;6`(Uvs0SXTJ+5B4$-)#~eri+Lmg8&@PAJ2wk-~%r-1CQRbOSf9oy0CQ@CaEytW$3m zz}y~v{57+&<`%ZM?46r;yzF9oHqKe2CoqNtLGQ!9I@6sKp3uKa2l>5jhXtd!sDQa= zv?9svcM9!M<)Lr?Ts`*3q_&JU@>AvI^Gte>cPoNF@-@RL%W|k28_kd0wcG6eT(%5L zGhFKoe(_r^jeT%G_JQA#flyu~*yE5%P7IUHFdM59Urt|ez;kb56Ss_=)Dz8m`{L+) zsVje9ndJLbc;xx=h4xCP!|e0$0lp@VKGDRp(eg&%6vz)cZzm(n*5uB#mLYlgF3{Ds z%;Al(`Egz!{SpOd@7Kveu8DWsH|KLyFuxmmu8}ty`X3} zr2LS}=GM|&^(e-;Rr8|JZ!*lLw23bs>xP8Jvy!q&B4SlGCW#Q_IV76i+mDyz!qdW| zKqk~dI55 zC)7t$qupO$AIvd-Ln>t$ij!ylImJ_Mha!>}XgXuB&m ztzM4rRy=U*!N%KZnWxY$C<`2gB~}OtP_adS35}mT#qH$KAgPn>347l)h-Ik~nR3sm zxz@eHTwlU9S{vVLK>}aifkl&Jp?z%vlVVY`B`p!QFRVRVZ(46u)jI`C(_ZUjeyF|H zMgm-{ODQ7p+)giY8yEJgP?*Tv+g{2`)CAsAbzsjh%f1Ts%c_0?(c8**@nE&+B$)j> z%F`I($Cde!?zr+J6PYyHZevx!t0O)dX-)h7zQ)MY3^1qPm=j#ML`PZFFO?T-T7ut| zRi_|Re8{N3Ett8Q%!~ZCR>8RK{{f0?oCd;053>cjZ_~@iRToi9hObmKm)k!h8~xC{ zA`{dRsdVky<5kLkx+O-~Q#tlIuiRY(bMV`!aU(}MTLnDU;cSi58yYov5*9t7tXXsK z)X13MCq{$GjV5-Td?ORp?#ms_}3K`BXIDW}AsFLP3d zSd7>IRE>zaNke{+!8tpeUObnhwho&ii%%h0UOUA1kd={ci*n-r_WL+>!_ap*9G^dh z8Y-Asi4bsm?O0%oG;&-rDCS3}!l2b?vMTQw1I;p7n`?z$lJ3TMu^w83yu;Ussy80D zir65xCO?>^sDDo-pmNRaM8+|N@S$J<+kuu~bS`1ei%(J#=d<;%%krKwXG5x%$jWv+AViy-3f71cv~A~sW@ z(1m^B+Bzc1Nz$(a%dM2Qm_w;`Ckpsc6euwA#zfuJ4IWMmaeqVAl~+Ioyewd4smry3 z$@hmyvApB@Dp#cPt@M8hHahL%bgkJ7Sey0*qKhgUOxm(jotq&;?B+HS`>Z`B0lfb& zEI|LN+5uj7m_!}g^X-Qw>$B80_jYMUCGN?}?^rwmk-GL&D|e7pXZOAGQ~E{ieUS2B z$A#^kZTfl~y{FMi-iRoC%8|C+BZzgibwI%^k6skjFC?l#vip(cM38>y4LUch5{==2y{JKA83N`IT5 z7>9YWg?zbAs|y7F@;AZ2uk7p<@LIfAAi8C2*y-&u7w$M#z(sb`giBN`u3x-INNX>GlbJw?tn&1f%aD3={% z{Bn|6lpBoMt*V>ti9y-n^2s=Px$7h1dg}X|@;RO5q}IsE#(Rir_J#Cn;ZWc(oK#9G0khD3juQ&%pB2RT< zy}Z^&kj1!a7JZ6YoLc)0^dI9^jtd0-cBYQ}quqKdME*=j#4sklxQaf zlM3g{!yzk4HE)#D7yG@}z_nF4cuf3G*F2FB-9uYgA&c8ASzbviWdjATFQd_rq+FR= zdsk-4Zb=blbm_vZlq@g#E|Y6asTLs_=G8H282b%|uysm$p8}z+@JR68mk%T~_j_)9 z-_P`x@Nf3y=G1no^Q-}5nbkR*xuAKGzU0P+Vs=R^pHuP@Se)s!N$$(^rIDm%*nTmJ z(04!8{ozkZC!RabMJUxwm5u-#;uO{j$O)CiyAot$q2ReH^we{L!{^hB`Nc zZc54Wv4-z2S3|HCq6fL}r$l%THYMVGIR&GWdN%Ma4JNlA=09AQ60nXDAq|%JN&hh^ zvBo{pb1T`%XR}gM(UP;Mq}{T1e7S9IT5S&@lcPy8VoCYd3o-MW9MXQ!VC1tignA|( zN?M577B&;L6jtia6P=}2;4vJ|Frv0mLCG;CYdhnu5s~c_%u_|+P~!HB+c7dJ@?;)9 z`nl3gA3m-<)uE2Kj$=7OIh3**QVa+0=0UbOPgCJ=kCi>&1Eg)jVfVRl&^wAKC)R~^sF|YC38gkc8SckpW<1J zQV$XBdJf=NQ>H&MeNG6<3=!SYsXeO|aq6uc)>wsJV4ASci<5f3kM(8J@`lHTIig`t z;76ADv6S)-VN{LnF%-Je`ZL~|jLg4NJOu=_L=J@Lm)-=%-oQ7!fLP6HI0q#j%WpEe z=e`6JM0zf2o7=){heyw*&V7H74_|OZP_3B*qDBeNVH1*=NtCkmr??AE!oKHX2~5@> zbO%$w+>~{cfUmcp~iTDZdDg2_t{>^>e86CL>%j zITRUrb&?<9q?DzCA^XuN_r?p)y=dg%3J)T$Df7vRV|oe_6R8}^+ET-FBe8}lxwemG zldBgmzE8ZSAa)ENSaxEyDxu+0=CDqEe{`%YI~Tz-PfhWTL%!^N;bm{DpWq}>3as>y zaeY!vmv{Er>1x^L`G~4RQIp=PI67J2L-TrkBGoB>|7st4seV zgrCtHcauE>{i4RldwevtBeIN$29%#JReKSpR4ek~5@8_nzSY4K=1buEgoU^e$Z z7P45Sg=8OTi&=s+rC%segdB#kZDHVC1>^3j?0 z%~!!^Ts_L*#aI+n34T<#jYAGCe>0`G%Ar>iMaH%+mJVD~kUP8IW)n7;+rGJs29QA! z1#ybA?QLRXku7~&-?M+S!LRnqzqVg7JsY&cbWF^YK_QG7b!0$ z8$z$v8+H;m1Mg*)yOfu82HqaGV{>p-{yGEkaqGF&Baok(y*i+pZ73tmzO|C=_~qI~ zY8Poa|GV#=!%=0Pm6ASTecp61x43Zi`mZ2}BdEOGUej*NpE~6zFNWz7pS@ERSoxLQ z!5iE+f<(~qIv_WwKt}+&tQUR_C#?`td;4RNtAY73pBa=cG4%7Es-~AL{gN{a&zEn| ziBAtef#EvgxB+^NTj8Gn%{ywt2E!a(V}f?l9b2x3^UVJD{`B?Fi8B3Vr=q;Vvuo#tm z6N(H5uwoxVi~wRK7egyiZqme3`n$v+Qe~thaVjD3GaOer%I7wFDWemQXT{??mQH@^ z<5+7E2eMmusNEaa?-STqv*5G4^F>Wr4@>9sfc#Ou(iCBO_&w`d;eLO;cxY~E(;2QB zdnO51Ov}$Lv_DRqYPX1h6+!#nRC6PuyF-yF_#*CgBq6^q0hO~lZ6@| z0 zJ*(rjXLA%1)$fOPiUY1kWXH%@$N{<$e0ratdz0D?{gUvGF%f7`MLyFC2T#|ly0S1u znFUuxcV^T=6+5%gg+2_S6b9=jgO>$yax>j0z65ZbDdnG+Vr&n&IAvjx)b)|MK}`C- z@9$+#YozYx;3KXH@WujitCF7ABw4fb;hEui@sRWqD!r+n7DXmSImHafIVIQxz%mH5 zq!q|be4>3#QSdXuJ`6KmNh8l>jQs>4$Uwzgz!Yt#+K}ySZqPc1D^5Ph!mIf`>-i!| zYTv+wPP(QIh(L84|CNF&OZ2T*4JkOz6(O6cDbTKOe1G9@!!Yyq6CfPcx6BBN(f9H1 zteV&SxT4~YPrsTpDGILi17HoUJuX4|>)9NQHK@z5WaxI>p=-g|jEw4{nuKsEbL zkbUflYoxiD2mgNe4A{BNayHs|_h{`-z$y$G!YYECa3>_rFbuGvq=*oNT@mVJ14G17 z6J-DNESf;rPeG`&vr&Y9LWMQ-0Z4-No54!=uDzWMo1F||5vw~$!0Pkf^=$|8)P5&6 zc(Jbe;1|;fIBjmeI20c;OR0d2FhSyhQYWdJ;ZDXgN|qeWzH~v z%oq1jAts++a!YvIQv28seYoQUK+P24R^PgJ@J!d&hanZ-Y}l_i2MGTXze)i9Nun*` zVg)mxOi#&|^LwKI`BmK=h!%X9WV8d)ovHb43?KoGm}pe#ol(%gN0%L-+qJf1CBK-<$T<@V*43{kR(1s1Rzqe>YMt z1}CHi-^P2u8AO19Nad{4_p9E>Pv|P>Q_Il`(8G4P_;|*U5_SRUlHLW~-^GjDOE&pP+xYspn38 ztfE}Q`8;6mWR>}uTW>#PPSrP4aiOI`?-uWmLJr+v`&}6bfm$Xc;8+<+uZ5-kkNUvL zmojNpC}PF#Hur|l{rmLWJ(j}Rrkw!ME}-(J0Hjg0D|M&y2BR6bU8K;ZG(E4l6~e3+ zwya7P9Uam$4lx=IBfldcNKGPBrC&E2P7?VI0d3}@E{+MU%_73BWdhFFV^Oo*FNIM1 zU`pLDdpS%+vat4gcyI{gsL-39+!JI8;bb`$i}qX$0i8#c9tEWQwwaEP8j!Z$b7b*3 zRq=-g+hG9Tv5SY14M~(lprJBnI(Q&ph?16~4rDbdV;|tw_Ks2ybS4r4ga(JziHYAk zO6l8G?-%Q?9{_YnupjrGN+J}VqUam0J6*~QR)>f40fd($wmiZ>GX=@;(H+!rs=k!Uo>_rk)4G7Qax<7L zYi2(DG_H3<=uIeeX6zzOzHZxH&}Q!OG?)t)!m^aZ1}4YJH=NXMxz z&Pymj4lVP)uyOh^qr_D&bFY^_Y$&n>LQ{yC{UUJkW9RlFS+|Sp9g+|v!efyLE}v+W zNl>Q;^u7NX{s;pa9^GfW*=1IK#E@32=b0K%O27Z*#kI##4-K~5tE;sq>RT(jZuv)WzC<}DgRC2Ks9SY{PaXC}11 zm(nvlE>}?)H1tIC)xEZ+8iLO;zOpa03tVDOMf%ORDF;CQn#@ah!-+-DGnCerG__+_ z{VtBxb;ny?-vE`?h(gYn-rkT&jy<#Q#C|#qVO;T@WgW9)zS!5kG1(BgGJeIM^U&Y- z4sKp!h}}G=MZHHt1XbA1k1MXi! zo_diRo;JbA-e*eN5iN1?O}Yf!+!lX43ig!FwG#89oM_{bLbDa;Sm_K7 zLnP~$4&*)bW?O`x&j{_WLP`V%mrw#xo&CK&WEXM>$ z4Q@%rv8jX;C!fv;P^F>Sc9TUuEoFT_QpR z$DV6m%F{cxXKyE0uXQr_VbMz6u2uUlZD|j~6QuSRI?wBEvS@FQw9uY6ZBhP~wZmw3 ze-Kqbu?DN=(-OVEzmv-X5JwQ;<7J-pr$G9LKC~&J_q;CG?Cn9yZ`cr;jq*%dkMyXk{P03dd0IiRo#*Cv9ztf zLuJj1@F9F*A*(TFCojF91^p;q?760Co-BD)LW@9`$fIsV96siN5YGdrj%(crEsJ8s zA`WA9%I6OJMLz8#TvF_0>JU!JnJH_Q!}x3BV^80NKaGOw`@Ml39!Sokq#l4^knIs* zVst!$^GR;0K=fMr*+x?(Ssw?+0vWC&*-y0#R~$<%mnIEDCO@5Q=v)# zL3uy1VwhQ;p#D6aPNJ$H&!jbEFn2MGcvW~hxwGvi?_L;d!qXSRvnf(cwuVM6*@*b2 z9ufO+BGvFc=fS#tjlqTcDVR&iNVn+p$o57jlEdB=CWEqTvsR26Or^7C_$u0quEiO; zPAQG3Rf_p4Qjl~=%J6<3QxWuXbk1Yh- zdW!^E`~~xe8->eMuI9RY1I8Yp+IywC{i+&0Qb@B~-;)@X$v98Yb4v1R0`G;5D1mZ8 zGS7EqymF9}I@8m$dKj%xLhQ4K%&*jleCt_MJ-#-!IEL;w?0dp+1kSpn!v0Xg&&L!j zeTf0qIL16$@-fq!0NXL*LTRf6DZo(V+7PyTzz=O6KXCV`T;0W24G(8t#G*^92@VbE z1|!4X^~59rNsea>7@+(RmGe&WwL?TzbESx_bT(=@Q@r37Ww#milWt85=^T9gJ<&lZ zK(7-rIpgK#d_Y7fx7n@zy003aBKH7FQo2pMXLX{Ym$0uIMo$0&agNI$&s69Z)#(dL zBwS#zOi#^YmW8I0zL*;?2;+jX1R-IqM z#{+b~hGRF@Gk&87Q?_fKIyj9elzkUl9ze}uB)+GLFYGUy3y$X5-OvzF{&rex&K42+ zTJuZ`8Q!gV;uv-s*S+`aM)9V1s#UGN!gOMHvAuG2%DdC(L{-V;tEr?!O(ltaq1W^- zDFZS-z*g*xWeQKELm(jeU%z(Z^~DxWnpT99vHo-^8`R*9g#l!LgNj?CCYOe0a(uiH z|HLu0&uHW8@N0a;dM`i7hhN$4kNVmo^}HYu6Fm|)n#kqtU;=MEBnW^-PP=mXp%%In z;Un_f8t&C=VfSu*#a)=8?Ff6%YtWM8NSm0YhT1kmE+hP~d28>>bJD9V#saFsGLvV) zl+O(5l9K_L-^fcc#FpgDsVAmWyK|;d)5Fga{c{Wo{nnBiz)__dD<=2jtz~hw2t>ls zxp!}Ijb=~lA}nPX(d;lFkzaDeiImW1n~(6A#r1f!@C~2l=FLPhRyPZOw+)ZVaztJD zF%g-@pLag_CM75n&1S?I&db2-J(V{3(e#>FMekvCQ1Y95k;>?sWO?dMJBVD&6v5TJ zdD(K*iajh-MTp?Ps4@I#ILA)=d`{job(G(e5YOd`*C zk9pX5OM;SiSlVVuDchJ6sIs4EsRbruUB4My>4|~;s8#eFNIAoF66}Wx9M*26Z1}hx@~Ryda8z=+n(E) zz)Kk4uxgl*j;7C#qMUxBR+O?Ur4;#f@Wx6bTn5DNhH{8TIPL~7BCkI>eR3_N4{2dY zDlSxe@w(|;kb}zZ6+P9U6#C_pfs#;!(xz{Hm{c8!Ob{%5ot@9GFfgirZf@_U;dc%Q zc3bR-7o2?Fo#~NyX|U5({Rq|M30-=B&Z-=CwdIQumwQZ2vgE*5l$8liI^j!jH+m6? zDe11Az4II7KY`;*wX2vXDFm%uHKziY&2e2(fE{XC;s7zvIgOACaXnOCMCZ$zgrJty za?T`ZY?!ArK=z7(5^p!lGg zpxJqm7NR&?wH3!~ckqzb8sE?yz%BcbgeHOgHxqm7$(6m?75u_R{k3RPX~R;CMpl16~adIDr&alBc%bH=0^}l{df*mK%&j_L9-} zR?I>ckT?oVS~yauCMh`Y zoPDkuOL$2!>WMbe+nXN?>3lhb_W00}3?Fo^Gzv>HU)#7VC~WiaMemJ)987|q{OH&^Vw zKnXN`w*CC2;q@)l?>g+ti1Pj}xy4#r8-@(axe4!)QKi(Dfb=gIVR}s_s9?cmwaLSq4QxUnz|-9e z&$jA4*6rCpNg(;8fE=cSWmgJcK3P51Ybru;q1=A^tnt&CIGaReG3wyjMa7B~GVtmS zy>ZWG*+?FGS6C#_4z8WobEEhc4FtpjSd{5^8OeDGGU@+|r?-sC>iPb^QIIZ4X^{>I z>6R|(mQot&ZjkQo?h>S>Q@TsKySeC&d-#0+zk4lt;5zF%H8VTj`?co`o^rzGI7X@^ zt97nx5SMCWwa0KRKgcd55=_SJ5)FMwCSIm0Xw+&eVt-e_)GQIclh<`?%W)=*YefT*%3JA_AaZ00({jR{YZKALkF9AU7?e+2$*6}sZ{f)0HP_ln7MJ`Q!g0lU z2;|T_>Fdw*bk%Mq1ZOWG>C|9UNEx+_6={o{5es~rvCZOv8|Mt38oqU^i0eIK#@Z1MNwm$$%{N5AGVWvoh-pCs$AEigZ_)EI z(bkT7_aTtH`&mDpD_G6&DQY}=S&ZRN+`9Gjn`@d))&(G29_Wp-5v#Pply_xP+ z$(V3}&`yoJ@M6izfc;@a?#MTi!iMgRSJW|8gZj`a1Jc3lT_eSQ<(> z0x_aioX}A3!Uzb91x4(M4Y;|#p(3}p7)b67O9$;@py9;F)@l%aL%aQjI+PxF4H_OL zxk$0Um%9xg5WXG1ed~3tP40B$W4FvpZCUmp$~u=v@qL?pv5LN5GW;`{sC zr~8);5_0QsBk)+%X+&cCijlPVR?ymdTiMExceQN7UzD}o2|ZNQ50!2lih*ck9%mr2 zc-DusZXC28@UFCZwyo^Rv<$|iE=n&}Y_Jwm$qN1SlMiT@H%?$@`JPmZn{YU!<%n~< z2xsS%(9*fAJrIB+H23*6X&yS&tPNv~mZ3(K>&)JcTI+Qr*6>2|pC5EN3&u4vaOGjie z?yn@h;hDQxQ66;CqgLK;a=_mp&tXal`6rFjllEq!3w>$mI$lb$u%MnK2ffm>(9nw1 zLO~j$0Rq7#F(fuNkRD!GJ$A&R;)^vK=_+)gpklpVy)Gw+Sia)iPn(nXdy|C-i}tZt z5$HLEqu(Y$cNs53w$PWv0U<`OGl|=iPgMb^mWlR15lFj~E4jHP zre-pqo!oee8@Um%g}!8~Pi*&=J67#DBmWF}jTSG8M6UeBr{<|^O69$F=g#=4TeU86 z9mC)NB54TLK@cYHZfdU;cNcKDpzd#M z#6!S#BFrI)1%q zSL1bA(&coV_dutcJWh;7+BFoFkXhe|6!v;j(STmG$DZjUp4ZECpiof@oSNghTQo}{ zM!Qy_uS10zhsNlF`Rm;th%mQ8@4{`5z)8gGteB%&NP)gW1X?_n&&lY$1P;TJ2?Oms7~`p2+>sg@%NT%*-75yPZ)USd1lz zxO}|2LrmRdGTtD;gZ(C!{gK8OkKYPRE?SN`D0B1uKn+5#)^0DX_K<#Mh8qYK1fzwu z?V9(ETp6C!58Y1lY>adwJbdG6k(iLYQ6G*gl0W45QdX|h?C9&S)>|3-HWEJccTcTV z^ort6|eXR6VT*`XaqAmIyrD91hw9P+bZa4I1T&+qX6d1YTAz!@+(L$-q$LJ z#nnnL0afwdnihuPt-sn@VKxdT5>wNkPD*q^mz4LSQ;XSGsz@YKa^;|dp9S6v`D-?& z&WqfZiD93ygn~*8(NJSJozWeVzl-ekh)tlNu-tteu4U(b-qIa=^c-kAi#H{Tpy-Nw z8g9REApD9=ZmoNdfLumAYxmCT8O3lR?z#6l$f_SB&UC#?Z#Oy!*&D-7igu%7Rba%4c5^k?UJ1c+huwOL7C%xDxpq z$T|1b_#TikrQoqbnem#P;33l z&hSR|4$lhde|j*hmHE|5Y^ZUg zRz_IqUOG1~wbS-QM=c7v?JF9aDjOT+BZNWQjY2Wjyyz%Z*CIR?t`UN1A6W5|`(KPh zo-Y?rUEw}LsVc((5JMXNE^k2-jHJljrsOB%5p?%no}wGM=in0}r?LA^5>&`ZN!(u+ zD`G?Na$iY}-w&~nqY-{?xo$B@)cDG-p(*`5>J{O2`J;4M=j+okg!*|RvCrjZ?QL74 zspa+|*mI8eU;Q!bfT=Pc$LrS zy%ELjh+I;K>!dMPKWcG#HSt`X`F6HGIt`A5{OW#X(s3Kvx4czqv-N(AcZcInYH9{^ zJts$>SG&GF3UW>{Kh_bGE%`_@}9oEnlqvZlur@zp)@YptcrQ2;Ro!7i=>GDf3i!1SUb~eP9QqjHm1;3;wHACE6uM2T! zT!yrhiPDT;v?2VLG-=XbB2;T-_BSOTO4M=#g_4Mfke{yVOmrB%!wJNPk}K}-gTYm| z9XFax`2DvX^|el{NXu{=QC+^{=HJd`G~^I+SPCD@a-X zQsi+kO%dBXZVJrgQBe~IwhPR?2>bzq3_j_*8=J-a{F{W2v1p=;=pSC;X1dmgsefvM zB=r7fE?37PjFJZiPiRO~t=fJ$pSa{;ZW*nr1?M!Kr}F+DVaWlvEGFh{TMD-WVs))$9YYFATY z`9$5N_cF83?@!Bh!V3;KE8h)NiLoumTj53667tq^z6bCBSX zL!lgGs-V5TQO+;#Z)lA6_vh&Q-B@UOcv2&aB9k9~_aRjEhDc@Ek5nVfnqMV|x&4}o z;78@fs35)7sPdgc%b?yxpia|6iNfoz{wjUdPdz?@dE|&by>J*^^#=TKRHF#tR z`yDm0^(Z8QR{Z>OB0~Q4o;NMrV2D1%f-%OHrG%~xZ>q8XCp#bP3)f#$ivdiYBq8*(0jLl)iLwTf47x( zft>|GR*PHZ-CZ8ji;JL>6MZ8li>j(iGQb+xmT777?Ib@D_=k)$Tj2FC_9hL!YMwHv zGdcQF#gew#Q6*76#nXxV#|_pAQO+PM)XG%V;wdaT8(j`C zF7DGEwip!X*Dm27J)(Pxi^JQP6L@%ImFWz-z--)Y79S+Q$BbFhL{MFgSyCPAt^1sE zqiLmZds)6Q-|QmW_D5vC;17?lso|u1pOw7MZg-RmHkfmCZ>E2fK~8&vk7qkpr5V>i zqx?_;jvkLA`eN0a^Fs|_MPtMx(A~~u+@%xO85l{<8-~glVggaWZ*H$^R<5VCIX1Or zc6mdCXpza*PI78S|MEK56hFfs$wrqoO*)7U5m#v$S=rza#EE_w(sM;El)Q^I*)G1(qm|WKlioC;tufm6p_-k@R{*L=aTO zSekq`{ABmkl5Q$@>LuWoFv5@d##|kLeUv0qocvoK9HSq4byRe!^xnz?N~`W;$jx#K zc_eXne5TyZaT7wRhOqU9*j=VqhBvNmSCh%x=z9o>Q( z5QvCJuIsj>bR_Ehg%4ytI}2*PqXlu@T-)=t^RHiGx`D6q_b^iqK)i0Y$2XMP`^3XB z?8PBH2uX7;s!+5?$=EcwPc1E*ANl37v%`k%hby1f7F7Iu#rI4lcJ-*hpT^0g{vGmE)cIJ1diab+;*+v8G6M30`|NPgX5n0}G>g5#t zzjZ4S#PC@STC5)1EPj?v;XY^{yFLQ1@k74-i1Mk$8R}FhHH?4fFN&FQPuexb^qZVV z&OK4h>^a_~Ds;f-4@}D~rfX?0_BWL&$G@!~5QV04bE*^(Ep`Q}z5PN=bKqVJJFMQB zS!WUev*{7!hpva3-aIaxUFo%XsfX+zb_xPsmreL6?Rd#^XR+r9!|&IZ*%N!A3?bY` zZalQfUFKH3@tFe6z1=}GQJDgZH)_eq{hfj19{gv^*Zd!@7g-3MmATU&rScivVHZ^c zQl?vm4)_EO`zW7EN9NSgfamG=TDiBz#+Kwm-#IvF@Td!TdR8_$D9^3E<(Q4=Jgr@^ zYqXmQA`X;hpFlxNHRJE<5ruT>hiUo$;+X!jvx*R6_k(&Qndm%JYx2SLyQ9N*2dCUU z*DC`i`$8cNoLTmFuo(jb=i%1>xfpKXqruXzKBwtPX0@oToj47*iZV5aIr6zLP6?Qr z5&u;dapiSKrPa{T$+7fL-mRnQ+pBJ`ryXaolUoHGTs57-ZylMDtD_xge+$f&HWDp{ z=FkzeR|@#bPBRMj>u((CJ#bqNRL_sE6EWr=dFI=$91cz5CJHm~kF(!$u-8fJ&o#Hp zzw!G^$RW5gb!lW+N**}!5c439J+4dz9jxk@explIz0A9|jF@=bkspbCXv)&FK$Ssi z(NphqskHfKXURERXpVh+Mu+TfLVDKM7X-JW$&@LB{!Mu>WWXcaFpGMfQIW|a{}&2q z16!IGLDI|O^;CcV<=yS=Cg0U;k)b}DHkHFVOv@>!OC(=_)0)sfQ9f*71IwD5&tK)Y zq}vNauQeV2RZ(~52Yc1LU1`8vnaIzj48=vRRh;s`nptPX(|xBViO>LXr)n8}jN7Fv zm%piuTw|b#MpHsZc<1i!hF1_TgWXv6wsqh(+t>|tVp?s~!|XYxF2W;KAnk@hT+_3+ z_=-bA>9G%+uka}8(BcN6_p9z&#*|k_K{rX)m~n4=uBiuh%J(R{=SM%QmiUWeC038S zN%9CnQuhT5EP+d`|7BF>9RCF*M!z?s$!0w~V3se-Yy#inWus^R*6e*qZp!lm_TYp4 z!S5Tj-;EC|JofMcXNQ)CHcGE+0LmL^#zkaE&c{ZKgos}KP4T0TiMH0 zRC46;lpD!|?rsgpw9Y!+S!~$1oBOU%HpfzB?3ySz#u}DJxVW{o8x1e}DcJJerum92 z2>3t@y3Rl&XxCJDX-G5pVRA7HGjdRYZLSKMQ2aZZ2UkQ~Fo<5Scb>ikC|i7d2MeNq zsy2Z)X^`Zk_F0Gce)N@UIKFq}6`qf`kOlV2y5-lsr1tkmz|(n*@`p~%A*6_@<;7^T zp-qs`3PQ(Oy!ALz0eXt<$~kculBNpV#F~`Jy(YwBk=J!Ye z4r5n`v2iqTZrq!W(Q2rFUdiUChKC0^8VLy4S!~JpdpnMghEX#a=*jN}=-zd_U-l1u zpC8cEYy2GbNhXJ}E$ntMQ}L!)F8a9(b3%zUAYUeMe$vJ@V4mew`N;m84!ZporX#Gt zRGln(r@0gI{Va}5nNWfD@o6z9*W@S5P4;+(fajJIecCs_ zRAc`$+*a_xi%rG(P3^`@T}JMcYnXDDG%041_i`HAQtI!yKb_P(K~O* zE_i}@t_UtH4QuZSf7h@>@Ezm?$GMSum)FcEgSzXq)bAmcy<bde(4U6`xo7rACn++s| zarM&(BTPlbrqPc9Oa0MeW3Kr-#6ljYD%)k~L=-GHe9x{mmNvbuW;lt0d#>>-H>?nm z#>B~VtLiLgNAo;hvh2UX-4fH&IdOmMoKH^%P^*t}K8AM%V{6tL3xWM@IZ+9@h<0x4 z8n+Ief_-e211699*BbQ~LOiH)YMic6wWADmqsA{4$ug?;jycMAohMntM?7P3SB|?O z3r0YPAQ2rGxEvkSL;U=BI0%pf`eC3Q15!N=KC{( zrzEv|JxVjJVDgQYvk}c+@=PERe+Dw$G*^5!$Y~A4_qx8@%YM>iNYI3!1x;U`q zx;aQ!w639v);{+9(1wun*WuZ9Ly>Qa7b2~nwxC9< z&&B(!iUtiWgF($;=P#itjB+Z86I&*zuNmUya(qbdOvarpPqh`#&Y*TR%VopfZ^kpa z&e?%i4qf}ZPkmqN(;GbL)3%kvH8ro@=sVZqU!{Ac0@Iyv^!{j^)}MOhyxegHPlnoe z%ER5zWQ%9>mX$Q*7#CbU9**;gxvyDsi>2iTChdv-K}lUk;ivOwIW_?J1u1BqQaR&ZX+~reZ3-+F=@0jpcbZGvJ7stY1$Z zw($64kHT*KEaT;$9hpJaTy%kxkYKTK6f3ald|;M#?4GK!aiKaZ8(?4*MPaxQz+SJE z>oexF?zDP66ybOg<`=T*4V^SRTz_^B#Nf#j9qgHE;C)_lko_Zy)2ju}JywrVzQDQp zkdlh1)r61l)=tvgbkiRxt%!-=XAHpRoe444m4>W393-}oWuYu#_a#j*wO;uD>!=b> zP?)!u$Rk9+PNd^Ga`#9T!NEgq3I1nJt?NI?a`xARE5u;Fqt;lEXiRMRO`7@7!DZ~Y zM%O*qo4)frLplr^f_M+MyHh%Q! zEw*w=eiET`lYNZxr_h8C8(&y;yq#;jh-`4_5A1C_%4@L3tJa5KsR=|hOzFBl?v4;U zW&Fx|JU=}{Ko$RYfA3X(pWDiP-z4i8>!G*zp-3gyyxI74>^4uEY2j}w==N!qx6?Cz z32c&+b;O?(x@J&ES&%b?XJ^hq&ADCH&QMW6p|d(Igyz8b-y7Zld6 zzMfF7@gdn{>`@58o_b~ZwDCq6%heidR|gUlJTEm;y-u5c#4n+#`w?yNqE7!xF|E30 zT~<|jxwi~+rLgvPCe3`IBV|9Zm-B*cF~Dl1`Pmc=+tEv)_|6q;RlE7!3`*5rm{&KV z`(Q#gihuCikFZG-%lK^jAMB4tLZl2KDap03^Tn1F+HU^V>v={|9n;6;8&h4nq}Ufe zAR7dreiha~7$6in*4xB^bVUX)W(j^bc!+iPoE-)+*%(Qku?CL})%UH5=zm;J&MH{0 z8oDY$vd+^}-l;f&Oc#~X@-yE(WK2wY4T=+ECF>}b;d@X~3_EfV1t3eRVH;NxT(V4m zxTO?yRw=Y%0&$q$nGF{Xd_aS=wglOWG04x52?yThw+`yi|LZ=gMyNVub@<4r5Zx9q z(QKIbuw-u4t^Y6m@uP1)>J`UR9ED-s#`)0E9@Hc;?a-CuM4?>nE0EFlrpvA6O(uQ( zm~?bRJ@{8!*{>;QeSWBobEdB?p^lu@;xPZ9Y}1N1K0VUL#-9Y=vT%+vK>s@@DlC4A zMa~z@{GKH90@FF0Uut+iSU;Q2Q>;Y5HtW0i$qfFrB~24)sXSyA@Zjk(`!lC!N5H+n zu~Thhvn6W$*``72mu`+v!luj3jj&;L32$J}^>oI^&8G4LWMmYGj@H@^nUnJRxu5Y8 zjG_!lTN;YC{z4<~v3e@qrOhWQHJRMek;24BbM{F+|+(lm4?$nZAR4)+fBfR{o)g zh*PyJCS;InVCs5~*i_uzD-{3h+HdNOU6OhD1o=4mPh|AEEfO<40#E8}gxvXL_kHc} zG1l;bjQ2-wLc98DU)T~u7sW}4DwD_lXV11N3)KX_-X29}q=%-}n zL-gnHv;qSEp1js8Ca%Hb@dNw2B%4!y&t648fybWSL7L#syDF$Mm;PIYBJe(J5$_QtEYij;o zp0TWsd0qMxDk+nqqwdNQDrIG;+i6E=<`LdgwgglqC!xqom%ktl2=6d}Vmkd7=NqlH1(e`!YCeMc5+RkIdZO<}xfMUsP+>0~{0)9b{y zJ~T9tqab+E4y`!S-7lf-QS;981o}lv)%o+ilp`rqZln`U|MDur!&qpBnTNZgu@Qd7 zkpN@^3L6*7yPg6WF6WdFo(|=SK$c$DtlB(t+KC5gToAkWwV&c#@RC&c!sTz>#T)sJ zz~tgxel_88Y1egsAm}pVnkM4)@Fm!_U}3wIP><@e754^sh??s>h^Lu*`L;T+kmg>L6cepY|NjgZZ4Glw{MVaCp(bgz7bZ(*&9f_ zFCmnY8W$h&--TKkkNg=2)=aeF10t4E&XKXMJv+lJvv>L^9Pjpvzm+x+W1`67J8R+V z)WJ#(T4KBu;^3JB(=?s87mW(%z8BmX3X7RR-j3Kqwj#E?r0D1Pif@te@hKXli)gh* zqYHHHYPOs(=MC?LmV68M2ss56@&bIu#zKCv*!wzt6&3}8ImnbQtlvF{VC#<*o=p9d zfB2_VFD=DnD^AJAlPscv(>UX%g0iN74R-r}tg#T|n8uHeT}>9qJbOMNsn|F@{Go`; z&sOm)5~p@`X}qJnTVd$pr>VLV#wR$;1aCQS-(8;HJT}-%Ognh2l$}-D9=bx;p(iDr zhvofl!=3a}XdehO`&~-xZ4bg|#`D$Z-|RXWmK**=Rr#C-vS>I4_!>WPpb%MoiZbcU zZz|8?E8h*z5S+{s7#~;;$gHR^IPl}BF(cuW`oz+n6O)Q?hiFuTT=-F$dVx|!HNMCCr&ve#w(nRM@Fb|;JLTNatu}H$;4|59eFDvlF;-zpiqkAY}dOu4|dU28q$54 z`(mOi_fj&1fP!u;^?emH%1@= zHXm-71BJ63RhtD&@6a+lmanaoIJ9ZbX`3Cs)V-t?mP-^nlqO~4jC8GLxX_HvpMemx z*#_~@e;O$r#;lb&90qUUR20nc`jF86nwqimAr??Xh*<`% z9>mpM8r+PRXkUm9EAahfA*7SwN$;|7yRqh9PFv0NS_~`jH`0`>HZ}Ew+)lB9PKO4TQ1M}IG=eLW#Z8Oib}qtF z(Nxpbr>qVCXSNxyt28OcM^x>q^c)7~p)HtZ7vM4(D-SQQ(#NqHjiPYVW*?Sn@CyHw zCxB!+r!U7QBifW_L1!|dL8`DaS3#Ll_~LRV7&0Hdl)IGD^u=y?gpWh@i3 zB4A^d6iO*5>~3XTx(v|~-#fwT;gla95ZEyjCoV#tg3d=KYti#!1&rH$Wim zxVv_;`k3>~4f%K>j^k?Sb1!MD8wcOYeTMQtNff`057-u45j~bW;(h&h=r<&{ef8Nn zEgYRzj@KCBn$?FrK1m^?D;6f^S{SVcMW350ENpMNqX|GZ%^iUbbdH@Sm0(?%G1R7<8=Rt|EpV7 zDiUhx$Xa?G7Dm$Q1~f5Ixp?CVqlcpM+|(ZiPH9*VSlL-neotJb7X9Eb?jF~lMi_3+ zHF|*}?Jw3|9;wc_q;IkrphexwwvqiopNo8QB%ZC66@?Egrs36k3c0r70;jL9S~GlT zZ~odf{wItdsM3anig-JX?p6uu+vX5Kt;SUUZcU~D$5ZwrETD|8b56^L3hIk)J>Kwf zHP=5s=W!MWXeM~tANwOZ*)SikuhW&lUtV53uEM=k0MY`92XyTF7@p^!AI`Bn9yXXn zvxQiY?K7>`c?{(gN1)MG2Zp=bTrY$AcLr)J8=C}Ym=O`TwX!(9jz^AfN|SDw{@H0q zFVtaG92F39oolbNhvt4dv<#D9;kM7VGi!Fze2Wx{0~LWa>+eq2`u^vBsv9%jJzX z%n#Z}_X}^u*Zlg6ZS7K1$*po1(rL; z3~(q5uxDD!7cvGnfY;)FH!q!3KY|qLWvP&%wy{j zKAoKd-HZyMRNGBvIkK<{V{{`j^3D^`C_vC&1?%W_HO7Y>`Sro{OH@=`p*LVSlGXvQd+SogIob(+@4PsX z%C9!XK|$&Mb%G5Z4*BfwT)!EYATEQ)c2n_$QCK3{qi&kXN=I($Q-&yo=BwRcwsB_X zw2+6XO_+fIuN8}%BKmp+VW;nCm#Ult*)46vKVAF}$|bXs;NT!{P0og0Xf>u9pLqvsWNAa3nO~ONBPvsj`9`C_6s+C4VPxuZ zemfb5lX4lgnA1_p6wl4+*T5&^=Y=9;&7?DWorj~IMt5w<3WuBji{NQ~@;N#34pt#D@l)znU3dN@|xKAB&Xr4?O4 z^Xgk2WK=p!^pYHTyf%jRSG04~HqZZeqOH`h2Us-_i8^Eh)H!oqxmT0W{Nar$U4YPO zUr(WimhGTnV$c+;y2dS(Q`rN(Z@(ix_fz}d^^Y8VX(5C3x9o{r{p4U^n1MV@bNd$; z-f|H>MXaD0p`?k22j)jai&La#F6#HZXM%muh-1dg^8`iO2lX`J36FUhR>v{3h;iQ_z;R&x#z|1jb{S%+Gv~;r=h;}WM^Dp|z7>DH?hu%yW_^SM8zaw*0=|kP z1fmVWf`(*JCL*IEruHxVh3I5jj1~9a*^tNWKQ@W58w5Ixc8uQy98)r?2bbZ^4hRbC z3-;q!YF`DLJ=IYtYn`_jozQDZ%&E10I;J-QBw3yLT$;OEaE*+a69_G3&Pm$@MZIx5 zqdrQ!WL2a_g8KGfpF;dX%!u9C9_Un8bt~5!-QB`1Xj-pwjX)n1^|#3hcwOh57zz1b z3IE+nxX@S6^r>%N1$IO#2`G1JYPY>n7F&l{WzsZ?Lu+4TIeQj{4v9|Dr|y-vVR~#K~Z`m)Q>dAIbxg zi*?Gkg}XVl;Y8M@I5c|vQlFUS0c2Z15a zGns3tTYY#&1Q$uU5S6*g42u3@ocN7yz|L-RorySE?f<{&V3Xr7=j8%c*)OW&FrA(H zC&r(>6e`mHy$67YVXjgGwl=Q#GtH+2=>PCYfLzA0>X(Hvh`ulIfz2}Qd%;2hu`sV5R+PulVJ${m?mTeyZ(wJ8f3mdZPCocd8_9B=|)C-#vE0Ji}O<1xymRZ)p{7IrnNPlz&qKyhG$|8e> z34#Vn9X$mdxVTmVAJgZTC0`iW_4Ii0coMiNa7*5u`B;iI@&$&cy_T__>zB85VnYtV z9&3uJFU~6Aq=O3yvB3N7fW)IRfxBH-1U~lS@nL7z8ww=mF@b0sm3RLWjQ%Kt8C3S~{Xw+4)S6dcB;O~%W2>%;RFnP6fyoS~< zeIIMC3;^e2zU)vkRQcafysQWE|E&iD_5WdM)Ng*6aMs0y*ZmKp1qK+e5f&^;9nJ;T z5rjtT{)l;u|4bHmv?A;FRkG{ElilBMpla;`&_z@dY22luFlCR1J8K2*Awaa<>Fg_iM{_s}}#cWz-b0H7b5@ ztNxhiHsb{mU5|KBEp&+#h-fM`KipxFiQ8CBD!?iO4`BJlGEq#S3{LzN;+YVG0elWv zzy}&YGs|IY5!uWRv|hJK;eRx>CLDNPBN?$Wfm0_UdO@Z02nY}?mw7)Zt1m5eExRJQ zef!y=`waF42s)Y-yncSjd>0MhA(MKdtRnec4zpJQncV!#&aewr6SF4|08=!Ff0*}KZd+Fv+0vOArF(NuOblXw-ySDM6 z_bXVhdrhIb6XS6m-mJz-g^^_#9^LHeYJX{C_BkHxp4HW~2`t9;!wYZGU)3$T4UozIukpY z(%eG4o@dP}=_t%<?zRUB6@%KyufV3GRr3t*WmswBZFY&7Di6&Qf3K7%#chZs z24h#}zREMl=XN~%dek<-Zyl#{PHbQ|9E%lz3rai{hpa7?lo6~geMtAu(aql|7F69+ zD7Kuw?Gn>`CS?FfBq9|tmF(Z+H_^OTwSXKX5y=)R$hvdef`MUeCTIjxa! z7N`BTOUBAQHY|n(Z_Sgmx%Qnxa~V=V2?~k_?VzTlPEm~R;Vs6B*E!37hqZ3_wE|#} zNEzZd?ctqVxXK>fCkkZ&Eu_D>8Fim=jSLsE)>)GCnm!b82w3hno$sRiPDTMq2kUiC zFm!BLVKK(a+WGn1HF(jDarq5WUWG9Y8Eeg9(FXcLC3JPZf8XQvMly$07bIUc?Y+l3 z%(7^x*rPd2zV*bOIUf@;tIQy*Y%d@@#(9;zh*fLT(?_HGv*?l3bcj>b0@`1E8Y~41 zYT-f*Fmv*VA!m!T5v6oI1Zl^M-Hg}#GSZOm$Lgn$`i5;XGU#sqaFWhJKtkoZRe0l~ zJy3ZzT$}nVzgAV_3tFsxt5-y7qycpk@K+c(kLo|z&y5Z+92WRQhk=v}U$wP4i~j}5 zXS?v$4-QHa@kz}9t~w)D__R1$JhW z_kpR*$L9r|&41^-_D1pY5^%De;qGJ@r1cJTfPT(6%b=_N6Q2R`Oq@-%%5mXUe%KIL z*KJVbSGG~Hr+IDjhgC*+hSgp>cFVvxSfnwMWB)n4AOm_caJ$lU{QlJCZ=!qH!bxSp z-CM-YbD+j{r;0xd!m=a)Ad1EL?Dh< zIx#xxuK4LTm1O+{8q0=AE~uR8k6zcmwIbw1blV-Y!Rq5v1m0wEmm5~ry2=A?z(6y0;i}aHmrZ`^r5{{O-Mm zW-7{Iem+?O9+73cIu0OhDlB|bIU;xa(W^6oi&|O;1x`g-&Tnc#8VKrP5ALspOGGHH z4oQK4JGbxZ-`?|Db?{T1+M}SYce1imIotUsr z9($8O-=U&x-|3aXAgu41oT-B3Q~DL%V2@Y8vz#@SN+m!MMo7!65mpW$PoF2>A2RlA zq|91_)I!3u2w!`!?43jH4*DP+L^PIEUMTEaZCxy{J-nLdQ;~MuEAYumfTM|@JD-$m zu-F!llgiSIfrZb9(b4Am2WwwzFe%_wfcNorZi5(mAqE_36t^cK=C6ibs5_t%-KkyEt!N{5~IVf9yNdXU4%YL`a8be`7TrknaR`os-;!Lz&NfT#R#k-bX*es-pE+f zAAdEKfUN#7j#3MF%xdG)kGitX3$@IF6@Uf?!0bkGX%|nAUC`XHilUFztu4?qN=HTx zT_$URtLa^}^7IAzWnd>L$t1HFgVG*?)D`UCkSl6Q+ z9jX99>p>i-Lv>^-;?d)18#4BwK*uqLAseNn4NAttzlMD%e@TdwYhRZDw0278Ht#1qx@tnR`1Nh@p5? z>NCH1?l{Bf=MFTW`kN0GN&-ID)ROW#THN5KpKz0bHs_`~Gyl;{H}^bsh1_0A4j#=PC&l>>W2#_>jP9HY@CDQ!z)XlH=tdeS@^oOGbfZ)$hbU_k(Z=>KD@==D!HDU)_`mRd#`lhA z0w>-I=O3ZsOaW+*W|(p_Qx?BQWfu*~PWP6)=i7j+hiWQEQxg=ta+{ih;jrv#GeSwO zHbwwVF@U!I=l8f~XC(|>tltZ?!;Or9aE##**_Y}#edz8%Ip%Cl@M=#u2(>~Z%Dsq| zjwm6)R1A;gGjnAtrjyxEUwm5CuPD=2a)H1Q1)7FH)>0s@5*#VuR%%;c_s{U>C>h*p zSYXJ>>t9G5?@A*_F@60^IV{vMdO3!HL4>Qw zdZDNpPfoTXho}^>qg-E_B;kZ-;BP5JhS*`e$r#JU_0-+Sz1K{K{W?5#X5IbhSq`G^dBXUaG#4uV0R zbCL!z;_Iw}L&s_5wpgOem32`2HCMlZ@f^L;H~7fYxi^xYH$;7a%Rln#*O3Wky@V^W z00C}G<==CAkV00!fF3BcT@c}4rws7d_lTRBmKJ?<-V6@ZsgIEjvo`c+P4w~Y*X zqE3zZeZ97%E(XoNC4wle`gizhZsa;c+{UJu(e+gfw=0&~cT#T@{gnsM$Q7*JB$3X^ zOxw&kPHm(~0Gd7fSlM$J(oBA19iQ1g(5ayYBJ$u0L??Mzr@vG!0&kO^MfJBoj%yOQ-iIAp#6xF`K_hoIct7Bv6!$*`i5l#k9e9O(rtJ}kvo~VWk|F3ly z>H|UK(ipJ8p9(ioRg~xaJIT}gJ4?wD9>UL)u@^m3rO~*BNC5pyA#;D#i zZEityY-*R7m|4Ey>-*A4>!g9#YB(rg8(vjLt!B;;_JHXuwM9c0*8W_wcS zR&D0F{}UC+N|Q*UqIu?u+|$1_dz7Y5|2>o1;Ye>VId z3y>iB{M#faw)V;O%ts||iy`(e?hFNBWx>Z^=K^L#o2cbDlP<1fXPa=BAiZ@z;)e?B zJ19Ua#NjdJ*_iOVh7J0nY|{+}7Bd)M6H5-)_e|M8Q^2r;W{!3lj&SFZt`6U>>qAn| z8Uo@QmYao@96Z=)HwY14NIWtEhJGwday@N?k8%%bvW-N6#YBN`nw3$4BI>vSZb7=x z4DvLiGg_#$)t12mLEa}{eg2-B7*_k^ykFKt07&dIbgoQ8_U#`*U=&{Ao_yu-n_Ps487gllvG14P^vKB+N#UUgjUqN z7Zo&DYF6qg-YaQYPh|ds6zm^#Fo~hqNFB~WCy0WwD(8PKI^v_h>))1T3+wJYX}%;j zhl|@hpI~gZ=OX&OdoqD24EMQ9ci{IRza;CF=*(&{GYL!5VV=P9-vkO0N&)3JpRe0# z9EqP9@aWjH+ZL9+;sC0B)~wP=u!~^R0=gW7A&Zg_meFqxPR5CTeAe%I0v^}bC^wjR zcqBoQfGAYqre99+g38N>k5l4T8LGHYceqMsW8>~)2ih+9rgl)v6L({;upUGQsFkqQ zxf}ETN%krKPB#D?aKfO)xo!-Wm6o9)=(Gdxf*y2x5m**FS#%|`novpy2q4&`Gb5vA zf#woXl{Hr=oCNc1a$_2c$IBr)EslNWrEyqL8l-1zyLXp($0^n$yUNZ$f=6k<5l_M__ z4tncG#9KS&GAB>4)6c<~x^f1fM1&a-KUfx*g00l3MfbZ9K+9(q9&Ac)Z2E34)&Q$Q zLGbqZTlH!&M;oklS`(8^Si0Gn6HKi}f=sH{i1tfa9MfSN>brM+!fke|FhFPDp9?I& z17QptzE?*~)q)LPVp|b15q9-e>O9eY}>EMLpq8y%H= zTzli|q)*@*Oti3CLD6m{WF!Qij1U}4LJG~&|2c!46PXnA+br_C3X~q^+P6C7gH_~% z^G+K3st}Lsqco-SUQ=yJ)w;!)ufJQDAUiJj(z{ox*J<3Bsx6lZ%?NriOUfaA^MCvj z=xyLU0@hHWEC{gS!}txmndGaygrJrixqlruSq+tqDzyDO$Ui%78bvk_Od6+FxnA6e z=xtt(ZY$gPxsj&P=!gLiC$sG4fI?N$N87GH@T2?xW9loTvWm8@=`JZH1PSSGq(MQt zJEf75?oL4(=|;Lkx>LHOySux;jrZPnd}H`S;5_G?XUE!W&pFrJ{wOrn>yfi9sa@wI z2G`dyOEsvJiCqW*2y|-RwKcfbidau(6Xf|+-RkjGPwyD=&g8J*8#&MBR?}D5p{cN8 zc?RaV(N|3g)~hvj&4XGtUU3{fqm{kAd$kf5k9wC?oP6(=edoz{>UZ%zoT+H_t&*t5 z1}a8O_$#C@{u>x{%#dY6l`N#wT7G#oj(^(^>!GkW&r$DX>pO$`nw~x+zU5i4W96{y zP*nb;ceEfLMe$zfBO)wLKU;zGh#M6+mD~3I+U_PZ=no;caCb@V_=T!hK-urf5*?<1 zxH$@F*f;%;-|orkNhp2$KO@r`FNmmGjF`D(hD;?HQ8`>e=hWMhgw9*hGW40VO-7kz zYdMVCqa9`GnI!1c;*Wogo+QF+%2PY?6=KAS^h}sy;V0_q%uThfm8jCuH_tOI321RUPj0EaM%)l| z^wjxnZYNc5fd`9gG@etozIkf8w`|-sW9{b*K)OESY0Qgcro&<9+BoKA#x3 z^MsZ`hbe5S7syuFQ;NCiW@qa2{~p5=NtF)|f8snz@JuSimC&R1$5{VRr?AdKvY*cX z_W5ErrUq|9J=vrd#kOghJ_!TW#rZl!;wvr1a0}8rb@}F~0pYP%YSH#HzjD-6=u;B7 z_rB8;GW!@_andbMwuDO>hETV7fpjvJz5Pl-QH*z17rWAJ-LbZV_2F{Yre{yaNm4eW zv?=~lZE9wD3h%y?N9rXH@oMh-^v%0Z={<6AkA2yM2TOv|v-W!fCX(yOB>q&~1;w2) z1a~vlJdlERVd?K#JFzJ8%MS=O(Vt_}o9|rUqOCDU?CtK)TI=#-*VXP|WGx>xb!HyR zVqw3I&3&01TyA&s!6EGRkeKZ={9`d67&EYHZr(XhdOQ7Iz+2hGGq|s>((a-MU4bCA zy}Cj9P5(R9&y_aIPM?ik>uj(5ls?VnWCCP6mORYuDK2}FZ3R##7oSIpcu(h^Wv5s8 z&m7~z?puapG+WvP_hw){ns3Y?KMhMvBy)8DUnbQ1MxBP0bejcL8c;b`>2{Uj-F7gW zy{tw~v4xu7UU?~3nA5`)YhjOX(79=^6TUKZj96tqmr8iJD7@5PUS`L4FWb}>m8M#6 z(`Ta2dhZiD+>1>Hd-OG7U-+o1U}|Hs_~; zz;Q$xq_=N3(1}G8DcZY3!4bmszLGN)X>h34=@b zuG#_*L>Yz$3utDB6np%lGW4oSOjgYLVgX6r;7TD#=)WxZg)F4;VauenDXNN@-rDxXQ`>mhgz8%E$Hx260Ab6n>>*swMP%m?wHNSSPVk*Y2Y?FU+SSNQZ2mMqz5UVn6&%J380S z#*1aeVNw~KNB3lK4k6pbCTLw4+8_!a(QA3QXq`LzaQ8}OaQO7e+&p2LDEen;-xNGR zR8R?aZs(>y7wbk$E*_RDnTD}Q|1gbR=?M4>U_hk>kJ?!M^gqW&nQbd0UNVloXu<>W zSQ-^jy5_XTpJ4l0`8&&fb6gx`eTmDCzudQDSHwDe&FPSf4UpRU9Z+|_hd;$Y7w=1p zrDV|Uex!8?D_7?ez>2MQaLTJ~%RO>dqV&>PQeTFAYsmTaqJf9t{h!S$YN zG-?(7$aJkA)=0s{91|A1qL(;tz3eGuu*TBlxzW-99VPPN(7NwvN&0A%WDl?ryp$$H zUHb~p@hFIm>3pwlen#e$NhO;Z@2k+ zRO9?Es#mrIv==T__u+tmz+jc&tC5tIf|BRQ!B|S8@{G;EOmF?&PtO+X_@F4j^|R#? zilAOQ)L;nGuBE?oXeVW*`+A>@&U`X2y*>edq@pF8z-vO! zJAFn0on;Z4{U2|rsIbEWIni=%;(?6?jk$qUGT|@>-Y*Wzto5?zyJGJS1h{slZ!CD% zN(OCvyP5aIPmr?8IKR^$HgW`t1hp_g@MnA6)NP1?TfRlB_qY!RtQ{CCPl>FEC0gR9 zvn;;Iuh-Jq2T(D3sY)!@OSzVBK2i2I*mYw|z`_Lj25Pla6S3N435?XHA7vE|4oDXj z<^I({&H;W$RLJ!r>)%GZQZi6S)wt7NrXWNa@hx|F4K>PPgAUMkE zb=mgc5lpm%kG>QYzue4^KSXbx#^0NoBVfc<2SgjAqm17FAv1x&PHXtkZ(ScyxnC** z>t|SFO?fqu9zFXTnGoLX8*oH%_xm%c+Gy6j2E`LLk(M35GH^dPzt|>he%Jc!Vb2i? z1c#ui_v-}354k_|zXiPbSBpwXo4(>asaKtFu<2>7JWaGX`}BIwcUGin*}Rd{-CR%V z-lR)=)x(CvgPwOgR3j9Ay&+idR^xf&&UrpBB4v&sl~~(KXLy#Gj9+Jv+9<}h!)jWZ zSO?Dn%mlYfl_{Y^CsgTLq}9vON=h=#)zA0ZiK7Rv(6Hnl4ijLp*NK!2ua6u!X@ACJ z)vbkmfo}*XWdO%>iU3Hh2^AF-^mb2%B?0mGPUAA)ris3Nrgy3ou1Voi|BuU8eJGK& zY=Mwzb+3!2;kQnV{C|8?lVM4J_u>Amsfkr|X`E^kjp(?(6yKVwe@jFJog!i_camQR z5qG9?xg>8f-Fv%+d(hIl0f(4`QqHsE;Hv=n@;L ztfdangzicoYiYFi*Bd>o8}!b2@ggH#y&WibvG%>XYl}^cxo|^`8w`p4JF@U^a(S!{ zwslBM5C~|wFIPM6NE0&16s00j^-yf?g89@S_pZ?Z9_WC+wL?8rjqsBXh*Rv|U)3_Z z^M4h%73ZCtT;cjvYq5H>&EH!w;vP6v<{q$v$|ge@c{#Hj@Hx2#+ zE0@SXVu|&Q%lUn$6H5Pf_LX8f>0POl_KjN+`|v3Aw?cGd#;GVQEB+$YyU?T^Al zM{4q&?cqXcvkS*6{vkLsZgliQpTr|f*%J@vD_q?|pH(k<2u2X7AY!K_XMayCoR39B zFIGVF5IJ!=wqD^o(go(_iq6^A-51ry`w%~k(i=U;1^rZ5;Z8ChBJeOgN5eO*>Ok2j zJWc#+p~`>)tt*MVbmw{|1}vkf)o_6u6!=#VMzH(5bbV4&gQ->Sy%PQ2s_^exw=)3& zGnCJ(`?j}1isY;u67>X?%gvS1PG$LB+kC&XcF*yPwci*$pVG8EIa9i5#tiWsxUfpM ztgOUOjvx}L(Y?{A7;W<0KqRzGWnZE#ZB4HbuCO2UinBqcDfWRJpRdXE87)+H-rpZ_ zmA+aL(wF5G+Pq&1+O94{SYMi+mQg zq6ljCuG{u7Yn{V?yjRcVZF=9neyDQh3rRv^yc*3C|I|fxS;{DUt(PV>3<*mF8*1!d zP;!!b`e0W*G}b~-wQ}Lhh(F_?HFxqO{I7q=nWC4#!T|Rm2Ay^uohynGrW4I9LxchiaQ4%x6hA$uUup4 zG`n^^24!vzoc^6Q=v^dQ+S_fHp?IJFM(Alr{sXb+jCK;CF&tgSYIB_80qG!W2-dP{ z{1yfLwFkdP?*%t7f~#+K>pNVOJorw@lr8T*kq2&>1lr19wTyzvWEihV>DE`ox_G6u z>8n#~8_q-r2O|2u36^Ep-8Y$#6fba%JRJxssgcD6CV9Ltw|s;jIxWy5sNchee~9_|R}3QL zZ-JkUr3EjJ_(;lkc@Ex_UVKrjFr~KWt}F(0pVN=+!llyyS-ab zyL_xHLa+~X({J%OWts@qqQf|3yM;x}eW^C#F1z1bU8tLBXZ_7W3agPtg^T@?!pg4f z(rl93&NUotR?%TeylvPphM?W1iPg(sW}epMB8I1i&4@M75#GC~jFuzk1L31yHGGHf zx?{K#lN!XrwJlLIoy{amAjmWPx1T>a#{!H@cpkJoi2S09)4s-}gt0et-_u)HfAHa& zz~NG=QURyZZb@9ZALepVF6*epX+*WLa|k(xk5wiUH``v}>heCfapTS(Qbo7V6O4?g z(jy86Pwr2DS7*4le4*|9RfYEbhjQ^223mr@-`1=FMTxvm8>}@1Gy<&DR<(pTrwR=j zk-LtcuaUuQl1DGgFBXnhUhlz0(;2{p8@YVv^geN&I@zL~x@9^l!|geZkld88 zAL{o8QWBZ=EbG^y_#kQSvaHxd3gAu^YnBxR*Or*dV^cZ%g>uan9xWw`#wad+z<-__ zOug?*o;+zGAyr}nNJ|I(r)R@lx4w-U%vIEX_FRk%(xD2l9Z7{AcJum(`FZz7n02{s*T+Axa0>%&Y}8^NPhO z&+k;Lm(<*~t&7{x(+;;*+CQGo)?I~Aq3dWKGbj&;T4I_dmpwN5K?hAx!Lo1a9MxAw z%?`C!^Bva{!g1q1*ch*hL zC1=s^V+)xg7hc;uNBe;cYv3Y5p_zc&x)3WAYLeUi3yo=gi!mBZ)|_9pW`bcfHb7OU=E1 zjz4IHxOR;X7L0n|t;A_KwGTWUYj`GBpZUQKW>@;r@I1Uxe_eji6lv{RrUdO1bqJ(t zLE8QLQN;}@gmu%cZEdoi z!WX&8`OeSx@>_pp%9yj|JBn1oZjRL(BmCULqXu~14L5Ivhx|t35kl17P{hmCIz<|hkr zakWgJ{j@XSZM~&}7I*TyjpKvEX-BfsQT!y56QLz^-tPk=@Fe~EZF64#M}}0qC_5So zpV1H;f0g2QPZtwb#Enjjy+DR^7KHXnrx}0qJc`FYxW(EdbS6d?7ru8Xmm;fnfn_aM zdo`SKSOSlSZ0|%rQO82DzCeViEcw;S+s0W}o~VrL<)^TtGS5pTL1)*&SKn7GzmhV! zae(FId$G18*n9D^v%juJ`t2@*XQ$!YZvrF!1_OpOn#YhQe z0D%Cz%Dox~k1xr{873;-f>Qof5)R$hMoTTI5J`L`kPrh=23I}dcat=dDx-nX?!vyq zWw5VYD=6xu<8jh{91MSJ|GX}MsnHPIGvY<`Ty19iyGgrsm+L6Vk*Qjk8*Rq*Cc5(V z?w{{;&o?tRuKKW}<@S#LvWt^a>YuNawSp@n``(%t&DwBvw};b7v?nieKW(0(eVw{q zxu(XXPJZ$a*xXrrMXpLuWisR&%(h}icnE<;WkPj1B4iM8<9fYJ2x<;Y+uGY9d#*Kf zSOo@o;y<}O?$ICTdHN+ZSPGU~ibbqFb#8Di%@Qq%P4aiPd!yG=?i zxSi?!({u_K5RO($`nSfC@P0y|dw$gW%oQY86|Sx!RondlT?P`mVogd^XXffg+?}WC z$9uKXySeW$d=b3D)A6lRnYGjO$GeyMDoQKn=492j@1uU3BQj_Om;J`6sZX`HX8Kz(N1R6AdE8PI4t8nrOTh-S`0$*YC6p>uM+Mw?a(|Vb zT~-2JRfc2y&?$S1BS~G)hyDVOAp6IW=Uq5-@0ib#1Qgn$F3kYM;e6Si&&!N|ALtG5 z-M!uR%H2PHCgO>D$7B$wiSO8rBrF+Mr%pIDL`H<#$%5As zOR`a@r_Us03gZNCp&>T}Kz1ma@4q5-$H#Oy!(qoWx^R}+^qu3F zs}wA-i)$VL9zVyej|8hfX$u;T`2j9*{p;}Y{&XbieexK^Usky?gwOb0XU^}WT>d9?mlUbnn!$7W$=T`QJhqy%#REx&f{g?=HMrnKL<1IZP^hD zjzjFRKpOZp|Ci3{jm_E!ugDMrL32O?ojZ}=e!MbM8GTn^stw1n_$YfqZzVf0u&oY! zXZ~7L{R6GAi<*KCMfPqOW9{wPqE7pwe^WD&-Y%TzZDI9Vcm6>1cmqS&vJ+yhq6*Q8doY^Zq<)c!;xEoT4a7Y8lhVlMqOX$@U#aS{*NC z1MH~2tqS`h@5~6cD4$%qMC!2} zxr(&+XgtO-V&3PsF;Tf~^B|UJY)-*@n)4@*h+W+6kK|9%GYFJinsEn@6aACUV=8Hh zw#4DS3`!SOiO=Mwm8PQpJ$q0PRY`t|o`}E;*dHS&N6!o`{SWSl*O?p61Ob#kRP=kf zXrTZqzC%>Vr;C;veLVH_>34LT87~@eiO;-7r(u*l0O8<+DX-`s@$$$BtGCq&g2B;u zcb5KSl&B^8@@OjU@IbZxT$9nErV}?*I!)V#_x}XrKp^i<#T3}TM@?NuDHQNV*H1k3 zFi64B4h;h2QMfYb>z99q{#Ejj2nAqdf2*b)P$lLQHEDI5+gDcoc>4f9PaR*``rmJd zz>FUrXwgm*<0NQq3oF(SoB;hXvMx+zQX>LUjECd#)|!m}rv)IqdDr6)B_|)bd{ul=-BTwdJc4rHX}f-p1MW6F???Q*RoB za*o=QYCmcK$0we8tt?jT-SBdEzx%{g_ zqrXq$e?=Wsu6-epPYAd7LfKBiu_%al1)^ugyQV_xNziS7L~a2JzD9)8wV|Mv_;8h0+4D9}G5trt12H{|2WBiaL+g-7jM!&r+wW;R6Eq*N= zEZG4>vCWnQiJVJq;uK@vP;yAt24tByvXT5kH&py2qc)Zi;d#E*MuI=+s_y+XA*@+&GCQ& zQs4$9Bwt|%R>4luc@D1aSb5L_4HNA%sH&prsxvJ@KLN6)O+y^@GO<_D1ab(>q5#Nk zc-~FldFs~+K>5`|ph4(yPEx+ud`pF`= zNxJ4k^DLhZ3$QF^<6$x?=6cf?mwOhbBH6;hJmm$jQDOcDC^qRydmAzFI9WD+Ia#)5 zZk2nw8bsZAHMdG^JURK-^I;tbqf>6i5^{vpAQiP^tZ$HN&AnY&&Y_h2Ssi zsC=Jy@BU}59Ly0!>eB_g3(xap;9cQw!cnt2_FI!#Q7w6H1$B`LhonjZOFcAOPcaZ+Q2cn|Abn)O7)g z>*HniocCcmf9jv#(^;{krya=TXx}>!4#1NMn(ya}!UBmo_;4Zxpi}@Cfj~0ceymuWNKBtrsI9z} z$xHX&F(2zFH?|-(%MPsFs}YOJA`a5{0AS%cKVny@UU`3&dG5k{dFel0OkL{S#C|gx z0HtC6x9=xP$ym7Gne&D_GD&ViWkwLZRsQ-$R=$GOtozFnB^->I`{3CR3^RSih~Il% zQ08SJ9(ypdq28n^IY zQl#W}T}pd)*f+e(E1`F8p zb)vuHC4xf61^dCQ=f>rn>wR7Lh@g|b$s5raW3W!Fp;Ri6Fi2?U#DmpClGoBF&;OjLrTreR7X0eq?Das8+kPD>$kv zuz$8UP^r!{Dr8nirp)uY_G@ zP^);cp!Nx23;zgP$@=uy{lI|m`CNeVX^{2fR0#nyY~kqSr>~uG!jka{=cf^XN3g!X zta7{3Lq(CZ0mV0^UqP3*D)p?IFkV%f$wGN=;R~G431cdyz$Yr;zTPy%_>zPaQ9uqd zlOKqd>fO>4Kd%==Zs}_yfh`D9YATthjO)2E(&FOZr*>itwepM`hc|%UXTW~h{h3OC z1BVWbP`-Q-Gsjf=u$W8bXpSrSN^kC z-w1&c0_C1shCysC;5FDJigg21dK742SN&9^8}=p`6mS8Owx>A%9g|y4bt8siD%9qa z7}`<;MD!CmgM^rMxUR(kSzIL~03TQ4*e78gG0Tuh7%tjNfk>{;dR7=vma%3pp^|ie zDr&JY9*|QWQ_#x*Gv3?F+H>-H4P*llK!tc=UiM0>(s0H)hQ~SbV5#{AIA{N^m23pk z6BwMFPnx*Ee-{v;qD*Szt&+0EZ6i7d+5cnyry0v$R%09arlFy(G|!|#0_zoXE4a=V zkpdXQ02{2j8x}U9G-eC&y0fp9h+U6VYbBzLX)n%ybL9qxOz+V#Uhp-e0Kv#>bnj0W>IF zNzi!R?_YM3a?G#F_7Z z?N?SZwCMhb+rKU}U(>j)4?<1wM+62nZh^DxL-iYtet(lE-C11xt4Qvh(~7qR6sB z%(OHoD2It$+RmF2xsUs;S;zs39%W{0Ag0cgr3<*mf{A>SE89CiPKwIVg{_Lq=5o>l zXVut$DB|lGlZ#(~Kq=G>gW}aJE!{2hGgd4~8-Zl0borvTcAa<=?rT++@95cP_S0G7 zv~DjL6c1a;L8uDgw3g6ilX6?DM7EFAusDvSJ~2|h3P&q*Obboy^P=C5tqKd80}Z=cVc*4?8^03R8k z@cKu*-@y2A+ifgT2O$4ZQu@eicy7fekIVMHND_gWL>(I1u*Q-MFm8SEx}vat*x;xh zFdR%z*IpHUL?p>=GxQ6zpUP#jUy93oN62)Vz>=RBDHOnh0>U7`bt%0z5K_G^RSg}F zDXI+-FVk-`fNFwH`bc^A9W4Df0rsIE^&rBSOVC5`G3@s#x5+~mpj3V3{& z4MF0Bm7wki-n=jB!WpEu`ud@Pe8owW++cP3J~BV^d9wMs!Hc{8mn(~jVw_q)!uT2N z(0-6YQ{})=mRZ5&d5ihioG%m>ayCf^M~mPK1BOO86jxjjq-<;BZssTN+QEOSkJ{vr zP^oqaB*^kdYov|0Wl{_=mw({BfC5fgA93iU>5RVc99g6>^%3#Z_9o@LhipzTvc5n&VjOWIN9eO{}5vBNT-KwabR% z-glhQVc43;bvZ82l!d29D{&i=9*d7=kpiyRly0jd@rH9--&Etkxhnh_FlMNJh>}Z< zeSn;II(X-F!vuoVs&Y|omuHBu>g(Iipb9L+N*<@RUlZiI&X6e3RmCXJ#p>@hX|<^x zG|<*|;K^#TZ*L_%9*i}mjy^JPS%^)rNA}8X6L%5>>K`YYJ)I>7F&h@1EkKw3_*A;0 zHR@j;Gctff;lGP+s zMsp>e4e{l4cXK@Q{Vha(qNn#pil+P#3c!kpS**uGy)hYVn;&I*kN=_0^i})q`C`%i zM$md4nR$+!&T%XvIbn&7A(Y?`F|TY{9j7_%60T{7JX!5@2}~O?FIl@SiBVu@+;@;( zr!~DhpF?CW!ym`4apx^QEzrEk`VRl<Rq{B z`ya}LuVF8;jtM_ULSAKM=db4SM6%)>)hyDHmGysBoYlGbc$MfSK3FpAv=`^rbcmf0 z!n49X!r5tvD8;#uFRpe3S+c3ziE-V~HqpHVil0DiWCwPv@HQqxh6DLa&jmmxav2&YYQq zGeeICtG}$gft!O6qptqa@t)UsX{ z_Q~)jKkn)#JD^<6u0pEzNobtE)L>1&QU<6))ANuLykkh+dM+5T0Yg%N|?GpIO`cx|GdE1M~1m;UWP4eUXU` z0(h#GEs?y@tCgTQ#icdFr)?NAH+3ghkNZs^X=Sb7MUMvW)+}vnKhw$Y+Vz5h`V3>l zThteoVYsVMXLTUN7B^Ia=^qg`a?alg&Hyd!p;6y!sYZ7Q$jjaGy(b`H$jMlKw-cYZ zSI}R{-s@)fi{*$PKYco*zDIVH@t%glQ0_~w{x1#-K_0Xxm-E0y*(e@c|8~l>j6G=L zZ>lUP*F(jgCqidp5#yHvPw*b0uO}{UN3z|s2_RnV#d9eJD=U$8VVR^kmG|BGm)QVW zzMjngXgNV$J>{H~B>I!|=i=|sPU8DF0i~s1TCF^mR@W01lLvf9@CcC-O!Dwih8Ebo zb|sZOR-j-1rJEq|bAR$Xv?8KeIj8D?6hD7)_4v3OoI}@gRfSe_APY6Bwz=NAwA_;d zHhse%QE0%%B4)7)J_j32cBYQiDzfe)t6B*Ke%XyV^V6wm=nQxM{?g{SBXDK`i)M`; zC8l#>k=UgNwB456bK;2B!YqbF>b1}ZQKpwA1DDgFRa$A)GeRB)M}}WEvnoc|>AIY| zk7(s0mF5HL)aTZ?ksz5o#ya&TqEE_+sz>Diw)k|tK0d!bU3@15-Q0G;#GA!jeB}2N;gI-;Sz!ky zb_-)ElpB!3%Uj}EXO>>{E0$j>vGzLE_7wB z`(}P#3+*H!htih%FI^#YCM5`|LD|~e zEoa#VDLYQ_$Yoj>np)01gqtx(pp5t9RB6sZ!@gYmJDCqdD-m8&+pk*$9iKfl zo}Z3Sf7gWX?NdjXY@G8!@O+IyBJWGa?}R8ShN>z=I>*FjDt{}@Vt^%E-^ih$exH(P zX6*?4;8R70TQS#%j@bdVKv;37#$r%)|2fH`J*uYx0acmbm<={a8*dUH&B1tQXFyWq z`)LjdCIHNFNmmG-zRbL4EI(F*lRVp@e4nG|Xcycdam-zY<#DIm)rCl-K)}KDk^~oE z10hKTIGRjJIclyh${_uo(3nl)5*i&ttI2u8d#wwxq2&xGMJZ&9(JS$!|L1!rDrvCT ziCfMH-dfh;_(66a=j|=IfuGv|)e!Cc{RQutZ*@_cP63v%KI!N#`mWr-&Zz zmVl{~hp9Ku+XYzNcJ%&l;!ph@(T(l_hFh;cce1!tkp{MdLDu85@$q(L3>~TaUvK_u zOW!n^pfAvgoo@v4dp5PG;!FIX6*J$e4RE>1G$TukZaV*##OQHM^}U+jqs@@;Bq?NM zX}&+KsBnoNY7$dXsxG5LQB%@%k_G}`KPy&)Q~26c8OX4)y>`P$GPmJquEZ`do0sDk z(_Ud-Jt);i@;iGSnwmU&(UtMZpza847}sP4q&MCkQWju4_#b`R z=-k%aR;4CVnjvmz;i4C6gHAKf zO|VTkun~7y+C<4xgUy`yOXyQO{_P|B4ayY5j|QQ;R*+@GvBjOzFhs=6a@O;o`?)x z3FzdCctP<_J;6B#=f}|ASFQp8WMo2BucZ>l2l%b=bks%u-M5@K9F~f)-@q2~;@g__ zQ{dCF=WAAcBOntCRaBp=u^Bhpd)|o{ZbaEPI$l{)Kox(Dgu;7A1tc&9kD%tXOkHst z8kwv0ErEvLuDn)jQ?y%Y(a#)X<{oIIZo$g7?_PW#rv+D=${5AWbJ^i;WSuw_anJtP z-*H<#<#D2xgAwWL;OVlX(HLa1R8bT7B8Q9eP|^Q!@4ep4ef3tBHG%gz0}2&6FW@%& zy0(s>ar2C>zNOD*W!cNYcTsHOM~xe|frCV!x=s$U5(-+@gC`B}ftbmQJiA9M(^Q>v zL$^r2QIusq$tIRR9~JrQxu=*A^;!Ih<~NSck(R17-J#GVh-Q-#ZAA)$=zuUTI(h%l5)Nw3TQN7o~jg%{;F>af(mI*e~R5h_xMv1ym<{=n8CS^ zAQz^p_rQ^0Udwa^YdK)AIw~D1gkz1kF=OLg<(SRLj0Z>oyhN_+9<_(=mGH;RjcPoZ zQ*me4#xDdj1#vWwl9ndmzzpjp5$NOQOIDN!1rP!W{89i^h37FSx2MZcBcNt=O=i_l2*HO1NjH#bs0Cd z-ser}e^;z=_3X>v1W6#cH3uHN`uTrc3o@vM^RlF$rYog-YHsPLL?wFxE?=528|#8V ztMQjGKESUT32L|y;^@R!I8;ox(9e2g=KkG@1~Kx-2@KK$}~Dryap(iJ^)oD}94 zDc_sQiWkHT5G~LT@#xjjLp%^lV$J>!xokgP9v~sgYkAtN>r{2DW^Bs4zS>T(Z&rf* z`9GW%;JAJwXZ?Rn%RmbZfzx{k(}GL8?FM|aH~&X)@EK7JX4+GB$|+Fjf~Tb+#ZjbH zhHwY-mkF^QM9uwsF^m_c>O+vM;Y$(0?spsrH5S&vUTbSo%-H{*$^!LRrDW6}?#>`S zicrMSI~e^p2f$e4rOIK7750mbc>$U~ijU)`tkr9>2+{nnskYnMGAjCefAqYAxK!48 z&=L9nsX?%f8k@%#H?f3h0!j1NbhaP=@M-|#{;(CapaD$uIC#O}0daE=4YaK4pnt6o zE=dWBT-I}x4DM%P#Fy4gZFT~!m{4g7vTa|UZTb%J&QSM-Jk+Y@gt<9r zE>RN+T88zYSoL1;{?{vSPh-1VpR!^s;T0eNzj(8bUd__@o+=D<^c!@~G9ksV7a{|2 zqxQ3CDQ~F0{i5Z$aFSlK{q+T~Jui2_(7%X(hKU;56ptDHOmF3Y_ zy{S>zV1Y!~;}ojzN7%aHsI?rpo^&mW&tk2~6!m-cC*1{7cBK#O8=Uzeu0TfnXOsMe zc^b;ZQg69f%(h3}7;D?RmFIV}rrrEfU(KLWx>9bW8*QRXpf{3ic87%qnhe+t&DwJG0LNP3~1h(lLH9moUFtc;wTHgFr>tE|WWS(jzS_fLDifU91 zEX2g>DLA!Srhb5R=Tl!x7$AZPd{bp1?Uf}cTMSW0kov3mpqoHS14K3=&$D?QP$|HL zFB3r!ZofHG$=KJUZ9~+Q>ElF46R#X9-sQ*!n8lOBJlS#wuV@RQ^wTG&i_KXF@*$wk zu+r5Wf-fzK@o6Ks1C=S)>4#hK1ou0kn%29}NQ8TXXM1UYmkc=vWf5u|jVOIn7-TQS zNIcJD-DSE2@8YV*QX0}^P_Vha@ebrJ+JG>7(oR5-uuK#!3fS4;)4bRYx&Fgn8<~z{ zGzq99KF9OpU34Qq>@$9FIoss7h4m;z=i)ZV z#t!tgSX%AoTtlI-d>Y`Rr%S|%>cKMMUP@j!7ua8^r}tB_jmZ7xRrR6)r7NbP8f@7c)LBkMVDCn_kfeDp;_1c*C+ zxn;DCmD25|KqCQR!&z!xW5N_h#u&z`_5ZX0&HV4e)Yd-S+q!Q98u;x1wqIYyNXGAB zS%aSuQcJ5R-|3n~iu@8&I%62GnTxppxJ^xcrqTLdWjQpF40kh`>!UCDg(|<~Vr*%p z_n3WWXUMLYQmkf0CHFVh0 zn5j2=!1BVJYcbRRST9e6#%Qz)5*`o}t$pg}tvEVe{PhSg9f3L5Rv|5m9BN zTbUrLrntJ9W^V2>eTGyx(7b`1#3wf4emrZXGmFy;sZX7(Yq4W6)I^0Et*bj8?znx^ z-NPS^Mh6>G4_JCJnb5IBnLEo$?ee8n`wpuy~U0* z>IoGi5_ICGip)BNOl+8@N?mEN7Tx|)SnWooYOp;ZkZ>8ppW6d|H&>!Ssz%#FfzNWA zy6I;D_CO5=~+O-Iv|sAyBM0&JCZlfTew`r8)F?TqNu2T zOViJbLQdyMEZyiAWPs;LBrPr{zXpD^six(OJ%Rxk)IQqnpgUJx&g+4tNkY`#KA{mH*Zot(#U zneF3a=rfH@uFfj(!RTtA8Wxh9?>_7E{`ZPTlF(-|hQDCzQxU6tu%a?ZZ7_CmqF#3H zf?BGsAf&179h0p7!OgBa_Kkc}G+y$0v(Ig)bM`fufPInz?7-XJDvRsHfUWEmOtBmjdnYH(&e66zZ* z0?}c={_N3`xA(}(UiW4vk(wsXrR^dM=etRKi+OGK0i2B!PG;AC)S+~J{Crs<1wR`` z+8A~h<3PY`=KSgf_%^vEbGs7aX9QNtv2t;v#5}cbGSyMTvoi>)OCdoD&I_Wi(k?S1 z+TAC#vP2PCE#pCEG!~NUTuJs*j-0v@NLu3KbM$2UrcM|;msI$L9%N~skBpRTa05Kda6rn!% z%E(!jcZ^`QV;o}lE^0&)wI%b8UcNs;z?hI z6WiUM`f6*89#3Ig(M!KCRItDbY3#*|3o~@wqg2nv+xC{`nnWk6 zh1x`$yCq4jIyq|;;rhm)jK550dBReV0K24+OK=3kuH2Tx-I^e`kkFG451Eeixh=Xa z3fWBSXh7OEJ+>_>)XW}jLsCJ;{ z++CyIS6XvKWW$|@AxYsmg&)q9P_*bBz_D<*|L|RDSbZdiO2*3`oRPsnkJwrLB>^+n z?0kWq0>_?KoXbf{vh@bFiNw-Dr+U)m-74>8{b)?5lP*D9z!BY9gSJNG?NJ3x%X1He zD65K%P?;}1tzYmNL^Uk^j$(yp0kPE(cPOc5`DwqKi1WD|Vc!o{_ENo>x`p^&C*A$t zimPh7*O-EmVesY}&~#m}R?0=_4zHYc7t%YoCvKJqpm*>GML1J9-bHbo?t3slY&@=x z*Hr4%`brP@A>1V>jjBaAn7Lq#l~*10#I-f{V&7| zS?Cn~X{SRT-y^QK;3?fZ8M0J%8LupYaRG%k0{HbDu|msB9bQ)_e;z8Wyg7;2{J+_K z?NZeZ&xRq^hD2=aFSaXch9bOWy9K3sPbKk8c+|PxcAvR263uD#6`+UeVU(MQykFu@ zYAYB}4_@OYU-qmgux3QgVBQmwskNzL4L#RZ$Clcr4j3s}8FM2ANp?BL>$*);e%|UF z%w}s8!CLalhOqP|fgh}@EOkZQ=n;}a*K&}(^Ao{2gX0YH1aYwCZX&R~LTv2~?;#In z)qK)h1W+_Vau-0{#O|GaXhvs!A@oYHSCEL3!6=e$=@(Rrv~bM=jy@!23{ryw+!~yY zs}q6;*>E`fbxB!U8Y+$yO{tSy0TQpPEX;Oe<;`Zx8!5$(YVu*NzBLOI+^mF;TFu|} zCEwvsoz%k@R#eW8Kxx6mvAdq;1eu?v5?^NsSP8yr(_3gu4#JefHi1x2GGK&Mc92o@ za@X`1-y~hJp}o2Akk9P&aw#{FDnYGN=Whq z7V}MK0+u~0A!U#Ah5wYr!C|NlxHN6yb9!!ncjawnb=~I6)hQ0EBt038LwH(A6s0S< zj>BDr^0tid=(a$-X+|QN#cwC(pYRT(f{zXTWM1<6`G_^F&P(xqlU0rd!8Zrjzx@=% zPdl@spuzj{u$89vR0LX{w8&~o(ArDBD-f_`@{4ba?*%RtCz z?ELU}$A$x~v!g0b++UZI&sz-Q6N3-KmsxH@wo_AR#R+aOk^uzwaCO@BMT4560l^apqZjtr^dJ=G<$cTP?wR z%SowUjls|VcpWVA_ywbqJbb;)G=Dd4kC9l($FT(*a-iSnrjL}hs2V~rBFyM!6zX}= zIq_|2qeUOR@faxqJx?kZ2y|-s=J4wV9e68Kj&`^36;9HAVxA(5E#Ts|vcpZo;Elgjh|mRG+t>mGTR=(gnso2morxOq?w9@RVEQ@U6#O2MybD6pZRfR z*AW&0B%^QNrPa8nAkd;(CD^I+1!=v2R_8a9l`oK4*7Z>(H)>4+i;B~nrqHP+3pU$# zsL;|Y6WO+u5C^Pm>8_x3miO2S%)72mMHF&MVzkuM)Fx_ToC0MVAFklC;h8d<*7Tq< zl9G}b5zwRBk3?FP7*uNxXiBfl)WjO(oMKS??o#Q0qZ88lFUs+FQ?QZHQ&Z=K&?LP% z-E<3^SWQ7FsHr%`c<%7+BXFTZlHzQFwINQ}5#iw$>&&~jl-aT-kE!?&^P$Uuda17( z9HlS80I1XG97Z}KWRX@#is<55qesM?E#ZCR23-iiIBM5$(O)sOyaeLpou6%=w#}hn z%aeSD%+?{D%t#+#pEw*lKgpmckbLubSbN@&K4R6F9M|6lA+*oO4E%8TR}JiuK6_hswfm&S&G)aflwA3=he;=Jlk9kh>e?!ZS^bf5R z`4w!v3cqG8I7PhW`a^;F={w;c`Cq<6()sjNlA|k}ZeICnl*pqaDO6V+ z{-lWjvGoof7YApeJAU{ptg>Ubr}`2TlG8pI12i9rCEW)~_7tPs^*}JRB}4?j$*P@< zg-MGE<_L~gcWA$8B+yt*3{0RTr;|7&61(qlTf_Eg5(^y(*)Fzo4o1-rBXc6nVuUT0 zmr|deM;2*9!TECMbWUVn+)n`oVr_`mpZ)L-gBkw2&XXMQ*dIHqXy1N81}ZEA$$x<- zxR3?DN67rl6htE%>R`qZq2KtDU@>$){|t5}v=De4nGTdidYX|5Xrn%Sx)!4R(uQ;o zTkst-M(BTU31+(CwFU<_Fx*RxN}#5_poq3@-Oua*wFqz}BTI5rKrsn8MsUo%>9mt>|8ZCjlOxoPibesc-pNE!!Pnk=qViwPFlN^ZhLRoik z$X#_Fb^U0ZcKN*_fv}rzXS;0^4YSvUo`G}6rqg0klG;Xm?Pip`aXP)Hqg@C!WpK#= zgR{7^`tonIot_*TK4yeo7yO!FQ`G{;tI`~#iPfGTbgLp+onZ{p?_Y9Wk)uPFlCe!L z-KIhy{3vHI_Kw|6^;%^kd9TUgtRM(Vp}n_aOV6qymOhRq*b4n z7<5OE7BLb>S2ntIfg>O!jXr>_^)gO+Fx1$UT^e15Mlu~A4LZcMVf)jmF1E+)ex$A3 zJXldkz)}64Pk*reeeinU&5L`>dI6U(>a(ruN`_cpt7{ETJzP}D^EI;%YGUVxh6)3Z zS2vKGx6U$vSn@PK;+>jP-3KJP5OX;!-cJKBUL-07X8eg2+%6Wd<@rtaT`Q`_148iC3~b`A5jl z_5nD0G{VBzdpqQw<|?X+=vskgI*`VJ5os$gU8pc6TQ}ozIwQTWtrXSjlNMVB(c-E4 zjmJVyz)w^`OiHW(b5TJF=OX=l>c7OpuwP zNuWFMN%zyCqyM$5Cofl%$LK#-4%(Bc=++#VG8RRFsbEAV`Wr7+MeZD-!T~7H%aRP7 zDRp_B!QJz0o3{pT?IAM+Y%n2UbWipg?L68WxB5a7zz&EC!R$$)@=R`1e@{2?7Y*WO zDm_^-1XXl7r1c4Vy4=xLu3OU@a1Gq`0d2-|#P6<6-H{NN*;E&Y4CHc*=BtXo>K3BD zTDJ|fk;lhGi=c|>Ey-o!Xy%#2%!Iry4ug&X z{sThij;c{x)VA-SY;c9d!!yV6tUL6CDP^5p6%_}eSWb&Kua8J%i;tARQN>NKthJqHE63Q9 zWe?*)mGE`-u>v~ESRn*5!bKri_hzsFc)*oUL|#AKEswDsky?h0kmjrDqZ#V8#MF@r z5Ph9K;oDI}rsxN)1N>v_#x={1g?~(;4O?GA*xhAUNYkN%<9xdU+mNU$sj!Cw=8oa+ zaQyO1>e&v~A3XuWC|0^vfvnEe{D>dbja&id$L0OWNn#LDgND zI#NTAf@eu$JKtQ)a{N=So>8Z=QVtanhzYtGx|H@Ty75QMJ?}SK+9LT7F!raSzu!y= z+7uX~wMxoNiQ%7yPUL&At}{*KRtcOP?12=4ctppn;b+Tj#mlUPtzK%lKLQE8Ae%5l zAxL9B`8&!8a3bH9$vo`TOM-VLp4BU6pIIN)En!;yH)i+g0YVc#E;ueXLj8%3Dqpaq zype77eQ8kt21I`4!;$$dpB%%xQ*Cj0T@he1aVM`=yVAr1Y8r@V{Tg--XDk*)R9N9RO|BQhdd$XzRq)rk6qDTT<>~OLj23Y1D55v>r+)r8jQ{m*jU~HG zLf7Y2azOxP`Cl$stBzX?+P6~EGy47 za`c*}r86GB75$hu?{&T#N)M9xFxW7)G#7k2Jj%=%sm?~3LEI-Gy=iu0m&)X@_Ck)BOyJg>SU@7AA;*k4)eo>stDCp8BecGEpIgLWUOh-| z=P*2m#Qne?UINW$@?D75akFp>3%b^+UHkM-ugOBfP~Wj;wT~h(X^MVZ4cLd&8vQt% zm{pfe{-DkMlR5sysC1u2f46QJ9myS~@lRjUW7xLq%% zuZn(P!vSo0P10FEW9oADPamhj5@ou^c738n+2z{Q)w}!(ZMKNo2)9_;aRD_(-PmSJ zFe9q&Q34pQ&k2fHQ|fFU)YOc1aKlE6PcP%$7qVA04!fRaM^=j2YW;pQ-{N>A$*03o zR@6= z+zbxqrC#AjYgMX0F*VsjBaF z_vq39t7}bWop-;|hSx$qGc%RlfeN;;M1C94xrp5b8~kCWF8{ui(af++zq0#+-Tk5} zt1+=32uZwOxWM6|SXFy3$Na;ClroE~uQXO+p}7UXoU!8<0>=ezjsA1bd*eq^;tHAe zyAaY@;Na%D?YHF!J8pR2eG(XGfJ+j?QnroDD~Cx9{$};I_r^DG`C(?-u~UJa)y5NP ze`=AF8yvv%tREyF2_~wD<6ydabg7k~xV$cWQHF<6`sBLGk!Dw+K>4Kh`n#oz6L`Ia zHS4kF>-mGnmbLWdm~Tuy7qt5smg8){1?qSQx>Z>13)A2_u6KksHEUVS&vO$Pv!m!(r3~`05n!eB zrXaYoD}&MzG}{6m68|z&DVa4TNl_T6No3mK>7g^3Pxb5jltff_O~JZ+bh*8^I~qPr zYZs^hIf8E`W0CP^_v?mjjMHZ*c71;W0&1BE9^S_naC*Bx^Kyn{vC)vH zN4=HtqiS5Qvx|)Pnb%Hp3vLZ#Whp$a$^LPFjQ36vH0Av)rUSVj`(T}P4Qw5r=Nw&PJU!V!nvzbN&|rTfhkXAE0j~13 z84K_^cqG(ux?=N{iZ_#Tyvkg9o7?=%@Tf3+4a8d67LNJJCa3(^+i#khFMn&r&(-~i z(;OIp=RFtf2{@~EU%}hCK&M)}Dx2zm{otZWfO@OB`}SJ9LW%qZrM#4Akj>ERV@-zP z1A=akKO&k?szc-3UjD)lpLfDQmY=sQQ%m`au8xKEZ>JO zBJf))SCNq@AjfXs@vj&CQN{KwA~QS*dZpA+A}rhC8R(1{ZU(e3K=yaR;(u>O3;yeGN;By=YLG4zM%=xHzv3 z(cztU#qDo}ZMv7Oq#T5AHC%YsG%F+en4D7Ri^9$HmKY(G(qgV5_nFG}a@)TecOlGDpTIlg^ zF)No!*|!$=sthyTV}x^UoEP^Zws6rc60)A@Z@Dnx`Su*VQ(2#*FHa!0DrTsU0jqk* zr8cNXGnHsIuwigjR~>%#i-z}?<=pq%N~^)X%I;F6YyuZP-;zP$PFkt6nX&Q`jZ}8)J9L zH=0+`45$<`2ov$FcM#7P(3<4K05N_yROm1>b|0N5?UP_(e~w5|zMUj{%dVUzqK;Kt zp>!3X|C`u%hF`HxIo`A}Cr^9HlW#EeY3S~~Rj)%4=OuUXQ;_!i?hTE^21N8r64#=4 zsmZ-b4}x#sY$sL&fR>EBT>2d^%6QT2mmz^`Zr!Z?-K%)-G$bW%T*9xv9Vgl)!q;e8R-FlYTz1>OO2RjC0NL0o_yH*MJ6S#Z%4JN zz}Ezta(m9WyER0I>zG?^YXTMOQ8&!DGTkGAzD(vEe6Ws_@A}bSt^L{qBjvWJguPE& z=RmM<)Qr|L$Eu3Mtcg?8O9h;Q?dZ^g4=OCl+rM^PNKqAKxUHB$!nA;r7&i7^2nwsFlOjpeE|2&N_%mQwT@*fDav0|JqD$& z&#~Gjq9V6DvAU5HNq!$7)=X{m(3s1tjG7_6RBh%!Pzdcei73p_9vnF&#I^`;mOt>a z2HflRE5T0+9x;pi^!-mb%6c_TwW(JPxfvPVjHmz|by|zBTF)lVGZceZZa2ZI*%{em zh{Ok#uo#=bnKb3LI7+JTi>INbQR?tlKzs@u1V~Ytavu zwbZmUs&Rt2jhoQ4|9-Bpo$R%GFYZaHwvL}ocD&OpeNseWQFqKXiLxbRMLvH@9!hegJ`rz-mGqGzjN%ebGsqgqY1+JiRdL!S z=J#(EMu(h$YEOAPsqF}lovG4R=(vz0eyBdAKr*BQj&)*01)qCeQu4gG0~r~Q|HhQR z05KTC=Dc&#AhH*LSY;K@K=>y;o7#Cp!Xoi&ekcM$eV@-+`c)WmeD1~z!)=P!+R7Kn>8 zf+r5~;?^*7pYC)sJOl<3jNRFEMD}o0Jc<{kZ`GqhxJQBkh?aAZHg^qEC&VE@6qfug z51?NUlqR9S|EhjK&-X_>^Te&M-EkYP5BN?YUGy0O!T-xiOUq7=GaC5814LY;R#w4-{`okfBQ zc@$);<~7VfuhYhR8pqS+l%XwX=xy>!xVn);xLW&zdagT+{NI&Tlmo96xC`YOWBuT@ zy6|>jo*pPps5lavIMWkq#R$c7N(l?C8ru_O90uIARPl%wegwdS<%4wXGmCiZBp zn|N_Cl>C|zWFTsCGCu=;f?b?&69~5qZ2V~F_D4OfZHIo!XB=*@oL38m*td7g`IcsS zs@?_S;Y_qDL9>48V?kg?RtVvAF;Y$afwFzniq!qc*|?y52iws?$vJ77hF4_9bdLR) z2}cj$|v_(2+y+8QIHA_28bH0T%?vehaGYr+>rALb$ev79e z)rf|5i4$9V8qBGm_2{d$ZuMY{b#F~U7<_MTfxest2Q{?q-f{a<+tnE06NZvQ#3?#| z{(9Tq9nnVKx>0XbKIQhzeJ5Xd-qqj5Kis10@9viFpVOt0BR$g4X3?iri5gGi7V$kU z^(3p0K$F=a9-Ihsr?;GVlFeM)jFdGXNNWq^d%9k!4Wzyr#r8k@QHU z6XvEetUjF%$8DJKQ9)9eU+&m=Fpq1zDn*#&tY&LZdzRBrL$#SFhA+E?1l>e-( zUzHsPR;9caQsrWX?ROUPQMf>R&DwJ9&ImPgG+@!*58 z)6zSQ8v6-E>rok~ zQ0F1NOlUZnT&2RK!<30u4AN=njeBxlUhhJP?vB=hv)txe4Qod-gT7}E58tJy!{=kj zeX6L-6pW~yRn?Dph*uNiq66FNMGO=Jiaa96ct&WjED(I0BFor*9r$L;vKks^`rk6d z9GcXqETiwqHrNQ;dQctJ1jnYa>%WDgSiH!TRo3(1yss+Oy5#*POPz858af=>`!qP3 z!r$IEq)?ST{7-w2tp3dQT>IaW{8Qq?E!yevuS@4tyw9({DKirO+W81-aARmxYc5DQ zXPOdnXq7CeDf`AQ$zgyTW}*)w%sJkn{ob18ZQw-&?;Nt1B+LIlNQ+9 zNG$eaIuG7RA2ZaKsODX=qoT9(z!@=DTilJx?zi7}bWr)8%h!yP6hKcn>M}k;#vgh~ zQyB*jxjRDDu{sLSP48G@FLspJCEzho``<9bDP4dVTUUJA zAaWRBECY0-AIVRR`#%vbZ|Spk#=#&czR%WQ=Q6tWlkoo6|0j6iy!ua-@QM89 z^plZ^F5u_j@T5>%gkq;rUZ}}o7PFZ$E<77urlU?XZ2ZZG0|&umiIXfsDP6|C!8Nys zP0Cw**NGcjKBvq%trBW3zxY84Zh4w8IKa+Wv{n5mv*z!JR6?SVBqe*nvTi46*kYrw zJz#s#AW z5`pN`b~Fsnb0b5+*K&4rHSXDm#<8HFCvNIT^(R z61f$0QC_MLSci;}8)P6=$qd-k+nCUjc#q(!#q1=QG#Wl6>exko%jAz`BmQQ0;*m0)6sZDVc|@Z zTu?o2es!O}bJDf&7}=T?YPp3y<}ATeGp@nIj4%?8m{x5*LYYU`oElQ+k3Wq0to{)d z1o6+4Z(sBHD;Og}u+L_xe}C&++?K93-O|cIziN%@U~q-+;JX^*_%7)DCQ7oXSeso4 z2vS|Q>y26D#fOHrUUH4#~NHn7-TiqQ=?XyparKEo8EmlypYmo%#Y6 zK+oO^PErC645raYhhMKOP&4Ma>HC#7>_9gTLTag!k`o>X#M%<9jf`b}{RSJ5gE^tH z)T$gQSYdQp(`Y1*kyd78b8f0_;P=ZWf=M?#-{_;to?S+YdOz@MgcsFN_`E%==&T<5 z{ny?V93;$h9J$)GKSFqKHi}=;sc2Q>irG1L%?#y!5P;8O)G2FNuIE*9ZHsJBy2Cm{ z-l4#IvaP%y1&P2*h$$(*{QXVBrVpKPkUD-C%ZSkht*k7Ut{KTz%Lr~po;0&3ES{+s zD36$;#lm3(0`5;ek}H+74bAxHl%gyNvJoUY^?;nJ*EFexJ5wxcsehVCTi`>GD64g=zQg3 zG9j(?{G?7|wz3>)rbJjgE3FEtK|tEc84`itYXSeR30KC*z|emdmzqPqH_lwBEmqlJ zfRcBIeJ=L|XXqEASDc?NGczh0l7e_5Prz!c%!bx)U1p%PMu}cC4ZmM2|Tqm}w!&lYM0AO#da%H09!Ejl$S(Deio;W|3U&3dhlF(QY zO{e(>#=jjH7yDWtQS^^zFyVC*J}{|+;?k7XC>cpDm*)g0v+KOnR9IIPm7>%(8&4JX zt#-ls%4h4==FhBBV6W1+PuqBq?f;sP_+1vh)6n8I$BX%>pm#Jvli6T@qKA&8zp>lL zS;nFV$TpCtQwgn-4FeRmpVh{wK7bt9xf*5g`6({*U@rT&xYBq3T$mO9yFjy$~(UiUyTr8 zAc*EvYZCkLmzNF8JD4WLFB)a`!L8A2VP{9`P9mQu{-4XvgJ-$9ju*jJtc8-yBHD8F zv3j0JKbIlGe#d@qIeBkgP>^VXTH}Kg9S76Wr|>w2$#7E*wCi7OdFiJS`MPaIC&UHD zi@l5c_3GkFxb1MR6BBV!?f+ch>$gSZ z)@(VwMzP9Rq7xTI-7%ldy5Qw_`x`yK!AoX>P6DG3F7yDBcg&~c$poN&=s_l z{jTeFz*T%Q8v6jz2sPfWx|$YQGkHYe!LF9ntO6YOwY&zBuK8VY1txiKIztE6%0KW^ zN(f(6LNKl>uO+Olybxd`{KvEh%axd6b)jeu;Qe3{>8cu>sS9f!{NNQY6yh~_(UwiqAMkyZKfLwxyX~|)Ioj(8v)%rv!p^naaC22{ z_Ve7jG5kohnF>caeZHL@vBUh&TX}@gLh4A~@2gd}}5ew8Z6_ zFQJMkDA#lu`V;YKzJ3qu{(3(3TE(!5gkbJ_@ri{mI5QjU3ip{ujd$XDh*ZZdhnWpp zxfJrbvj0WmNc}zY!;F|6m-m#y6mv22d|B*=-8UlDOC!2vyMr*a_zKzaT9dN3#)SX!LvBceyjbDnpH0E)T_!Q z4H^29E-#vn#kn;su_U7tw~Mx6_zL*|7zk*0gcXjciL<u=~&u6^TydF7?tvnNfkUJcfamdpEW95cfE#NCZ%4i7gV1lvLc`YHjwVtCKG6 zHWzB=NgKr9QV@1y$-{~)wAGxR3+kMt8MiaiCGq+E4gEp|7m&P55A;bfePsV5Nj+!v zdBx+$NsS+*s=C#I4D)ys(TtIdto0OjFDpzra!oz*pJS=glKIdyyg$^A`7`&#IvG0-CVdr;Z(dqV;y8<}~P-tODw6`FyQ(bA3}OM;X6A`vO| znfd9u)+|ZGJI;bci|>mZL754*hZS0(NdsM%lMZP|Al51!p94$j5`WhA^4O6X)c|3c&Ktcz^qq*R{ePwVY!z#GpP% zOIqGdobEM+nu6ITh~+c7<7#bNu}mo$S-nZQLA%NsVqL>e&aCdnmQ%~7k*Nh`q%JVN zzoq!muGJ5WWa3VH?yNVsz=XpCXQg`cq)9@11%@qH(b*c=!S0J*NpY{dOTh=uf_Z)g zeKaXGNK1~re0UBZKvmvDyQN!svtsJ5;`bCZXH0TQzYVDK8;nYp==wCTag93P$tf~w zYREqX)?_VTLm{AMj*>!8uJMh;gQnG^T3Ur$vV(U~FAyg7dGc`mbd*hRTA&2Bl{ zy@B5bS**()Cfi65IZH?6{@NFTe1%DU*xDbFJfAfQkbnla)`8F%OX&+!Z1wyjLX}GT z^-(q?wF_V{!8A&i{iF2HDgZ}@vA0EQmz~!ua^2EKD6j*!D=C5X?*s2iB1F{iuG}NH zR$$k&KC)^c=^TZ7zL0m@xPS8)E{{Ojvr{2Dt{VfohhghszdA_=9!NmKLduyZkikla zY{HMc>1kP9)D;|izQPAeWXaRYt=HJfFJOJrG{mwb&{n=)1_Ed4vJ;*hy5<+UDL&7x zptsb5Z05E%7WI_J8Fw8V!00krIzqjY{T#aA{fJ7kBmn9r{rU3~+?79&;;t8pGNinY zbHDxVmV-`2vGkYKqRf2N_gTZXw4ys-Ss<&s^JaG9(RO-$Y(BY2PnSGyJ|8S9EhYDN(Z{ z=tL=rX{0Z2;B{a^SbDqQLhO7GD+oN2iHX0M|M9m5gw646-k5j|6t=%@VTNQkSG#1w z(F5G~Oj!#zB4g#X_P|M3B*bU;=r(0g)Xk#cc$H{@_c#P-H(46N!{BH53s_MGS8X#93&9S>AwXU z2`r0uu7yZM1lExQvKM9R2ModLYj*6vZ;!)0Q~&w9Cb`4zUJbj={>cy!o05{gLc$G$HzP__>z+*~FMa2N{JM$$<;!4E!#hln zCr}4em&-}C>matO7M%Am5|?^BzE7Sx4AwngKU)*d*u5iw%73q%<*?IPs{T-)gwefrGl^`U*0+?6^J-_;nzK&w|Y^@IA z)gNuVexqg49U}@|tIy1q$jg2`)RYs1qm z*)YhcImo-qmgl|SFV)Msk=q)vly7mdjX%i6P@lTk}j@(unZahu%poj>OjB4;7U4wI`<`$GP0C@F1gm%A;WSf7t`5``>Yp{Q z_L@2Vg|p;VS>aX zx{c1L&pgplmi&T2Qd5G-nOEnDp<#AZq{1c5lk2gFaJIeLwCd)tJ2^DJd@1Pq{cazN zAx^TEmv^aOX_Q1mO4N)SP;eFo&sJZ?4`NXz9|cl#K7}+$Mk!CV0ukp70ux=8|5DZH zI?a<4tEzPspjd&iYJ@Bvg&mMJtSv-Uws$Glyj-e%?IoNh-8~!bg zt?VWyM|xF$)27xIZQcxs5o=3Rs^VtypxaXb*9Hbdh5Gbc(-vL=@(n>;j_Ez7WR1D} z5TBTI_~a*awD+DiR3e37Is)C4ABD;SZRTrkI;7?eQFL{%RcvpWMtOXU*ysvs>GkGL z=iZ^Z7^HN~?72r(8@OVUcY^Q;=R)x9$xVWQ)NZ;@_iu8oqXG!0jr{XpPg-pK{!CD? z;-!MFwAZ5@fO(!`<==9o683XVcVAxW!-SJ(j=I6=prN|w5uJTvrs@r_g@qy>Zrwh=HgnJwfebs>Hi8zR}Tyw!!kyLhVp-O2-4hcJrb zMQ1YMmjz+HT|dftn;)OU!)@PG09sB)M(+i`zn=h1tIC=jFedo&YSzquIpZiA@ zB5HTX*-vRjU)WorX&AIbU}QYOnlOyD27)6}TeZ4UTXbfcHapenN%JFx?Z`kkHFfhy zV^LsuPWlVyI5O!~8O7=~&6gzB5{m9fM7@)&2|Q{5Mj)>YE4^p8>S4jgVl_6#`5&p6 z7ZQz8_w<&)Q~&!DnfbpPPi*7U^~4DNf4_|iIeA1Nau-RqrR-aUfJ;_NNwPxRDDeLP DryL@( literal 79108 zcmb5VbyOV96E2JfcL)S`2p-&Zad&rjcZZPR1PJa9i@Uo9ch}&HyTjeQzkAR5{{7C` zJv%$o(>>KwUG+Rw)e%Yxk|>Dyh!7AEDAG~@6$l7uOb7_b75Go!Gnj5DN#Gv@Cn;@L z2nf{ve_zO9$5J!!$uDl=T5hV27H*!#F6I!Po}NtB4z{kQ#!lu;jxLrN=lu8(5Tp>& z01-8>%+n2TFSUiIo~sMr3%S0kZK#$4N-T8ok?#zYR(h);zU8z`jXLsBHoEruZRCwj zI)UzDQq(Q#Co;P9xr>#`@X|pDEkWoZ?d0c3x>6SBFE6Il#ug*2E2%J3hu&Vd*}mtU zj-;-K*{)OGaiw%WjPd^WIWwl>h3IEN{_o>MGWYBM9wHZP$cgp+-#^hFY$yo-uhHrR zJLUhHBllZz{r|o}Gaveq2`e5eC=WyYLuG*|B62v~&|E=b>ohX*EF!i)JT5%?Q)~n+ zP-(#v3p>`hVRYRvI$I8K53LYM3shgg0pB3Sa3eup8vw^ocx9<%(*NymlG&1>hJS)* zHDK6y;EX+A$hI{CWtc_hpoySL!43JxOeA~$MM(-JCm}`rKe2d)g;A8XL7~ErSeU`fn|!amcowJH!eG_rDH*@)S)^y5*XAygwG%)sI1TbuZ_haTW zSn_BO;8wO9`L7C=m0NuLA2WlZMA)3Fn1nx#@gy?-jk7A7HN=Ghr>-bNk~pn}+`nwv z@vmF)RQMpE_pUXbM4Tl)=8qYN>(HSORgV4))G%8$x9(mw$vFbFd>9M4kGrAsGFCubn>+I{2WV z^dB|oZ+bvc=Y<;M31t0~FB5;RYbZIxK&+pIw>+&m%V@R-EI|Li9tX+-Y-bTpGvQYT z{*y|3qT>v?mS|2HELEI2_<1py%LgMCikGHYqET&+?P(yW*)qZXrvB@<k=>VMD@ zDE^zAyrPEc+W*~#GrTs5o;cmh5=}gkbOh05drg8jQ_|I&b-?X&dASRRrixu60?EBy zNM>n~Qdo&`yAhe69*0Q?oEe$@W2MJ|c75m-ciRU#@X$EeLzb>PQG{4N#edv$Ha8DG z0I4h95(f`@VZ3>~{skREXb|#|4WoWKyV_rn{fl7M|NHlzBj<&?jz5c1y#AF<${Uk? zYC;14b=oSCzI)d>yL;C%Nr1-yyxr1>SU}6gH@+_9pb*~Cp8l6T^Q2;bGgv4R%0Q3H zc5}9jk7kzWb8MfrV}~1in(5T(oyBC27gsg~F?LCDO-qKVcS+eh;7x}n{i$QwyQQDE z`$FU^LVv|4OWx0lMa=irb{~o}ody#m&7NY@?`daN+8qzw8WAI-r&p5P4z{^E*Ul0v z*Yf)42cDvf0$<^zdpKAdFN37FI$Cj@5fp@_iT^5$Wgn&y4tQ@h7M-6(mJ2RtA8EyG zNUH7nin8)jyH1w+4-_TB3ZRZaOJHtg@Rq_9Xwy9v+_|rjZvP!D96H4lM%`jIwlA=1y^Af&SGRAzg%5X6&o9$qP|Oi zDJgi);V$QrBDt3h?ER3hKOCJ%jXq-QN%2K{JqS9$e_in26vN(xgsF(%q#E10KlJ|i zq=%yBoATI6^lqF^K~7Iy{Gsx_8vOJ60K-@k1fj2B87_CG0^`{$C^3`F)NK8+7 zxPUpRZql|)L^zN)Vaa1pTGKy+a-hlvK0LdaSz!06)er9& z3_*eIj7A0G_Qn1&3o`i2Gv%5~6wg|v=?2q!Sw*d_(7|(%A)s}%plRWw146YmU>psy zWEr%QB;NwttxG`HSlwwm3@Y}%BApMJ608fO|D%3AT!dDvQnXX*+_q0C+ftmQhO>-D z)0FE}iu59Na0y00#4i!O7F&dUvSgT;AmXm=@N?aJr?h>EQ#yvL z$X4o_MPsS*+m{G*7@85Gx3Xlq?*zq7Y>Pm#wL;?=GQ3@*%jW(EYz z>%1*`jcGDyWv8`%`2Hk*cDJpTHJF^bU?Z)$3o-+B@!I@3G2{rWO2WkuK;GpD%zlsw zuARtkU>1Uoh2p8M%xP`T;AXzV8;;-7x)ke)b>(*C2f44^jedE44WJ|`)(bAY%kzIkSfbhc7+N`B*9;?MMmOyj4uzbQQC3YI|fN@Z+@_iVFN7bulz+ZT1tFe}U4Zfi|!%PZ#1c zMa);EaB1|sid@Smca>WKR#P;?@oW`yOR*k3FSk`I61I2ioM#BrcgU7sL~U&0u=;1$ zeqKk_2srs*uS3x)v6|qrZ6>ZW7ux{PSI@k?^Br9Z>Z+N+jLrnLAJaCry)i}_ zGi0y_G{oR!+~1js{Dgk%?_Sdlevje9(K%aII(-S-cJKLGeGs?CcU`Y5@Q05W58BVx zJ~2>|n9Jd6bZRxS*|`0M(kZ)PqnAF2hjF1?aMAa7r3UP@;rH(Cgf({Ip9Gi?Sjmhp zCZYuH&kKQUsAIrok!yzS9~Sh3(fd`CHx^JY<#n`8pt{$JP`eh$J*tZYoH!4L>ulze z?EarJ)Bac^gAIZv`H)^nTJ3LhK5s6A2PB_PaBA~DMEng1SOa;wkItzMu%vVKGZB5{~HpH%OUV`{&1-rWC`(v z7QO7N+v%HM>5?9KTJQJV*5Z>NQ-zfcpSINJpl-z}p7`SjS0neiw)lH2$UYYredd=| z65l(-z$hG)jqDt*rkBGY2_$}qbbTE?-D=0zpNAZoOrxu}E+1QKz^%_8h0!E=>5#~9 zJUE3qW=nbUcgZgrM!BJr;cv6&6|dS)Rc(0$ftCo}MMb_Tm%Oop#u4w#)^j5C_=mjV zBX>OmLpCH#&wq{8-{8J)QU)XrcuAfnDpT{J^agsEum6}KOwma+6(r?=^0XchYuj~- zHNV1RA=Pu6j$B8OwD`?zIJeG2!=T-?nUi9Gkbs4}Y6X&&jJ?ix(SsP+W!rS}Ur zA++jxs1=nw0Zw^}KMBZ7By16p*#x?_t)XJ1s`O* z#qo1R*aMi5^Vxkijb=W2z>4DL6TTPk>#$T1(VE`ySk>++0sNgo6a*p(0wJEhxUN3z z+KRD&6&0-GCxy~+o`asD3K{|)BS0qBK_LU)b2=%unE zq#y+@%7K{~x=g9k5^G2Gx6hLIkb({s^?#SI5@Ak6)h<$%x~>~14cJuv!Hz#lCtEwM zXtf}vcd}%6P00jIepmA|)Lhork|%GHe^A@nFe4q4qltGIZcs3J_0?8DwrBbpD=|m6vKM8JUT+!?PM}#Zapz9}$#GKH>x_oXrPXaV+kn&AJ;0 zOhvAnAFl<=h!X}624$J--}*){HfUILY3`43lI9yK=s>u^9&oq*|gOv4WpL#^vj&JHd4KUpKK@*sH0z{+W~`g z)V~L*XSv9IR9#g1PwR3zRGW28u8Q7zFwVRcNe^cZ=;)`jAtR{QLCy%~5bHm4*jwuLB%9SeD;AwzR5)uR<`+bz)PM<&$ z7u2IG8$Y?uk0U5Rtgw1{r$O^=;{l>wMZtv}y!`wo&E;SZV=tupRfo9sp8@Y|^X(+f8^L_S)av_jd<^Aw3+?nbLpg<`!NL=d*C3$7Q=SjP z#b2Bc%5^LFiSlK=SkP9=!GRzb*n8fLJw5qHEznfrJP>(Kc#{TXsswMmgNi4VXZaWdlwlKEI3QIcZKS;Oqp7~6w%Ba zvjWrEqoC7-23w5IqBQ-FCU$Or>zyBpo{j*YFj^YVXNDRm7L*8<;0sB+PZ&?%_$qLv z5G59!mnYTE#;^dK)S^+69BaQIlx_t@Ca0$}!@JcP8Z?B8zs=Z){u~Da z9pAiqotzvW6wrpS1`=MZ+VG7AW}#5tri9tV7IdK4_< z4SYsJ3?W8uug>e$Ut2dj-W?qUy?TFH$UXj}(oIt1ZPb5YSnl*aG<$U!vW{bVn8t~? z6pa=%nN!YZzLDti%U+kcpUIU+NYl(meWz`Mw}P$IrG>VuiaTY@L!SC~7rwka~3q8uha_jJb@n9T*^XedhG zKzHiil|l64T7%B9jV@u_(m!TiO4+_pR!IN~N{0aS2t*JJQE-Gmk#Hshc!WM;A@g2TNHZfl z_udm0)ahmwJDU9Gq z#`Io6pVZt32N1rPoWv<{`iyCIw4W|Je{EV`g8E3SDfGbz3RD?tg|fUTCMjRr`|X8_ zJw``Sj6rtUEj;c2e(qCld+H6IPa+q;vN@P1Te&M55Vxh8eD#|`H`C@EpwQpm=ATMz zTsV5Uj;Fddp`71v=P{pY_@czo(<7-VUJDkER@;S?0KeCz!z`P>x}?Lfcmh90;MXF$F=k53zU5^~)1yROMQL{F z34D9g#~DJgT6V$cBMlSZ9DI+{<)xnO>C$a-K)Q`MW=Z^gdH4iNY6EM>hxb}2AS8(! z5uu0%(Tc8x_*?R<_s)YvGfSWO{KS6vy+DY|AI^I-SL;^n34hN|jmS6Q!GVeDDt2LdG=&5c6JSrQ-9dr%S$Ri{nZ=py{@C-p21^_F+TGl4CKyyve+V&OOs zYlR}Zq0WpsU+Ur@^ zJHF!cP6Ht6O*XYBvLn?^ss+a4zzuyc#Z4%)!o*kDm!V;TSZwk~ zDwD!as^pclDlwOXkD!6q?rV)=A<3!Z$p|L+n9g`F)64H~)5Aes7=wnSgoc9t`e^n5 zh!O7XzpgWkR?nt9Z^^j%5hbyC?n%w0k{ONG!(FJ(9p7=c<-~r_{oy=r z@rAsR$VXWzDoYCo+_BOi~oEpGCSg3Z1n^jK-X|5MR0Dzui$wznvKJJhv`2 zc^nY#aA9><-1}@Czb6mp_@podVtCaSy$LK_cZ{ncRY5*^KL5p0ga*(3TM%N~3K$&+ zX%#3baSwryw)IsuK z5lGl}?9*pUyCDl~j7lF%Qn*kZ3C)g`k3odQKEl8lcL*`}TShe+S}`@tT_2i1nzb7= zw%4nc_jl$>Hqd?2X!VXc^1*_#|ELst-vzwm1UeDg0pngFd`eb z)@-GCHM?tMR6d>nsP{`hJw2_|$R}fx^M7`st5E`=d3k6qS3Y@Ta5y!=T`5LW^Msem zeDr)zXn~Om&0b2?9ZstNcdug36pprFC~S(aZ_PHi<XCx! zkGDMAP#L{mOK}VwfHlM0pdjt#fLe-@GWX2v2Cv2}p{W;udv>!}S^h?!FVqrJEdrGe z8F{Pbi~#6*zI7sTvE7Y1YX*9!0x*5?mdKSca$}e7I@9khZ1iqORJY9nFJn^VfPYnH&lTlUy!`n^YN$_l#zQscGN7ubV0+6QuQwH>|By{f4e5vL_P zWr4nf{InQPvU}vmv}jE9IuG67VX@&LB?r>0Kn%u~k`I5t=Ki9t&z-S|f)kUWiVKrG zfs!08Y6Od%6`171v%~ukc{`gXjR9jagaH!hG&6u{qR_`4*l|A_7v3%{I4hKo32rpRXnuG1$PE-A42IaaIGL|mY|Q&wzeg@!&O~S)U%tc6 zqvE7gRtkHO=S#u7+4n)Qu3~G7iK?Fjn~6r%=`G4E9PjX!-f!=~JCA%Na>sV^11NWv z-mcfKeiJoCETFm2Nml(;>J=UPQWr3S8e9gYdRCoTY^?d`oG9k_brbJH=5BQZLsn>m z6Q+gE3UymxDfv1GUWb!~jAMv|-aPisCkb$V6X z5tc$%fbxfQvQW-$$VH652?pPumYgdL2e3El{f2%R)Pw4Qt`yVk*J9hh*W2e%rHrH` z=vUbq%CH|XE=@6>8i5k#LFn-T>{TG7&U~6$b)XQni37~hYh?Z%2^B*&D*8fq*~nAO z8sedkELz8?;iYn14yDB@wt=!wfL-gZ)=Dn?dVRaemu9zp-yYwqc@#99WvX*6=q@{M z46?tnqMH{te=Y+(*@pn7DUfJw?l(EXZ(>vl3A?g?3VsbDpp3Juq8qT+uMJK7B$-_s zxw_cx$X4q>Rd@(>&2pJ>IRwp zjgV=$5*IR1+DJVdmZvj_fLPK`7ipsj zSxz4LiR#<7FF3b#q!VcYxn`$%*}Lo?a*Ln|dWw;Vc&gYLr_a#-{Ltqh5Zv9f6?Cm+ zmKK_sRO*FXps7sn##|qYHh7L%0lD*WT$}ja7jtcKF5;XrAi!G0zFO9pe$+pH&E~vB zKw}*x;!>%|w?6}yOSP!>@FjRtu);co`Oa!NZ5HFP6qK0m@JDTC1KXdn*vKVB^jlkM zIR9zxA)#s$=wz8Fl8|5%G-F5_5?%^0&%}%mq^8SOsLec@N>80iHy-F$InX;>!j&~D>xH=eg!`>u?O83Rs<$#J)GQ~ z3U)1PqMAm_{~qL%zodmZhMJiQUUtciV#x8F1qmse3RZ!=8d*6M&Bdwm}zrMqoxls_| zIS3a*p1@3^mDruu0xBcEiOXVQO5rN0DM!du(?VMM<^TN_y*a9)0p$*r1J55Z60!57 zEkWN$#1_AGVT)_1_oAKu_n09pg}hmOeqGTwH@1Re2YQm6z2kV%X~lczLl9lnlb3~9 z3vVhy+>G}U$q3e|Bm7le{UUr>2%_{8)A#d?0oo(w_?wEe+ztAl92~RL{1L@mQoTQV zz6r-o(%3iTG_MB^{bTC)AE#`Vx1fc)h*%my)*sL^SgC4fD)gb@_nDO=Eg^uG@a6!sIAIM9-?c3D1KnMek1qVAocn|j)zva`4DtI4No2E?WYI)W>VD*nEW}0IoK}`vz>&2Pk}w$LJC7nTUA-?2S0BD4)f&3e3u?K9HFhGi(bKSt=5v!EB_EqrBla zKcJvz6m#0>njNko^h?!oR_vpM=1+fK+RWE;Bw7fXxw;$mm)4&}%8?x1L!yqFMK@*>`6>N# zlYCY#izRl8HM>x|UXDsSO=_#NEtpfHaO;PtIUKrgpEM`$PBzcg%Vc~LoCAq>6o~&i zJRj%=h}(B!9VK7D^SzcR2Zh7NwMCY9$2>SnvcCn`eH0oPW;{7~ZU{vSjHBHXm~0$< znQ^NYtOYebJ3<}Yn+#a)DgJD~pTJwbmAg71BKg#%-(1OuIS9WUyX*nKq1wK_to^g~ z1y$bWIrj47y7IYa5ml(mOU>8AQr&Sro(HoObVzXe8G3#+~O%p0A;<34sW?zNaJW zz25|T-o!;L1{a@nep422iSD`U#8K0Qh<$i#iB48qY=i!wRrTa+LK4*B_Ya}$qzW`h z6T|w>FIOK}fIWa{zmp(j0J2*6h>@sRrUKP`YAT8x#Iqa!?M3o(ih_xtjD%tXO-F^t z_|FLzcFIl4+XWIB;@E=sms2^}1L}D#yot$0xKE$Ogrri;}o8+5}BDn#3GLPy}eRSdNR@)C%Zhbo(%{gxUo=#g!9=f?8F(i79aG#DC8Mt~4C%F2nhx_uAijRR9Pn$E}VblEXShZGjZKEl#D4>dF{_D&Uulk#$rXkH(3M;~J)ih&_ zMLt!4LC2RLwZ7pv?n3HCU_04aBzY=Mdt+IkC{K}pX#D`~;-%cu4Gjk=tA01giE6C? z!Hr;5SyB`}Jsm39=e#L>i#V}C{wn*d0Z?`sdN2Fw{@~sYez#v*?aMv>u(0$WvCp6= zXXL53`ZBtMD|W1wkXzaRmEQpC$%+fX113@)OXbr@Rm-a}NR0h;kP4S73a=n%Z}Otc zljwOOYAyZIQ3~YKX<1YfjhfOYu4)%ocT`@`4H`{1(CWIgAXliv!r^RFuD}Fx~RA z_0h-Q8sKcE2qK8SZ;JZC^TU{x*)+?EYAh(Kxt9XZXP?y;9M#m-?n2L7ID^P>l-E9c zGcKGRyU0ioAs}yP=QrXA>A#tSfqk@8vV{L4!1uyZ9$h3N9c>2&dJGdM&KPftF$wm> zHfV|>l3)9QP_UusSs;rd6TP((bqot9YNoD*02Anw2|O3XWtS%#e>X zlC@;>o75FgDsL#~5#?gP`g|n|#*<*{GxWlLUpdBOkvXR}Z;h89 zT(?F36^OWxAx;kYV2yxOid8(WApJR?*J7Uf5zM8`8cKDEW5xGOFj`}khKjD6Ko9IV zu(n(1UhU4h#*g3MRB5PLi(^&xLPqSBy#txK*>(fp8n2%?&aarM$#_AX}NxXR#OEQo)*dPk@;Hu9a#WexMI zj2Z(`5}1Suur^X(s=MSoTVj3wWy~%Hc*jxCePs{Sct6*PZ!#2T{eZ;m3boInr(`0;9BgwL5I}H&t0Ipl@65 zQ7E9Qk={o`A>y!SKzu&Hgl9>@p?n0(T*$j5gSwPJVwlnq|D8OqRWk;)ilFIAG9t2s z0_4#W7y@AODrv+CBLHIJ#)IFT_zur6B#KCe#B%hwz?P2jy_W77bnY{b+UuFJibKo+?sW;y(yaWzIGR1wRO;&f}Zc8YZKs zzs-IfN$|M@S~#yhW~wNbALG)D6L~vnT$SE-%u=9QI|QElTSL=~%ktE<`U@mA&(}LT zoA zSxIb*53hPdP3s8D7*vXy*cJX1$~%5bIEwn)&J5)r`*B%yn7MACe6bMDRytlKH6evf zi4`ivtftmhQ%gmE51|C9u4|G+Av=da*KQ=llDr=Pzxl(7)Sue2hpT zMoZ87xeTCwQ?u7_`(>HwqeNTLDG7>k>oXiq#rDVL?Ke{DF+FSoCuf6ppE<#q={R89 z?O7@8X$Z=5M@&-EzHEC9&`G~U^Y3I%(#7gGyIl^%U9~4qb>#`uiG?vbg+G{MZ93m2 z!71o>tc$0TNnAc@RYT1+;$;1VWKHb`2Wv@F8~`{+p{mRz&Mj(`(T6CTDa6QF>R(>8 z>ms9Ct(@fMnc%wrH_BG%xXf_8(n0+mvd^zphKSRQ9II9|@y+xV=SI(5A|EMPPhf8mVP?2lE7Er3`O!9etmj z?w?Zvo;#UAl^3+nEHV+~iVJs14`9>f=6vd0q87>kLp(Xc#r1r{8qRI-seSCb z$EEHX6W6ms zK6{!G3(l4^)mEqkulqompS2Dp%Tws%>qKZM-5<+!Xm`z*^_ap#!#t(*(f72*>X^4a z6rtFir^*?oYpTCP`Pt%Q40*2K#S=i`T0E&J+LXB{vY30w)Iqz3dl8V;b|W@$3Z|C3CDzQ$q{XVZ_xz= zLg_$Uv-fS%`50b#BjvMwP_DfR!y!__W^VuTs><8JE=25a<9+8jWKJjN3Ssg0(L7O2g1 zbUMPS{^S}v-M*5P&yT6Cq-3!p0(T}G`>ROER0*X#ry26qch!*cq0l&we2e9c((=`F-)xDY0J zAF}KlhOb^H{?i=7AHty$j3dqeY#xsI$+_>r=&DAdhS|%k!#LC3qppvXmX6ZP=TSKt zBcjkzTgEJ!lkDH2fHZ^-HRYNvF>RIP&x;xw*cE{QOW>SNr=8)dKab$l-$jtx=lQDW zb&U!oUa49SY3Qf-l(H~H1DwuDhBj+Fh(%;0*|}mD+Tfl7)kE17cL?{SiiCNRub2W~ z^;*22Zqu7}xd)EO#t9?1?ar1Sn%|n1vZ80XgQtrwy%dg1U$a>+j6i<1jfNm0n2A^h z&ApNXoIo`4>)HLdOqDOXcCFA{JSbJ7Muxr@+YjL*ZQpjm6TFr!i_O4oPM|ca(V1Y4 zG{iH)pX)VLP_ntE(&<|w$Ymzv``X`w7Nn#8f*rq|em{U+4eWDQG=EVc`H@wo1{;v0 z{&!`1ktJ{A>sZRYYj8RF7)+-u<4SBORgJ};+okSE-r@n*DysEDmTs_C6%XqrLz)88 zAlOl)$CwoO%ynl*3p;s6{tR$5zwyE%rC2}8W{yB!y!)XS_iH_DtKJOp zm1mkd=nqc(&+u#`{e{C_Xa^ZJIf++rY@WNrW5-sPXrIBz7H}m9oJ86=l|o1oqTjmP z(JXs_7K&csXa+iMs78eUu?ps$P!MGFxg1wSVBRb(<#I8+Y6ycJocsKFJc~C_m_H@e z^_;CN&-8O9IP#g>FHEW*zf-xmyGH5+b1CY=^7Z6K@wro;%R_Y8!It;Z0uUeW6bLS5 zRPfZV7jThLL4xgMEcRO1yz|N8&cm+3Z`AQj9V89wSNF)eAeA`^=A#Q^Fgk9`e|7&%a-JaWza+D3MGt#f^D^Lz2p95D(eo%)uI4wIQvN@`uj*8|w z0q}n>$`Ms_98t2<#vEn+g=b2FzZUpX6LX{9%3`sE zgu;4>i-0g{73McRON_G#hfc#joL5HZhbVYj*gHY;WugvhCC_vZ*ks#WxgAm8jj&p^ z`+&44|9*_M^>IDj-}WGH@kDjPNq_%E*etFKg+Fs&_Ql)Npq(Q`Dt79|7UwJV^X8K= z3m*Sl8ojM&sno7tMCC+gzIdbqBzoS8^uPw=M=*xJnj7b(LTF>r@GTTmPOXosx$(rW z(LVP#f&oDYTCAO!Ow6gY8236^38IowSX$5f-&_L`g$kEnw`oPHlrk9r*Vs=;Qo`Tq zg_;u`!z+Z2Wa*`>AlY!n7K%;;lQLg!@%WZ&pjsfLG~KLphX>NV`{k(kf6uC%&AR_O z&u()XXlk2~#mPb~G?lK_p{Fx|Tygn|E?}(PN;jAC+>`obnt+L=-g49DoSmW+@}Yk)O_3MEc!qAy7NkRYI<%Q^UzmM{s2=9HXn|8AhcV`ZiigeA1lvi` zKEPJoyL#;!*eki_p~?Txmyg>>b!w8aGLj*JqZRVnf&~ZA5WP|C@OktA)#qR0*XS6Ih@Ox?`;lH4L(S|7~Iy?+9J zie!U(98pNnhWJOfDVPq$kv=VqB^P)cUPvpFI-%Rdq`fcyiMJjd11qBu1$jMAYw>M| zxaVhnm(fFr6n^*BgTFU9#N{Ko$7qV)jpcu7vzw0iaTsBxKNDl@L!Jo=)aMP;}&Aq zw0-!cZDNl4@TxJL1}G`>RBA2Ex7v^eI~@`=Zj{^J(U7~^6BG774Dtdiq82+fZ9uEH z!P+S!ZNVvq;0ug-S>BAcKrm$AtKL||7MCoq=~sD)8O?W}YQA6Ra-KXA%PKUAl<6m^ zJ?_*^PzSZ@ZJS`6Sc3CD8~Z4BzH(X2_i%Rs z=<7OJRc0WA7ReIK?c_-IA&)YlU(e3BcEQH^ClPVuiAe*e+AVP+bW|KbewUFNT(FU~ zYQNQ?@km}C-L--4mD$rQQ_ZyYx4{=^?&MDX;X`Yi3cBY2^PT6{#mSU^$!EJ4-ry2C zE_*LtTP;v|&nA2z)Xs zfx4EfvWD~OjQ`8Imf?M(XSz=wQPleMhxP>&@++$o4vS8F{TQ=$^OqhESXpcyiQwSi z-mT8&^U**{02evL(URx1z#52i3g#ZAG#iS4y2bIipJsUBh3~K<%oea>3wGJW^E#b> zZ$7`O$J+{bITJp`Q%zbod?P95*olqi~-Q1bYi#bojQe<7{&#@YA0OzDL#hmwgocf}?mq(t||qZBl^d zUH>)0B>#n3deJYeUxX8hy_8!F%xIYR97l6z*F#&I=T8(|U~{NPjPOree)b|^=Mq+e zGLhO*_oJ8oQG%wVXA)6I+U8&fjOA{J#&E$J^xgMwwP+U@VUv71fbRYnH z{FzA~b++dXVUFVO7_xo47I(m9|BY`NZ4^L6o6&`MJyIK~)F8N|QHGqu3+;!(hG(TY z<+K}#B8wCs;=vv77;6B0hdm$bSI+LIX5X~{QEe5}ak{I4SE5lbh_KrDoJyrp%_l*K zu=@?%@MJ_w$jbvXo&kDs>@fSh{3&$FDY8j8Er(;1oi&e zmf){1ctgb@*wk#gThDexxJr;9KKpKWtB0MuBmCOEdDV(1397JrJN0FWvT!~B*n8gv zsUf@tL;yBk!0xxjd9oplD5D!+#CrUZ#xo zF|w&9GkPRZd9A_P$j@XI=ZmyXGztzQ&LC*h$X9tI z;O^1e=e;wZFfyI z*v`tL+jyR?d270BS-7m$3Zu8Vt8Ag_$D>{wo-vGPxy-|~>9odg|EX~InR_=md-I-C zn4@rtwu#}~D))(P!0X+*SeKLXtqVy4!Rpo|=1(@q9Nv%XWtGv}L;IyKkrce=toT(1aqokAia2eL z@{d31n-U!u0j&8!pjJB(HIBN2v&-mC!TT^Y42-KaCx8ohNJ$~jmEXMEQ(xuzbbnox zU!D}DbUzV)R~?k7`E3TnVCAuY=LKd>uU{Tr)yAo^i{K?Su@?zv*1t7Tut6txG3MoE z5Q}gHkGoLu3`Q{ryp8Fu zKV=s`_PkfUwlTE(o#C1u#PL{fr`&mlU=N;7x)UZ_rg#|)95jVK7<{~9S=vVRJMYHt z6J(L&GXPqUSMKZ?YM88eEE&5iJW#$L*ow@SLBvjAUFq%S`rG9Ae~isn2UD;e#1KxH zvzy~o&}n&6IGL_F7iTg3;%pK0l5{<8qOW>3;scn~^JN$*Slgrl^&+*}k-Z@dd(A_l{L|Hn#bD^rso$H&~8#KZpJg59O2W{pGLBJ0lkuD^wPa z#~8k?3wK=KmtOq*groZZn56vxfZt3-<#?dc!15Vi!U3XO)JiAX(S_QheDahk>b0Y| zp+T4R8-e}4CS0l(t4rV@c-x%MsIa8M233YE#g4$>R|r0=1u<-5eI2cKS)UoHG3Ijp zMEwMf$fK$CSXTjbj$rlv-A9~=mrt0_VT;{Z_>QZF>ghy%ST@KbM5TUSN5AgQWNh0u zHbE^5MM1nEH_*Xi;7P2j+G7lS9Q`$yvL(r{6ZI@hR|g?(D!;k#gtME>(5Ny~!8*SA za0uaLz7>T4uk(3Ji*YZLQR<-9L&j0@7NG`1lFrwiulA^p*41g|!?5t?1~#mRnXrii z37WzVIafzt13@5L530c443aU1KU0L+K9#if;paPYqAgOjOyErlixD6{JO8gML|UJD z7>cR>@r3TJ#b@M-Z8NpWUq?&t@lE-!U7*A}&!+n?9p#m{N)ybj+E{HY>}poF-t)H65)`Vg@Q; zl7bN$*9ukkU3e9V_tj-QpI|kdEIQH;;`Rr7um^gBkrqG2fXEj`60qe9;_-(%Ph+0a zs^=)DybFM_%UqdQj3rYrYFfRYLhs}TbFd|GK_}SIUcxmO5@q#<`+cv;B>i;OP0SNm z@nr431>mX)vuLRP2d?tHc;?*d+T0Ga*GWdE%9I!3C-s3D*R}M>eD??WRK9~W@D>$E zn`a)s?B1S*-hj6_nAiBbdW#Zd%x3otSlc+=l=+?C-68MN@#Qt4Bok>(m;IOYA452$ zZ{+t$L;?@lOs|&#t_o#`LLeuC*;AzqcAO`H*w-!xNfrMlW_N*+=NU~Bx1NLMlj$In z)-aLm09=cg+2Rnke3FD8+oHPuO*(ta*Y(THO$Q8V@6VcG7o1iY+;ay-L%V9YjZ>T2 zTDW~z?Q%|ZBZk7hZq$2)2vs+puq`i(WUHf1lYatBaQkVSRC53OW#}MNoh(0BxD&pU z!7f1*V663RyKA<3i-^wdC^U?!;Q!SEB&r1>3EyA-HU!H2Fozum$$^>b>?-d>wygU{ zvIh?x93pABcFk0-%X1qnm|exxtxuxRil08qabu?j7qLN|I)Sy*GicR05P#-U{LUiN zXHW+78Cibx&%Jjx1gd#zG-Nr6)fh8u*b?Iceah#mEprhW8Y?19n*$Kj1A%h|Ax@|E z;8)1(sP&Sgm%J@|a>;No@TpL`jJ)puw^wENwcJ{{X$p%FJ{jxzxC?WP*ZgXbL-}Oi z>|d4PB=!P9ihh%azi)N?{tf?ErysPM8rTc#8?fKHr&v%?A1dvBZu5-2=XA26#Q{6s z9}hb`sS%7NmC>C=MqnZd5E5#nsIAno=QXU$GzRU0R@)kF^l2NT7Ip4Q7_`wFH5hDk z2ldH};~}UW8h;^3DX-%yEi$k@yuPPOkF%~kI0;Q9!X0#7_+Nb4-(0`RcI~3Ex8-OL zQAkfK$d~0Zd+ZyXUkclmYQQ{Ui!_LxiZnb<=&T2{2Df=~ypTu3e(`;5(UFNWc3rel zioE7Zx!TfJmKPGV-HV{RSME6{gKpxGSulB9w_h8w^l73;MmmE~JNQ2|eRFhO-}83d z*lBDuPSe=BvEAUtc4Iq@)!5dJZL@J2+qRv&_w#+%Z~uALI%}OhduH}C&)}Gax-y<7 z(p5_PzAQ9j;to%qT@P;gkr2Ht7q*d*?BaSUV3Es<*KrlTrmIt}{`7er7|fQ3SANFt z;zfD`@nVRWz*waHG;Qo^50BfVLngV4PO@9(-&AX7P};X0B%s{sR*t*Q_35p^-N*kV zvF_zy$({w>M6SnAcL$UqXi@soTG_#!(9^^~;G;xcU}4GUq#l8cg^(|Pa<#C&ruSwq zDkWAYNB1_A=r6K8L^>Fsi}&R(B@A34ZCyv#M{0y*unEpOA@%W3^E~4Z$|FuRQ?F3I za2lj89h|K(d}#hKye^%gcl=f}{?*CbsOiuh`CXZ|R;ClHPQs?DJ5AqBtZ$)fB1dSG zXfjo)w(VU{teo-XIKl?y;o`}6e>w-rys8xAVy5JfIm4Ird?Y`O;Um`tP|>&4?0Qjm zOiVMok+$qUpKR}Mjrl{85p$fWN73J5@Z`6F+H3Gg|2mP2M&o#DLaEw5hzqktD~XTG zSFfX^sy-zm!0Gh&-o=iafGk+j==4;Du`%K*S-hH+I%+FDk2$`=D* zP>Iu1LN)p8{a2d1&xu5W<hN z@%VBjlJ?`}4Z<)B5sjqyk@Dh0(k!No~;h?cne4{FxZa8Mt8n z^F#l`KswfisTI9a;+_AT$ps~yv6?)@q6Aa)rxZGvkdblf=11e}XJ`XH!ph!N zDqlD#a)~tVYiP@a8J{$vq8+ISc6RNJ4#;@noEKici-V#iS}?*KTHlw2BkiQdnmm{L z@Zo3j9T|@nhpMWWDQ*X&2=}yTkMzXI&QP6H<21(Q^|)`Buk(KZ#D&t@&)#Tc2ef{7 za$+OCX!Y0gu{wknrw)R)k*U(A=_ut!Qm&^l|Ac+42AO-J)Foe8Gh_6Z8z;8<>dv6JkjKOC_jrLD z-~yx@AlwBKUB)YEP?II&!BXsW+~cOB8*1LloPJ+Fhx>B8siu=ox6$UMUzo#kTIP9C z!9ZWrRP1CS7J+xeO>K_KKO2>5x@G;tF~ALnlkNtj|IoECpvNpy3!nGkG#JKk6e?Be zM$1Zs;jG-CSBv0}u$*1n#PoL_m+WiEpFd&NSZv1K)j1gNG(zjR4?Lx&c0zXI9C*L+ zQv`kt<#t1Tw37EfQUt%~hVdiPC+p-CrCl=Z*7Mhb9DSqvjUgF1=)%)#lcpRoWEM+s z*GcWNvh^?PICu&8uzAyK+@CP$7jm%wZa$@58v6fAtG$Lx4r-gnp@krl%%?<<_GLnQ zN6*2zY%=-0&i~kT<^Q<%iHL!!8A~KrMQdlYGT1u^qy3a@%tyT^Ymy)jl)@7DMM+It z%6?*6zMR?Qa7Q+_^9W^qYi{p@y>(Ya3HKr^5BNCSQCzJL*H;%kM!kl())LEKcdh=S z=KlSc@|onbQ$#kj6dsY2sRORUVF#<%YDz|O3}3~D_$pQsEWKkyA5?lChk)1eT{D8u z=Y?=7_u2h4THwe)g`pUSS$Tu=WcEbss06<6)O(wYQ0zn3^AxQI)c>UL-S1$PN8_ zSE8Tf=VRtFHHJg*sTDo9~KTE=V#^LB_kQ>NP3!KH0?6=(u<26>@4ayO2{4NSs?S+*l=&pF2M!I_3e=t@}m=?CYjGn8F^ zI~OxWfO%fLxT(}-TQLFlDwDzgav}nZ{>HHcDB~FG+ppc$fjCzJN`=K;k#b$vEr~&m z4AQWunh>nEFBAs&pPj-Dr5s$WUQfs2O^1-*zfs1;;n(xUr*7_<&-s>FDkA<9;)zSe zNeGgm|5^e$HE8aE>c-#(nZfDZ#ermSo#w^A8l$3?_?lg+zWdM4#MGoUzLyem;2;f^ ztq`lA^v|SZkK`Zb@OjBco0aRmLwS1IQ+RAG#u`z;rfm`|(35{e|BdF#oBOb=gMY_q z>}FGUcsU6Q2Jss--neR|ktU_mKxJRF00Fry?a-tO2a69!T#Y!GUc*_&ap@6kY$(yq z1C<2bsc##Ld~moedoU2__4bnU=QX5MQv)jq#7MnwM!5{FE7M2Y$|D8k$L~5I@o4{fgvI zh}!-kPU=os`9uuC)p-H8MsZV`Z>Xwoy~B;U4z*5z?KiW{AqjI%N>rd*44dt_F0Az~IBHTw= zaEQVNz`8_-!oXit*`tV1i5Lw*zMjBskfhmiik zUY+M&a}ct%@a@g{p0-q60tsDLmiwAJ?#7<3b*U&Jt8>;V*nx;d{*Du!-o5MpQKUw z{6BbakL9~C@!$z!2B*$LkdWQAD}@;sM%gM$#Ms$Bxo*(VsPjlgXm~8=Hw!sxu#AuB}h``dDm+b0!f-NHpl~C1soTlXaGj z`e$mGf%>YXDEDC&T%wERkEXMEuYXc;c9rpRDbie=tzDpD-gao?T-X2}utoAJhV{N5 zQ@xPguRr{M%t-n^58P?r^P%Kde2p*8Ih-zYrz~)n@7qaDqqsr;4h@-zIc?1~zT9T0 zt1&(mHEImcC3U54R87%*IK}MQ@{y$TJAc%s3*1Xe2?zmES^-t!l$8XTzdg7It)Z>8 zksF^L42*5bTy!KT-<Qd1?XOt4!)BB(NH`zKaXcp-K{FSMUL=h2s;R*z6o3ONW)fCEBfyz}=QMJS zE@B@gx&H*QA*nJ)sPfwar^_-Iq3q)Q2GExhy&QzA-q2b-@1j{r zDcoyOPx^5>8+i$NQRc}=KC4P475laq`p zT{qq|ILGpWmcF+?;a^e?cPptJ5SV_EQ)K9S-;99o8M@oS-KqbL>;jvwh$%bgm^Lw? z5@QWJWFF109AEA)35ExM6P}GQW9}HmpQffE2hp5^gKP^j6#G}%sy4rEqy>1&J1zv{71+=km2ge)Lo9_h<88zoob@K6iKR$f*QXANL zsF#qXTpQYfsO#BWX7;{0N^Qi!1o+ElF%cJ7&AND?XRp>6i;y5O_km1o8`tsjMSkCy zTl?VvrGmAZ33D+SCBl`({FC5_j(UHA3;5SXh!PZ%U#Or2<*pZZO}XnQIW#g!h|(J? zBaQL8x4RvhM=J|?#<4b12TkYo>^Nd4#1TdRb%12Ikl<_(1+?gOSU0tar`~GZc6&$F zJs8Yo)NH3$dJS!IIp?plc~&XY&;bfM!dHFlMPx*t2SSZm;1gH7VbaLZAvVI0TS&*z z#1RQ}yR-cOh8XWcN@GoOKP63vRE=80J&^ItR);cHrTA9T9Ky2YbWen>f17K4_y%FMdY;^U;c#TMsv$&~PA?VWFwzd0 zQO>)>7+Rd`S>})9GQCNoGHp~ZHJxIk>kvQ0u^ZTvt->h@djaL@bY% zsYKmV|Ig6^N=J@1rWH9v4ZAm+Ly5y9jY^E2(|bA`&bK>nJF9|%j7YFrgvE)TymiJcOFm2SN&dW6-8=bH}I4va|JXuTVgl~&<*QAnHXy#agKn? z^w-$qR$`mve)687W$|)MM+xMjH5U`rqHF+{%s%=Gc`LjX!aSB>VEz!~X+>O`h+hAf4AF%gVsD{|w1%dZQc3v8q%z zE~mC=VeH~5q(8KL|AnO&Z$EnmiTMjttav{!FloZKsHLjLEeIzFbF{;($FQP+40vLK#mX@a zAKQQ-#_D$!>GGBmLL9vLDaoJE+_;Ui@ANu~I(ztV=+b)^jZPb@DE?zEBp^3PeY7RP zTn*MH!bCN20lRcy~2QhAJ&8YYg_XKY4%0* zs+o?{nYFEGw|7Pb{QlO~*7QuvoPPdK{D^mziP{h$A496Xco;W_^aks*ts238HxZqR zjGT)MG1Xqu97wLi;74)dLV?A zvUbnY;Q9AmW%UBlWn(+L^!MHjZId!}aVAb-JtDcGsXdp0SqKh zx54CNwkIvH;veNEyIrP-*UziV8B?<97-DK7@M{+Q` z8!7k7c>ILi+=jJ|x28Se)8Xq@SzbStoyPrX4G6WDeDZx9h!8@x;T?%Z?K$=@IrjN+ zb~*csj9&X6ifZ2DFI#m#@C}FAm6qkiXH;R8W(5cPkbGZ^~2aF34~)R1rPasnH4P&0vX7b!gqRw8+`IiV5RR|%e>_rr}IQBoqdLvu39)H z$^_FdUK0Hlaj`MS-IW_qIU4;i#Y6T2-O80)V{-aK-Y@UA^+HBUP%@&xTj-xDkbK%U z1wlb#DP}X{IH%Y{b6kg4kht3Yu0HyYZq5b)eL<8$h|V~9ZDu9HdDj;?_3l@(4SB%GS)AID>WWXg9P!>i~@+_oV=vn@WXsBl-O2Q z5(_iwSypm&Mi+boBGuo_q$>VV@fPB9@Hg+Y6hBq=$o3b^XlHkolw4iVUIGn~ndk06 zJy7sVm%V48)=N@c5&KF@0ypx4Mu48We-q~_H^EcZI?B2#drnLFbKK;{J+lsJ-FO;M z!HPzE?DDf8i?0Q5#gbBfsC*ma>Cu zdai?aPBTdr&}C3z;?8*CT)w5MUoBG|w495>(Z7mu{MsRdN!W))ygGmGEgHZg&H0^A zcIJsrcBYE=a$)P|@#aVJe@5zTr#D7)AHRy|&LOwx7+^?|&OGAu1r^5CV>UwXw)n-D zhuUSVme{ad%pH8Ngvu#w$~cEZ&JGqloGFhz^2fOQf8I~fi3Ox^^?a%x)uhW}jINiO)YdZzr$ zWRS&hsYMh0M$}F}!clrmQ$bnc+56K^J|*qt_y)<%-3ajtVV^_xuNbJR`(X#!)egdd zc4u30W38)toW%%7eVvL*hvokOQ62}rdP=%{s}}>j=eE6 zTAN5_%&6sik^*m)l@8%sV>(R5LENitvhk?%J(fjObOStD!_2prc)PWuT^Alnx- zmx$+1V?PJB zPT6+bxS>1`_Tli-0B`cEbf%8II^EV7tX~#CoTp@>l3g0`QBjG1)B0!|roET$g&)aA zoQe;JTM)KM0w~gb65)yz+c9&VlcyM2-Y-%krh<&k`Hy0bR(Jf4M0=uBBh~g(f18Jz z3emS3o#6!dzZU&xaP>u;E?h6k-u_0YGeEHfXipPKof+(|s+^ zZup&_SfU^9yKK|7vLxXw3gS3FO4h5zc+xOPdJipSDT_)gS#>uPCY!>{p=(MKo zPx63~MaLA9N85cRS6L>G5uVKbHn#c@ZjCOb6*CF~ZOC z#l|OMMxM$eNswbFCzH<5^3$0?TL|Sgdf~rt^go2G>#fh}(w~29dmd56!^^cyX;Fbh z_Mpqhp33Od696$0f&Av5T*{ftEsdka-XSpZ!y^B-Bc0pZCuY7|@3ajjtAdBPF(xM# zdGAnoFnhcGgr!-AIq5afMY=LvFQAb6!m8Hyt2fvTC z3x40(Cm%f-QhwR21mq+zoDxy31Xdd`iB zx}MVZzt>N+PR6V%xDi(M(WrYUUM9Wwz~7E*|G^fH>)TliBo8fdbx@uc67i?1iTTR7KeOshJvyg4D*V2-`!8 z%>2Kaz4?zu_Zsr=>X~miP-FhD7Qn08e7^Q?eT2eY@e}j)&@YUQ|WqI)vC( zQZ@YUl4ew2=YlWOX?xn|$uQy)e94z20bWnXS+9%j@Us$6+TBN!bcWOf1dZV|z!GTq zYuUBCy76zJm(N;K%~571b}Ie07vsOZ0VXT}V#CB>zi1A8(3_HW`keK9i;6w93GQ6wY>R zC51`IN#DwArJZZdFF6tRFPCL}NsRcboRpy4%3v=o7qkgfU&P<$LYd&Rx?H*L@!*J+ zeD2P(|NG$ec*p&bp>0iR&#~ejst``P1DjlK`+?Mf>6`o<-ecQj`W4!VKE`)`)h&dK zhjulfh7d}gny%{{LrjU-gR8I+nrIbnNg+9{mrYkR#f^!%b~ql0SqFF7`q*^vl9dI%m4Ql&KMlP*biA%y zT3Fo-s;$Srxyg|>A5>1*nr8)x4P#kdSgJYmPuoZu$($q`GZ}BdQmA?87~oPY;$Kf? zXC4sN^RC`@rysf>9=6UQ>(M&|o3?(=xfgIiqN~wBRh~#u71{F4;@t-KZIkH)C>%?1 zm&vbV`vI1ZlO1kA)U{LWW?ft^V;0kX6fZF#U6oB;l%sWFAxd7&Lqo2?s{656L2$bn zTGS7#{eDUv+vbgo9ogDw4>3ddXa~e}nUAMy0pZgP0S-?EYyH{0(|S+o@m)B!G~rxU zd5Y7L0c}KG`VsZgy?KrazVajPxK;lYLa$HF-BiU-vJrd%8C@akrb-U(-JAUWKdvu& z!lv0hUX_h@ZX%v1RJn@4U1pBS+rhk5Z#WFLM1fwi_|!e@vsXQ#w0oaqU!Q z>9i@3?@^c!?-GK%C%91G5Ip;>)B3xt&l<5`-Ab8jqBx6rEDO*@IxM?%#3y8hv*?Vn zV82n!$ZNG<4XpQE3{6f=;6M91T3x5LN_sga2+_&Z*dp+;@XT|G7D2oq$fVlY=EN^uEC6xc6s&RZBfkb0;{v7#69XftPGGHbVFo~>P(%tx4NTS zAyE1)jElAtjZAcdj}TIc=ZJ5v@&_YR>Fo-zMyI?SKL$-Rko3IV2VCtz_GYq+74ktC zp(OED|7OeBLN~URsPK;2QYh-!xcJOOfVb=h?ca-q@?9TaJ6~tNB%{cSwlWufOW@jmsHL=N7$wCS+S5bzLt=>TE=J zMyzjHo^>>HiBZ^lHtHI!Zx2z9&8K*?4QF$%!_u}VU2i`Dv+tncriH+^+fr*71<}@8 zy1ef6ufS<2CML+kJ9qyL%f8OQ66p3V;=KL6ax&&w6|yU4C>pm?Yln({RSSJ{cF ztxx5D$bg<7!e(9=(AjNpN}2lr4xN;Xn|}jaktEJfZO;i)Qiz?(C;K^-eakr(&Y5lQ zXoyhWaswl;iL;U zk>|Sy#y5y#{4i|=vCFTa4>e`16`tI#90O3f$ze-be1WH6+=iDzCj{!`oq?setEP>o zKcK%%>awKg&vnoh-ZoInoy$;pD`|6VC+xqA>N{H30B{VChzG-3(M-H6HjaeIR@|(? z7Cdk~YRiic2x=0#kq%=kgb9&fLs(HR9 zqjpflRY!Lb6ES<(o0oGI6i4T+%amLVp;38b((tFo$+I=;L!1VPqQoDvVZc9nlvquk|nEgbyB|PVzm8(3uMy zvT>>BFyS5?47{~hH3q($oWTm0C62k+aUBL0M;r;jX%&3ePV%ZXq1&>J$in&mWY9wF zX^lZG*Su!sW)yYi+R{peSX$qcJC^{#`=`m#(Cl!Oqlp1AFWXSEKO#J)M&y?VqbEy2h{covwbc<8D&M_I+fZifeQ6+Ag$yg+0? zc9dm=QZ7vN)O+yMGC=%Mi7b|`5&DWpV$ru)HMGaRd7ZqVmi=#=tn>y}Y;fpwB9WX=@giuhLF_i`bdGSZ*P8Wyj@-J2Y*!*A7QS+l0Unj?|Gj;BrYSQwt zp*}|@Vme#%ZpEo|5^*QTmo{*>Sr9M(PAifIr{+$iv2tW&pkO}L@hAY2?k6ie$1@Jv z;rHpLzc|(#vt`s@|i1L_i z&JMInz$O)&3ACbvl(ro7L!aNwjY9cWbb<*y)(n^2lMBcpe<99a1&>Bp1K%I+v);Jm znbZ``72_n&2mTC}8Ph9R`w)(4AU$mS@^(cFL7}q=Jz&OugFFD-*!_Wu(D! z%`d<+_!!(X1>;-UR}F1Po;c|Yv3;1ohCIMwC9(~jsNG3Qp{r6EWGgjKW-T++gk^K4 zgPlDF-0@&u`;!+oXfNW{HYW2qGk3YPjFD7kSeO1L+&yiWaS~!Y;{)YIvpdq`O6W}D z>IgESePJ9aqo@#aCxg&{z{chbuWY7N(b1@h1{1Fp8<|6)>D@5HX&60C3HJL> zfP%{Y;<;-c-KX;Em%2V=pCL_8(*>2zznu^_N_vNr_k7wDyftMQ09E>;~c&@qB6 zLpajEu&HkQzGBIUjCyTp09)U_*&!!CBld~;O4EN&Z8NE7Ia&J`8v}OwpE+U@%7G>$8iQkOiBs)sYVSO$TlTbHv>(;a>Iw)`M4 z=6Mst%*L09MC=z;b0v-C{xQU6HFVRRx*;GR&MkdkqcadCZcx7C5cvb?HW(a-Tmcq1 zrN;h1#oZ)50h2^kLkKrUGUhbXw4rLutG! zQ-`GvCVIWvcKH0TduG;|!hnQ*-;4&zvSvU#c2}&_@{MHF#=zscEhPorJ(MArD-jWI z`kIII2RQC^DlVV%`DVlnr2wmdvu_;IjegvnpmItdVjQ(}zR@#J{h||Bz!VRj;1Eln zk$J$Uwz&f^yMCOlIEdBxnXHb$eaGXu9|OfsJz%}QGYo`iW4*j&w~|%Z1YG} z=vr8(9pBH#Poa?J!2>9cw+uw{z9XLh@-pAd&ik<3u+AR5@3m4o7oF_?LU_I36qNI# zxP|(zB>l%auqVGkxqF*7na@Yz|13s}4LDP~7IHHiT`-Ke!-ceAI|`-iD$oP&JU4Wn zpIQAc+0&tEdlB;UF%O%#cCo54@)!&vA_NaCL!`qp6H<2nR`YxVdu3(AuMhzv?db)^ z)qQOFDs${F| z3q_~p+{LpHZ?nfWe7{F`2NTm22jbHlre2;v-I_Jk@+lcdnQK#e-k<6l*4ZoR@3jWm zDvga$CQcg7eJ5=o3Auk$ukU2?)=mGffIP*S_i*m zF8v%0(f;z{i;mr(+_Q3m+b%;>Zg73j zLc6;p7(OzeOj32Ui(yM&0_nbznI7E6Ss~MV(u~OWa1k}{a&lMwHQO`wy0Cdpu@qDX zpKexI+rfe}4H5oQjQPL(gqPI=0~_IqE5j6&bfu65KjWTo+>JXa?i-@cld(%I-u%6=XAI zfWTSbgjBoJ+j3xih-MeFWh$)QFcgIcrh)3JaTEYPw_=k6&*bKP@c}LtZs6*OhA#s7 zMmW|3BZR+c&X(&)8UAevSzz(>xXAPwO-(V)pcSpz=(xEkXI$(YnhviauB{|Gb9l|hP<1cjsLlCl8J%%URfjyCoYTr#?j}ahh8-U9al5>FF*El|K&h)18yuqRFU8D z`)du*11N_U5p94_p5f7Xh3ukOYyyIb(MEf)yF2?XnqOU3j22Y>sXV7f5R@R$!s1L0%7{4YL|YFK5m5STVFYvAI+Lo9=Y(k>M<-6@E&4gBtTpJ3u)!t_36gX_OfV3BMxSaq-B^lI=hm}6 zd^~Z4_*GczZ5KXeKx(eF^$F!hpTTa3nslWGi}X*4Q{g1l2fzpLVso78!G(#3sn3r= z@1Mm^@fu2rynJD@LoXa^ROa~JPsWi!&xv&T#-H-tJ%m2CLhlFvSvMudLD`S*YfGw% zu*7ICV|2N+Hi+V4E79PdAWyVTvs}j!o^~8s#~9EDjW2LFGYpYF6JgZ>g&cn=Fx%dNjWYa_+WkTL(CsZeNVI*?kf#52@kd3Lvjb) zl*%$~AoZ5DgB=!aZ@Li~+Dv;2OUZ^jPpv6ipX$!yXlX<#4^N9Z$i8}P@G0KBYMphiwquSjrS(X5PUuOv=|BAD`zh0lNa{V0M=FK0Shn&g zK#QjYp$0FiCZq}~PZnFvwfB>@NE+s`OHekJe+eiuE#6Lu>$__HqD9BCi4?Z8WBhbk z8H(P-mLqtDI*P5iSIK=q$_*PCXpx0bJ1>HuE{YGT3H%YK0Wjd6&ZUHOB-oO0R#81i z9nQeo^`)SQM1P4K{T7^W#zB|L?n-m@Er4*fKRoq1@h7R@ZmaiNZmK?XHVrT8eg09~ zf&&GFV|}QfTr_d@*Gk-|i}};cL$l$7d7f0PH?dl*Aiwy`g}Tbx;Fp<(6~Bb4FxH1r zR%^tJqO-4$620XiSi56fu_VN-C-0AA-=ie9N99yAGoKgV9Z$rT9 z?_X#m^mQEcS}X~%Fk_rs-^;O?qV#p#wIZum#qpZCIhVsW;D#dEjZf?CTrUO25zFZ{ z6Tqp**ML`@%YPy_j{J%g@fV8>=tTQAi&#@sc{LO~2uJn6r38=6+FWL#DBq zt!<^MHMd>B2;^s%fs`0)wBpbNJxmv4WxALN;d(P%kDfy}cTe_tiP+)r`XB%JHMK!0 zHe4@SaCqMFa4*1d9qpuVbdL|q{g#59OLU8%sBjp8)cRhWyCv*26{~-Hz!q|SofLg$ zb5;cRjPd=p44=&x$fjOqpz`Lf77DqUhWU)!lX@Sriui4C)Bhiv9Z;k)@WG*FP7k~1 zPTW+yzy7quKxNGc<1hMCN(4#tsPgs!PB#8#%GSk^-$R?H(Dc7kPQcDiytLH#dsy~b z4@cOR-y+v4t1>P^}=$h zm)13xSRjm=Pn@`?PNPq>x*kO^RDvYa@jWn6VLo#Qf&XrYbAAj6{2Hh9$L0RHnV!sW zg&TrLIc{_8(hjwYi%MF>>{N}Bw$U?p3A3R}yYi`AcCY^7rn&O5b<6%zI&vS>>9g4o zZ!FG@v%fT2ua@$^x?>ln6&xjM$nD%s;KR=eRuLxgo@7o%fy)Xo>O_*(<-chOrWAIJ=O4he?rxV-86n%A9svok9Pw zHMu0<1k0jtv#2_0^%Fm{mq}C+FV24Ey8z|qy2fT+J=yOf+Q+#w6;c&W9%5_`ZYw#U z|1UyLZ{_cFA2?gOL4GvfyH{W^`UhHaAagglcP-|hx;nzd4-B$umhHC@&L8p8be7aa`FotF_D61%mCjEkAxd?%^i= zIYi*;k70*=)I!8Huy_Q%El@ql?dgEiT#50@DB^*b*ZF*_>n#d_;9gb#|60<}Nnrb= z5>GS+*4%O>_lz$;-qvX@pY|rl?x8b8g@6(1W#ZCwgNI>9m;RUrME)x#T#4mNSyH1D z_qcLIuiW|FDg`MjgYirJp3cE{v2A$5rT>Vv02}GtX2|{|iH|#oh*r|>3x(5!el1V; zOOD2u_{}fpysC3^h(vT&%wOk4PmaWXxXK&4?uSZWYqapdCJbjse8=63Z)vozvL5L= z_#rrkb^1^56OP~Tw;P=afobo{a+hLy{l`!P)1WY`ExN{K)Sj6aN%hNe#)%hZn4C;J zMy?Qq44A+S6bQFNE5>;2>=xBCHZ@DP=0P|31>iqw$};6Pyla?^d_ zBBIgq`kbxw@ls~l%SXvN;GcSIeGd_rV3$nlJMb-1nrrGcm!W;Xk;eG9vw%IPFt0k_ zGr(GA_jLE4Mrwbj^Tp`h6rYC|2*suKRskt@+g7au1D-0dUT7@FvFPPcRFyP}ufQR7 z7uUatjCoQkv6BZD|6#C<)0K?3tY!%8BQCW8Zb+HPj@f~>a=GiXEqwp+GIE$~#`YAc zEOsCi+8l6qJ)zL%OX=jSt0hEI9KKh44+0ZVa(ZgP{7L(Bi0e#@2bEiQn=&AUVocU= z_GIz+jSaQ-zMtwAb*XH%;WwGi6jBR!P z=onXaLhC-du3!PXeZrjV_vDqQpl;Ww* z$OgSmy;Yu=0l+{h5{E}@dU1E*YeS2f_<9|R&}M{#*IRm^cm_>OIF}KDU@huKCbnCX z0W)A3yZd5-nyL1@!Nnd&?HO;OH0ZscN)wOzyK!HeOO{4sWl?6+2-~==E`z@^$8YRy z(&DwY=jd|I=CMwp>%64`sEdWNsamFFdpRa6((N#V;Mb8jq+*HZeo%$-r)`S0n+G_y|P1qkXCz$RF|0> zp+ya4{m!XrlfJvEH(X{bD$E0%rlC8Z6YuW)2oWxEpaR#V{QWO)u2`<$@HzNNxWW2v zO_V-S`nbu$kc!uhyiKs`fIu!uL1p|-2~qe`4^+p(_{Z^+W!gxRg1LF?l)u`|S;fu{ z!5`^Yqsg}kvJ%>K=HF9g?0)}O3y=-NWw;X6{4I({Bf3?7JLyTo0mRFIg*)h}`&&1p zcOEVnt#$(E89!P{!%F;3V+*ghVf1zmaim?iof{LbxLk(OzxE8J+I(;UB+Hu!#`83P zQjc3@bBnRAvr6^SbGE&@#G+0ph(;9v)bz3##9;md_Jf38O%UyBmlH^nw0Ax`j0_kj z#o>$sK58iR4%Di$y28;92l$4kmwdX-UGvBv`za@Y!`)dJ)!A?*oV4a`woTF%h8?%k zX)WGqUUn?mWTR6KGBz+=^6CFiFsE8w=DZZ|{?Wh6kqG8FI;qvTF@Vwjsb3(*5Rc{| zKGU34KB0FAh$=s(cnjF=obdkZ*?bf7aw8!%brJA=bIMMW?xPU}I*5qN(U3(XL9jtq zJkKwzuV~M$w5dx&$y5U8s%_OPWGgFWF-Bl;wO6eMo0^z`75Nm>{<-29QcxG|U$J58 z#`zB)hvoX@evqe8B}iX&=vQ}~IOUuNn1_g@h8YaOM6zX3HAEnb);=2~ySRPM%^{KL4936Awb%e?I5Sn*`95VYRyp4z>^Pd+gPizM%7AYk#73-B-nUTiR4tGrmg1qMuC z)-k+MVQ;Y6wd_ieh-sT{V71nQS`N%*F6Z7^+rOPkIjiLIcT1`1lr*j3g##iq##g4l zX`V7vd@Yps9DWG1$YFA?Yfrs0xi2Y?p?iv?-$X6Xq(BFnFnQXyT|6I$iU}ZxZJ3uN z7fMPF_-X^D+n2U#;`-i9m=7JB{8KM!`4pF1R|^EwNJpZQ zM*rS^QSy@<+Q-F7n|whnQJ*M1FGpGt{F+)WRZ;@`|H%8wwm6~K`g9{P*E@eD%%=hs^^E}hHZL-X$kp4IeiM;L zD$D#Hh(i`s02?xjJjeUfmE}We-HGvp0V_HBo-jf}W{t$o$VkFX=a{Mv*-!dV&%f!e zft=VXllZ^5vn|LXl+%8%51DT6KL@ab@3T9WBVe!ys|&6wB5g*v$)XnIW^@rw8xi@@ zuXRE(%>|k8NEQ1-G%MWNgl@t&E^N^geBXlN0>*|*sPcoKMEB5!@rl9e*XBWt1)Y8Yf00s z*{1AEC=3Io9G@w%6nHcFWL6U!Mh^r8#H!yA6@wVaTffQ)Q}eRt3VEJxgFw4i%%D;{ z1LSwN`)5;8tE)-wi03;Y(i<%uOA#=txm_sy*lT^OM$XosU(JIoc;hUN5pJ`%L0t;j zQI_OZv&+;HdB|0W;_XO3&{5RrzGBG*=Ll?|ue-nm-e=35ESiJ75NO|yRtxR!Pa+_R zySscF>M*go;fJXx!Fy8OK>q2xDe@h+ueN~{%)vgRiY9SS(4b68IkJOSeAJ@ZRiVvj zfi!YS4rZ=uDA%T^p=T|BY4Td^8x-rL8p4AKUpknYKp6!sX-vf^K)pu6e>-n~%7Cjgx}%CE{$k8Om|Gv;@RakO%=}P7)i9LYg|CLI-W-^T-S15yK*Z2!!q_ekDrQMk zPS@A{U=>0$L4h#K$8V50a)Wk8#A`66k4voiqNAqgtUXt#%q^D%M^Ejp*v{)OThhZR z?MTr7!cE~}w)vMJ^?37qGCM+{Vo$}OmR1Q3il0kgAN%X?*$`(c0GQu)-4@7O$Zgvrv z*?R$gy6Y{&3ZPYGx3Q6O+=+KP#FGA8RtF%z)c9XD%l-l!JYXi)f<0}5&N^XrPS;Q9 zjy$R`)kpHJ$wNwOCS(zYX$C2e=%IHr<~Pdo+!d3zyJi|I@&(2Ydf}rDM5u#B-6I8Z zG@w!?f$M&=*wt3;Q(oI1lrCOzf6eo!k&jV}TKG!D*lHFi9Dnd8wbjSPcy7Q?&)94+ z_PA3Nng;dX)FIR=?4eZ!5B^np7ApD<+cn!3go#G4GndW7Z$||+Ma!6Hqxn#Vi%iA1 z?vMph0?!Yzt!+k-_Q2Bjraoc~CtkGRAv0)CYvtc4m0Kod!AQ)7Eo%PA=3c?@J?9UNL9bj4yz*LR!Sz4yCys#1;(f;J1WBlVkHc6IuwZR zW|G;Ntxhj}dqy506RQSA9X($KnL!bCjU*as=fJrP>IL54tH!z>|6p<4a$(!Bt08G} zlWZ!@jHicWD3Qk{&sKUe2xJ9gN+id!q+q~bW5r&H*uf@rH`dXVow!@1+JdT)do zMl$sVQ9&h&qi?& z*sf}hFt-vttWa3$U@r;}C3FN?#+>>5Z@Nc6GW@1vrnluMx#7ehk~&pXb>K z@J&#)n)-*a!c?%{%oYIwwpz2{1_4IGT(W|qkDHVfFx>Mh8d}n#0GyE{DO&l9y831k_wI8HpbmL{E$69iO?@oP_3fmbH{R{ZD&Z z;<#`WwVN|I8dZPVzuDbAjRz*GwUO|(YN{pkK)*KJ{{2lxGi1KtJ3hT*ao}%TjsfTS5=G8f^mw+nGm_zbP2JWEiAI5Ud(!aD&tN zHg>bI9rUf4;><4@GQ|%{^)0c9t6|jC7$ZXK>T!~K1Sv6Og`!BLf7A_&TJDi#peeKS zUVn=2x1^Ds3=Gs0y_P#NYp5gexNVH^KKSY7z@H)ly?U%?G2PjLl7!{k%?_Jmwbzsx zaM6XixUj}PMCu4fbJ>BtRszNy(GW7!S&&!=?S>B%R!f}~I>iN5G{Wzl9Hs&O&i`MwciBFXBjTm1OTN)kCw{od2Sb|f z%cr#hM~U;zELfrR=CWw3T$TrY%+EyKRZ4?5y-KSBDq7o$1WH<|Z(xl3oHf4B|N637 z$MQEr`FW>Y<}KdL5~A--)MWe5>-@X|I*f$r5<_!+0kb1Ha{Id@#++T-Kn7-ZutG~- z8@sy9Y5z{Z(Uk4oQwen54HSr z*MeQGIc_j_)!Wi8*5`|ryguMr|xi*VW61I}@k_Y~?gNnF3LdC28t(|_doPinBP6dyjzRFrHbkXt?^(y!-4jB3$l z3Va}j$v19WP$rId524lS^7Fj!KTHOa30=9#6SMqoHvq>~5~og!{2e2# z4x04USov7=!Kvl12;FZs7Q(rAL_b&loPD1AZ1C{@rB%YQJlWnlxY1s z#~R^th1Mvt`W7cSaI5OvAp=X%V;W8#Q!R3=27eSjggY$DXN0yjwfNqMQG%^9t#51@Aw71tv(Bc9DIOBwD z|GFp!HgBqvg)FZi5P1$ql^kv)_Kn0PoaJZ+j#@0-qL3}mfLQn6WXPbsw#s`Cgelf^ zfej8yg85TNn$Af@m*=>xrZAf)FYb81XFm`#yX=Ub*&1o*|BC2@(7iq(h{?R&2)ljD};}o;d?U|vwD15&K}>G#+0-8 zgrnpj0c({A?g>S?IwB4ff2L|wl~K4y92b;S;b2+vg|lQ0n+(CJKw1#HI`49vT?G0` z6#8@`42n48Eyzd-y>9QX^Ic-8sbk2m5@lD+{rNXM1ayiOFA9?iM*yQ8A^WkuveNk9 zVUkRsKU_A*n37s#U2mOv{ez}2wVg4H2O(hvNMA3Tq3frHU+OKX+lz=d?pHB|;7j}- zto)VA#+9juHUIGNqO8!p^({!o#sbagT-$T!&Y4|uy5QU|#k(8ROrK!4TcTccFImAO z14RbbT!C(@m$~oZO}f78YnR#cDjK_62lZV~ZE6>B+#*Id2ZL4*kTJUk;XD?|LZid^ zb1=qyp?UZ2Z};E-6ow~`qFcXgmj+|5+}{R{2Kug_^3?VH(lWhw1UX9??i?0YtaY>F^&@_vRPq-M*0Xg#aCV4;hm@_b z@ZGKNo(!c7&@(BuDd(};bH7Ie6H)bkRT0Ur(0%@GE$TTmn=1=~mQXt(oa0P+Lc(B5 z;o@LsKmxyJeyImGYTU$VoxlgQYur?6okRs7bHFyWG}BN~8=25KM1g*nyRlHy1VBDN zNG-Bytg+azX)K8FJ2iICUYdzWg+8oVrySl{M)};lfwUm>g~L}#aY=y z7zv8~k<;fM3C#v$`m6oX&)_x?h#fw;OEwS765ROnd;HC}O%Uz?Q_&4|M%e<$OD*A6 zAs{Z%mD(#KG5Jmf$7)~u;5hNGYz6x_Wj$%_Xt{KT)0-00leuhUn8q2Kacg}NGp{HH z!Hc>T+vVqu&UtU^wdonC!U!-oJ|CCHZ=<17&@ZCwznl3TYV)NKT@?0PDfu_ZNx@I| z&MB`yR=jt~V|LUV_Rj8itu811-=c@3R~-K;XZ|g^%~{+l%4jHGkmx#*MEi%1_)-Ld z9{noASyBF=|IdipN$@9PQiZh-RkpMjqis=327JBZ)aB{M!l#GGTT;0%px?{|JaAqb z69jFKFfY5d7v~t~)+%dzS3L||5%XD(R7R8D0mTf*pm1dbCzw7O51(4rzey>eSsE&; zh)?KKtj@wCf_<4%ALe52mn`bJ!v| z^d*uVTyDyj?CkuYu(X$FA6~k{4JVdaAGH7O4qSE?pQjj0+Xv(leqLRFhxoMPS`Q{(EA9Vbz$+HtW zOvcn^vFw1(+u!bJV`wgCYoxMCP9T6bMHfeA&#P1Twl`7ZgaO}XvQDN>y4wH&WWFxS2^4Z#Ku(8%1%n9!UdS7G6| zGCkcOpR*cXragC&l8eDpL{w8cLk)Rym)m6hS;wMa z?IrYFAFc?W6J^7!K9er&OA%l)1Vs^~X!@v;@^Ab2&|~_sg?s6^d>-0?{#1%PD2kLt z^w>;(M``!CR26H8@g4CLuA>wLRi9DUxWVlDRFLf8O{<`8gGW%cfZ)?oCxs#6{LAC7 zW3>@Y)!+8W>!XpaZ;h8;Pd^Hy?e?1dx=G}Gs+y-D{ASSyh!0*pCD&(==^pO^bOyiK zA0}&t4|%69v>EChFUp#Nq~9W4-)NN|GUvtaGxmYzC!?xj-&5&r+f{)?Ru4aG!}x2> z#I<63BYibnmYyZa-G&?y(2O+QDPddKmVDx*&yn~tqvvX_Sn}ln1xyoB&GXO5W+2UI z!?MSn0O4{;@V99AhmT}AfAT5;|Nf1jNfUP;iT%Tb|(|9Tuo`U-@ zsiW%qNSnV7^V-S30-`x{Sw31C>uT$ zNPU?X{zR^WnT6a2Yp|M}2|6SJ5nJyWmEw+roxBb7v6_PS;Z91Wiu5FJk4D-BdkZfO zzC`8deJCxfAOdqK>=bFeSD_Q0NP8RZdwxTBEye;&m)Uo=+Kb=jxafOd z@H;mmxrNmOUBX)8opFD6|JTuU1y!pm2dQVJ@V|L|$OuX2>Zw!S4qkHt7AQM>V}do6 zl`eD^l#u~gpR896DA z`uO7P?T@dDB+LotdboF|a?Z|m+;~7`x^W6#!tKfsX-DYKdmlUFQDhM}_b&34k64_jw%=e&A*}*Yu zm56_bISzyeRLt7-)}DDSO)Z0Ht3=aeqiYqO_~DZuYtKqp`56F6Ce>bAPDuE>PHIjA ze@>T5z)Z0klwsJ2cs=Q;w29NqY6j~nZkq0ogVniO`D<#UTe08FtxE8~cxj8O<>!NS z!4WCBT(9t7wvXYEwV%oB!0XhUgk`wfQ$V^Q|F+ZNB} z8eg8Cw~+FmIP%rD(8P$%6wl&k&qRCf1OLn~zYR?k*vQa&?_bFVmNOgipC%Jt1;PSp zO|nu@Gw8ug(^XKU!rtl6Y(;g_H!8JW{IL*+?sRsgg zab+xHJL1L$*rA-Tsp`c#$V_C8S>8s=afqw@AQMsSr@t1x8Mb3h!5W-LR7VpPTe#ZXmtfG`*d!YsCx(SiMT$Cn1M zy!6zQV=y9)jJgJSWtOH*=>7u-Ks}57bH;N-8$gV`Y}T(HHIQum=qlbpv5+o60F!(_ zDnF-3MKE28K4hK6aBgidO=fXFxkE!s{Yv$S6*uY;)Ck1Nw2Th;HSX)i+O8GCYVy-Db zg3e!tR+3DA(<6>5%!#72 zTFwXwVvU@s>&noFU4m_%^GhAfj+yc}2fi*x>e{Tz?08?;2EF065R3w->Kw)X89j%5 zsARI1RAe$#qutnB=Eo;4rWzovw=%t>M6D;DJ7C;#Jb1=6H@ui>w5(@I27|D+{9y7= zOO{ZhM$$hWI-BY0?XSEqSUT8i^!q1)KRZ?aIiaNNJWE<8YHr`do}Jkr{fy@Hl{Y_l z5Y5ZDbCBkL99Qy$+=fm_zW%a-Ct-MQ&qF=~>)uhrGq3LCUW6;HK>3&1ALv0@@F80n zVW4vt$|PumV!x)xWYdO#Ll>@?4@euq*W>=NL3CVIXTW6v2%;*q;b2Q#r=X@}L+SJDnsrTyn>7(mQ8l3A5QG_oD<}0aP@k(UNm~ zjFELx5Ghqg3~7MJ9(U*89=jpbu4I#|{8y|bDzp47R9$isX>(5u6X$xe+X>px-p=@; z#$*fuowo{(|6_;YM)UD;iE|aN{X+#P>Sz^1@Kjq>M1DU1A#pgrty3|FcNeXlXfmm` z_tw)!eYG=yqJHtFgT||1hmUvQT63mxpRUp;f!lo zp~{-~@IUbyE95jL!HiY9Q>)D-XF$UunImnW;SSglG`QqRM{dy`XU*$RZJYD$5&V2T_(&f*jSPa3vZsWRa-aKd1J)!rB)dl;-r;Kop67>wYl2tMRPQS`k&5s#Ydr^} z+1~Eg*IEHj1(iE!g4AD5Apws;BdbRxT(gD@VtDmjZY~ZcX4^dc7D>--7cFBngjCC+ zIim@Kg^+QeQUhW8kPPB%qCkNE!8Y}q8M|R`@Xl989vfc?X5AAE#P&>uF-s)KMKucUsqVL;ZH7Q zUMWkN(+_FO)e=jDJd)g3)_ctgF1315m_-kV%7BPpx_zCi)#EDy?-1wA5Xmur zJXIjtEq&zJn_ThbAYz@&jAr4+K3hPxot|K3X;;3>d{s#nvLDmNur*Ka7^lGnQiV>3 zM|Jk%*Pj^tc@A zN&y=_q}es@IG7D2x+5VkXM5*u0faLLmH zXRCL$pr(}B`(B;?HbZDTS2?AQeQG4u+r=UKJ#ym*&h6R5&BGFmSRUaa0i5*znEGCk zcHMOY*^l3Tu1f@FOs;~~oR>(@Wj>^~kR%A^S$B2lvh`S+tYLt$VNcyZIPKK_MW=9c zsN^*KMusxIggSyrm{uI@Uo@0lRm(tECzbbmBk}xJWzJ4inr4dk-?6VxgTyoZDk6W{ ze#zRx8_F&K(LUcZ=|Cb9e?T8twLR$Pb_iQ#Y zYWDl#-P`HuO5k}2qt0>C*(`zFvZOg@OiHMjf;~5Q{X9o#BWOOlEjMhTrsUdww@+g? zJj2R3lJsFt0QP_x+?X}zw$>UrV{h}_hIxb0q-$zsE_bq=gZewq=h#n~mTUcIyY-r- zz1Q5fr&&i{tf0%EfB|=Pxgn0?;~HNLAyI9QmfE6S6=A%qqWiFzB-UzofU`5Ufu=dq z{45Dzo0(3 z$}{p3Vo%-|JSgS0mNOgXeg65F@%0FPhVCQpQrgI73qe2~{aUu@2(BBu#>TPX+rrWU zEh#^Wf#%j04_k%zXt}LBsCZ0o@EhkM0`?!p@?-ByBOg`f%zeZ?yPHDPdd1=6w0F1y zfEejHzZDY!6)=v7`;n{2hCHcIwH{O%@ z*YG~o7|L-lHi3ANY*%HEO-kh)^v(0r$2I{r^kGk1!qie_cNcrb)Mx0EV z2a^XAU2z81ME(AV#gD?e{&V(~;nYk5KWf*E83(HF&rD9h`K`6FlWx$(^&&;K2{^@R zizr>pDRFpiqzUC7y^wE2Nv?4(o>1N`cZ-ViNgC{hXiQ58Zfby9^zmL6vOTDN-~DCj zlv%*6$Jh>VGoyDR;h@}%FesQmZ{|GEGHkn0H(}#&*Sp^j&iHq=qf-SMh6{u1z+k)J z9FaE(m1$Suv2I$$9vr`SkUSdmeqXYhDN4Ic>6xcATp|cKAXLa?W?rtXvAW~EzF^D6 z(G94lcA33J>3byrzKgdCAK8I)3*BS6?Sy$utSV_G_1^9yiL*6^9uLXz<8Y44>vrXp zE37kxRS1SxW3#+n&G#`XCFWq2jLb(XZ?pvEjLgjEUvvCjG_m@+EvGkM2j3;dt(^Ta zJrX&NQin$!eL~VmneDJKey8aEp1oWbz4Faz!2SnnB95=u14ygcD|d2D^B4S*ZcT3m z`1VR9lvj@2J7Cf6U3L|2x=wCNvB-1{Ix1aGGA8<{sUggIjQgH_uC!WXm7aKnQ&aa+ z&XH|5kDcC?KX!U=cX0#{{cue~jxM2n8?08PG-^7hlQMVld6?R-9cHl^r_E_*h)7nw z^yRSqUV<7sn^h@(j^&%}{LgF3WTGyN{%l%4!1cM&2BAF<=4Zw1vulr)dDG3L#c;0R zjuyEXABPq2ORbX8h!Ea%t3w4(6J3{fy=%#+W7s;Vat56yh1a3PSw9-TkYU3FnZFzf zVazd~!YhCcos_*H_g}*sO&!#ySKwq2B|I$v9^-}1&)-DybOt;+X}%s177-iirn~Xf zi=iI3$817875`@r>lDlizPI(>LwJ?93p%B*CzI(K%(I(SD3jdt!=2f6KckR=vrm$S z(VnJB!z!>Bv~=5EGKXWCGSs)L*3MQQ)p|6~=5p3ZLL;hZ;$eT4zs>ww@;%lb>avnK zs@YJSU=IWsZ?Qe+w`vY0CjYq2863~MnDlXPt{`<~zhk`euK?BWcHcdR=C|43{}gR& z7={u$=rR9N20+6^@;M!{K68{!wW6EihtVV`L9BIAZT;<`h5$l$ZCfs-lLSYSHv^rI zX!87`dJ_jkp=p9Y+^&of+|IMWxphV%YPo*PQsEsC8K>(HiP2bVmSub~~r^M2l3I!pI07W7I!_Q_M*bu8n z_#XM$V_Y_`GLQ}0LVBIIYZmoI7Tt!P{)6=#+oFvtQA}3>Z{`Y9OK~JX%L7BE;a5do zLX9xmbb{X=yRt@q$o!T}F*w)r<=9qvaJhN~eHlB;bVHd-NZT%aa9^Y4I_Kdh1elU% zF+Z)oI4qQ+VCtZb44&@cg5ikUlr9Sm4c*7Dd1BN*d8RfG)rBu>eJI^O8^=_KRb9Y^ zmHOez@Us6HxhGek%Rzzbp}s-F7!_ph`B_)5wd{sJNVaP@v)X|#=iFVun?0hR)w1!+ zkL+n}vM?4{f*CiN+9Qas`^S-mGM`dj0c;b*gNMpjBqF)Z1|yT!Trs8yj5HiKQZ_`a zjoZZCUHWb7SxbCIwGY*G5CTcK{HIZah)M*e?qoM(F#rKsAU~LeYO9lLV)4Hf(-a`n z_a1mpo~o&{+o@}~XBA2W`fFEewH`}02UAD7)49@Kz3|K=IcF69+jE31R{h6lV#bc# z9f=Wt-7OF0TLFh5DD@+6Bhubp4Aa|t-Zv8WRV-sA$3Tdt>(p(NdwiJIbJ^zn?uEj^ zaHikY-hD?TH2(>=fb$Xh_hPE`_rvncQie=mQMQ#uX5IE6oF8`k80vT8Y3N<}JNN}u zf3KInH)n1xPqhh)dtydn{7t57?kX{BrAnta1y1o^%V;gQ)3|j<#H5aKHVKm$RjlRB zO$9}}UcARXM5=xF(zVXOd_)Um>QVu*`P~FhtgwV9R~ zYwFv#UVRqj7;K70^6b;ER>+BqPrEBkV#dQjWnBPvnNaKZfrn%=vd-_)<0E22(5x=b zOrME#F$M5r)^SNIF>x(*tVAg{9m2@f3$oWo5E>SLg4?Acg|qSLD)WSrnC1*p1H|R_ zVVpXCh$nXLk(m$E&Bx{OHGHJXyv)%w1@?7Ne*U}qD-nN}{8VkvDHMrW!8+$~!gqXc zzyZeb#y?wFTn!!M&)u^M@`3dr>$-?PX(2_;?$pvo1mtHbkv%QRxevqSh7(vjAVFD4 z3`CLB?(>8UF8kwdj~(XqTmZbp6+QY5q~(+{528hO;WA}itArG4URQJ(XJ(qwv_srn z9=0#r5`~#N#kWC`n>SI3WYY#{h3OvVe3W@-jm@iVoW>4qUQuE<)dwjW~yQu&ZP>geV?8V=B$bn(D4nV}lK}fGS|XBo+vjz(SFHy24t) z#-GGgkD2okVJd?G^OYB;CtK?2$@H{y^)2t->Q10{`IR~Ef*{&ba69)>LEa(E`oaa1 z4KJ!EX#Yskk@&Dv6<+5DkEQdLwh&+Q;iX#QmJnI4+$iBR1eqH45*5ZEj@HLNyf^`j z@KH1ApQ0k4nmK<#05{V{R;Z_plm>m^KTW)9<4=O5$=#@7k*8k5+Caq4r-|WeAgoHt zkGaFcaNPWB{HE=6yvxCnk7?$n7~rW4Xg|{)c4+H2X|NUGWNN`;Y&Tn(pyjBP|`erPvXo2#CDJGey(XNGwOgv}n@4c03`ymMe>QD#Jbz7TpuEfHwqinj5Ky4O)XAmaN z(xr^*L-)rbjE9hd)WzJ8zqT^R_tds6TvTUjD6x$~{bLzUA+buD2pCpmb(2S6JMf3q zoE|6W0V+fA%NR@T+|O~PxO^_@DO==VAL0dk^8D z6r|ev8l1RuVYRJVZ(D_WF)|1+_>X;!G8)QQJKz4S;ifi~j2iIE{F&DLiX5peb!K}m zg;S#X=5+;fWc$QZ|6STPtnSm|A6YRz!^vZy$e_@&5ZrKu|7#R$}t+R1`oq~)GIrmuuiPbq^(w|{$^i^AaXZn&ma<>JEx zT~3YYk_Z}Pu5T{Kj3$nY28rOb^G# zb%iEfL$HtTDRQ2&)Tw2A2{T^o{uatFDzq=H6V_zTO~dK66*e)iT)1Yldj2<^!c}5y zVoA?Q$21My&)j^QT*T(BVt+UM%@o)EY%fhciAfrJ-4rXA+PL;U=pKtE&L$r)CDR>s z>H~MT9X>C9mr^E3=B`UhxiJQ!b)YY=N~1t^-m$DnBK9euL*x5q3# z;Jt5nx2_J8vO~zGib(%vWcoJ-nSGprTvt(jPMh&Z7+_@{4Df-Shs~+Rg=`ONP8Np; zoqJg9RzwcD+WGvzOok0wg}n@lq>m+*EV9;~Lp{wDlrZIzcU)%d`t=^kU`PfatEL&v zcpuvyvGqISnZ?;$rSyL+N}1E{tKdBoP)DHE8Cs-9pckNEZBshl$+OPfSg((cKWYP* zl}Bf{`b`J5bKgN#G;O<;*jDU-^x)j*dIhort!!*FuI?ibkr>H$V)Q4;qS5o=b}}0C z3NV4=WaP{6U(Trf(NgYHK@L5yWPWa%;oSKyJ+wH8oX+9pZ#ui}8{{8PoiLGsB*1cE zcNB9cU?%T*YT17;DhXZPCc2ChYlbqbed7#@5Bv$mm4tFpRvd&KJmml}y@=&2Wa<9d z2|I*;gF&N5R4Wl%YU<(gQ|@eZ!{@{DK$`Ndh&+BCtA=pxC<1bjtk@MjEXAG=M}b0m77RcuG8lB$utK7kCsYvc z?advP!Z>*&_XNoY$5BF@22m$=kTJC;%H%TMZN6f=yQ~XJ_LJCbDoXb%gBgM=F3UMmi+eq)}9fm#a-kbI-kIKhNaZVFvhUOscQn zmF-GEKT7JJ-+`is%W=nY-TUTdfxoM}A5Urd8(SsvB~ zCq#X)>MY4o`_#^4@d%{WSt-QPSnYf=*veazSuapg*RDP@S0bHARn^n9Z>e^rivH7 zmQE`0{ViRk?P_VkG^|0q#b0by zzTPe(eFEcpLMsz=2zq8h@-ouDcH^RbG=M2$BT8rQB=ZjoC<6k%Hgr14eXxSV~ zidTaf4WSR|QN_nqaZb75<>vW^#_pe@0s9dL(Y*pVz9~XT+c;D9Vg`$|-}hv{7~Yco z<-o8^-^|0|CqoESeqVMP*`xY`l4lXHeE!N1NkvJw|MHd5y17hmP@uXksuMcHGnmMW zB15STtF*)q_DDdF?CD@cSwPRK{9*V)*4fNot&t}t)v50}m7xSAPd#4A>YN&WEZpsc zVHh88w{=eODFa!EF$mzq`m97WEx^QxG_GJoFSKhA%=xg&nm9E>b=2ImD05qKO8hkk zoT74@bxPWrna(_Z8mO&aSeJ+I6MdD z80lD5=UALV?-jkhqNAib?B|_GBG+n|yMNxPdHRMsZ1GJ8`G#1dPf1wLdCQ`jNyW(S zTJPq@4)%$eg{VhXrLG3i@#B_ZE7!P|AxBL|jU)3#@YoA~Z{>vtfgVd@++j8d8A9x? z<-~l=U8Qf~5G2WN($D0`4>YAKKAySYk+ltzDxwwEY1d^iHl zlz&g1AH0t!lh-xAL>>crSwM z&(8^M6yks?fEuE&ai?bPxWtY$kk(Q9MS7(U&@Xbj=z^XE2`@X@`5xieFj8?gkn!4; zYO|n-BbOg5YIyU4#+0?C`{yCaqJ4fik`+dR?kJbLx3K(k@b6OgRFEbjDMQH#7YKHR zwejYj;PN<<{8=0VB7HnxmyqFD^pGb86fTUJ++uEe0Ewyr;F2Z7I3)Zr7YLQOzLHVb zFzhQj_E+>6knx>o13tEdh7^`Ba`jhRQAtbTQAKT7SI9@fkdKIque<^Ra>$c2f^^hj zIbLdK@&{ar?-vIf;mb)6_AAc7D#wNZEO`+4V8f>R_8ZJZne80RxA@6M6$n;}GKXR4 z^BXJ68U-UZ+5D>&`fTw0aMxKnu%!rrT+n7}pbYEs4xOv0tXm8)v;|C1V>|vlwWvrH zZnf=%;)rXz@7_K9s3Zw&cU+lHRQiSbO6H|C47)cGt~NLE=?s4 z9RuNr{lProKc4>L&=5{H_Rm^Af4rTR2J(@JRA(#1={WCCpVTm%8oIV#<5rju-_!OW zNh~eNQla5^WXg$slZ7<1M7)6$^dQ+k}AoQ6bK3nvnhd17!x z8hAURbY*&6%4&rCTPOz$op$~v#Mu8{6$M7ahfPo7yS~EstqS>%zlVLvb$*gyRddg) zK^n~aT}jk2dXnE$KUhoZq0S!vT_0Bxg&APxdcHpc+p9;>=;ZM7&qkElIf*=eb9MCc zJ|6fWXfoN7y7qV%zOf3c?|mH6!kfTT$kayjIsL0VZr5$#LLKBzG4Y$^qW#^dKFn8> z2{lx56IX-TxcZBh1qM4J)|iS(!}j+Fez{rH=(=*EgXH04eb#pH(xQAeLk zS+d?f)By?Y#Q2>HhKZH|b78iS6ygww~GYvk{x7v{($H34<--z6f)r9b;1hHt~dmRxv7V zo!+didO^2C%`Qn@bEL1I>G|?oTqH6XU|%V_M+PDvzC8Ey82xuQT{2Lfb2(@TZ|=NJ2tTxSih zvttlUxQn&dSNA??s2h=UEfNfn;Gq6L0U}vY+$N02x(AZ*n#CWphVkQ4BpgFwe|O}s zYqbPxH)1rR*!JW>>|9KUJq%u? z3z+yjF57EQTIz34IIcVf!mIvx$ooim$$R!t38+t$0b;r@-Qj}oC`sj@QSS>@*I>x0 zHxv?pd}-vGu%3FrMY?C2;p?cYeRfuC+2Sry!C{iT;kFI0AGX{KuPZIc%dSv;O);b2 z6basHmjCBl%)l<_p+C*2omLIL%4PUAcg$r|c;qq52R7OXU#j&RBf0@NztVy1&Pc0M0yWae7<;Cl`KH zVOASgTer6Ei+_{o$_yu7|B)c;`fk}94Ka-~1)d)o?^#yPsT^j&u?J}PS1|`RK;qD9 z#G0M8%cnrq)p8|4QHI|Fpp>8)LNgdOlza~AGqWGIPl-RsblY?RHid1H6GB z@oN8jq9SzTs{jnpiq{*VKuiDc3{zO5|UkM|98d7R2LiadSf&vy*cox|E>m# zfc;PZyGly}eCX?^1B8hHk@>$Xv(JDy{r|~BJ-NvOI9uy3oynJ2sGS_N5+MyyJ^qhP*A{-&Go(l5Id%6>ct&2&2=D{c06!<*Gr5>pe)_NVF4`6U_JitBv! z)q{n#zW+=eA0tbcD2+X1Uv{L$>p?u3>px*uYFEAXkU||7arjBI+1Sv%_sL$^%m3ag z4*k3HY0C3ygh!V4Ztl(gOm_2JkiDqL*jB{9X7Uw2$gzwx~MCPO^Vcd5i+~x8~3P`rg2UW!3mZbS2YPX*0Mj5pvN#PGnTVm> z0Pg_nz~F$^J^!P#xd!1>b%Y-@S%0P--}4;-1pw>hsc1o>`y)ICeAt9m+?SgS}pf+;R7k-E`zMJD9St#vi(5E-L@I-xjs*dFaRr>KYdR1$F zozI%BK2=rEzxDL1@RFqdv`EZoRn3Az$U;!;-X#P?MkrnOCdl=8 zb{cuMJLNtu@8zg{S;w*ViPzdn9za6Gi&*?PrH$NIX>+nd-^M||dudFb7Sc)DTY}a%RIXq$ZSyO{_ z^o0hFwOxkl_dEI(W2Jn1Tb$#b+N50GBa9L?NtlS0B1<@pig15EFgRdWVX;@&721`z zzpJY)M1dvqfZAfWn7FkTy+ey>aNqM4#ywXE|Cz3b(0LzgibBQnrN{aVN|MvI-l3~4 zB8nYfcitw~6_fl}BI>#^+BF&z;|;}w@~M!ruQzu*9z6>cZmvcrvG~RzM%uf<6-)m2G>f+_crc(63 z3}O%iKW$#w?X8>-(HVO&kGEJd?J#w!PI{}1_%zZ|i~PtOn_4v2 zpy2pM#&79g1{IN+06AW4rF(8+n3P`??*hj_n#X^?Ab#+>+GWzi#MI?V7#H7F-g&yy zcuMLeJ^D{HCs`OJ1GVfmw-xBNJgYpQ=P@Scstb}M+N<82|1gh-NJV;g0F0`d!rB$_ zp6T2+6KS{}-hq1V*TEt$dA~Krk^_DEQ98&De|Hs<8Yl1kmjX^1q2szo#QRSIXGwbf zTqO%s#N`b^6=;?<+$dk0I;g8wTBe&{_YafnS+YWrp798*0&O%uf-beG;*keFpB$;K zK*gr~s?l6BZooT}&PYGVK&YAf2O(gE>J!Uan@F#*d`b#UJp;{4e>1#)vZb_OO1dXa zpPt9+Xm2Aw-ObXfnwo~HxwJ472!gNd%$=K->qsc0`6XX}p+fZ_YgKqpY&eI)%b}c- zA;=EaP&}>GPKVD=p_b;ufD5vM+61Hf^jhxC%3f;F@uvI}?(_idc#|M z0G=%Xl?3kvf}|4bPd`jWY$kvOytb7EdE0GsVZ%Ruc$3Flas|85Npw0vdNakPJW#fs zhqVDGhuy>-fnaL!5ofu`0{2b?T8FIL_kpz7K^zpbMI!cXqhK0$c)>@qhOPOpgJ#iI zg73;(Fa>Ib4uWZa;HL89;@=>by10E3%zozW`0)F~!UdrjvBe}XyRxy9xoJ9u-S2xw zM-xb5ICoXmaM+q9NcYaodW!Aoy^fdcJ6BMe50?I2bvSv4*ePD$4Wy-*a^nottL4=G z!G5Wt=276ufz8ACJ-a|SZT<3pi2KU8sG_c4QcCGi>244Z>5`$lkyPpKZUJc!kZzC~ zVx&8y8)@lg=ey?st&{krXT*w_h6@3U>2}iz@8h@0DR-O}};- zP*>qw8?cG>*4H=nedQptG&j9@bbtS|EF13=&Dq#x=w_524K}cQo~IM5`K%Oc<@-I{ zv#-p>YH*%^1KDBh=e)*H!OhWuvnD3J4}vUFf3ol!7jz7t>RzW$be?}aqG$^ueI-ytw7gu!oh@ue6$|eO7HxL35g910=cwoYgAzOUhj*KMSQLJ1=g`$t-?4S-)|-Iao?CeBbW=2!5}!?1 zex;eAy55$<@hBF@Q!{;^63c`b3mG*U>2c24ZDT{i>v*mrY=9-3yRw?c_7Qnntw-Wf z^!QAI9}#ryw z6A)#*b>)=c`h8szCYV!#Rr<`c#lzaOHX>=L-gs%8%PpIi&u_GxLw4^1%LZ zrMtsYW**FHEmiDZsTVg=!^`KyZ2cj|Y13xiJIV~0wO==HTe}?jTi^3R(0`JkLPHC# ze{$+K{?19_aInZUMf;r27+|!Mh1BuU?4_dv)1iNAhR&tY@vnR_R0cZ=p5r2RtHIcb z)Y#91OH;;LXB<+yVMBIcd@?t`324DZBynuErmr19Tz3L_0asrw8f5JFMlF&Q8$BGI zk9ws{sJX=$m4T%F?EvfjdXolU_RquDO15nAy9nn{vlR?Yt$jTHy7QQ(fPjB1seN<< z2~EWS6h}f$hJd?n89J1D3HJ-H6eVPl7=2j5;sbg-DMVK|9=%dAy0uh@&SDE{&4&hx z4#v__n4z@`gIl-COjsJG1S#7_Q)X$^aL)Ewy5zu0?e7R)tW?XZk}sB@w5uYpUywz- zQ8BXQF*usuZQX!tU7iCU5$AaK#6j;PwrALMq;T}4rI%tNs?_rjiCNqsl)-pJ({q(Y zqAnX-?TPo$lX>l0#hvVIf4V|@BQvR_N=hO$EC14V12?IL$mHXfo0X;3d=C^O;t8oJ z;4GGoMS1FpDV^ny2%K&-0q}jvGkFoEBWswC{~BH9B6nKmj{14wuKY*SQ#MqzWg?g^iuOEQY}yWi)U&r#I}5cP zoH)$!$UjcUY$PA^_JFO&y!^}VM` z010FL;+>u2@U3T0mUYFgqQkr1;bGud;+W(p4wGEeyNPT?!|HP`^K<{D48VXtMr7&?AEiXY)# z&bGw-ShBo6T;7_-#m@T9$wu(kYDUM}Ph)M~|B?~${o}y}$)$}B&Iew%8bI{IpsoJK z^g=I8uu-37ETTLhrMdcCQSPwr{jv{VD}Mc#Sn(eOBAUwy&`AFU3kdl17fi9eHUuEy z1W*1^(d@@V$*<>XGnWRXrpW1Y2WsvvxU<1EDn{#+WLqG&O!#ol)J;WjzrCQ2X5~%S z!|EjJrxOO6?9QHwJ<;L^e#V)3*bs8D9i%J{oqOkgv<(%qWsGCy|90*+F|rbWU6|;m z_|64ps{k4)`k*#jw%cyn55OJ~rV$`g;?%}7-JHZpfkMr8&%G+qmV;vr&&YA7tZbj5 z2B}q7kuP0+(^(<{z^ti~iq+*_%()&{**5V8PE%%v1(5^X>DR7X zZSU!?H^K>aKi)ZPxxbsAhh-4;_f&+fvkKwNXnaBD1OWiW4b;5UH!}ooL#IO_Xy?xQxijD~W53v(6SlKOu+B_53AN zl(>bYv(K$i^(2x&a-_Ra^m#yoB)&2gQeIZ6Bp#nb9nj`4(1XdeDN8nCCMjHpw!kt7 z#O;kHwi^reRsAp)eM4?D5!q)xv&ZfWlj zby{5V16i}P3CFAYYTXoe%pQqv?MxoW(wev5bN*`c6ZKerHd(Y>0*_1}!05MuYoJ7JN`<~UB1UP!*t2r50Ju!!bjw`y=Wfi1UkjF-xja8~&G zx%8b2+`H?V;l(~M2e?arc{Upy6^Ln>(~Nb0c9yKYK;Fo-r$&t7beA2#s) z(0PwDknqDp)*urwh=QF(+WK5xXBlmbMHB{nK2Ii40tci^k=FWJ0XIM z4zj!`(y@nf@^2rA8=cAjMZ}+^4YZEv*<}ueQait)fv1QG6}-!Cynvcfs1LtpkKfHT z$_cplrrLQa8epGe2N+NmP4>6z)!cFf4aZszt7D6v{2MEUUH1hZW+H!o{o66;ctOm! zt!uJSz0}$z7HAOdc$3hSrsN%<32x;Cw?@&1*f8P~Fsw})n&+p_g*WOM?COwdrYDR>iX ze{_6!rxew@B%w<)A(55X;AQ&yumv({*1M3f?83%prD*w2lpoG!e| zAylPL(unW8ozM7H>Y|`c3fQ5f@gsTN60z$1TEa*3lhB2Uwp}zTWU{Sh`e*G^y2(Vw z=m?Hr#XHpG^+fazNBFNj|rqsDm;@NWo8N#MdB}G z2QECbAKb{a+Kh8OA`1!-6zCL%ek~_@UB_$iT{%1Yqc+#;}zQ%_PS5#uK4{KT-NGmqhBtug9tx!f<>m9PZkB8_lv&R z74}qf^-ar=LwYS75uXGgt2*K{(1YF9t}D#Aur9MXgc}`{%@xQu=*A;1i~QgH{{CgA zk`yt{3zZ5y3m+5U6F=B?)hk;q?@hdgF052Iey2hf@1RunpyPvBGB^f_lZ4)ot;31O zM<0A_dcG<~sbA9s4q&()QC!J6eIPPkFd-c~dxcwj-)vYaI7hAy`yrT){WeR-7keNH z(_?#l@8(3@=nH}y1$tr9llfh_!!emispU6J6 zMxZ3W;lX>P|4r~4ln?l9wRY49J*6&3&hl}{#kk`wkO*mIpu`&ybguK9p3z(+9i}$% z7Vn;^8NtatcZD~5s0O-ahuta)IjK&L8%Ry2%_pT@-5#fOFa>B04g9peBq7m&8@2GQv8vsa^h!M;IPdHgfxy2TYfm(Eg>WJPz~NEGfv}|R35xf3 z4b$b_)_gRrEy-yiwZb_Msa>pAv`pe*M+IokHJ55drn9}#`WX_mY=0FHTQe;1w?}oG z%F8@E1*63RrDH3|XdmIoruoi$tN{Wh#)gkG%;YRok?yw{57At9Ir$D)+>-=%K}%yc z6MTG7WsWhtMw;xit}l;uo6}5ooGXc#FsUrLiBZuc6-41M98eQbj~F2wvlETqyEnnB z*dbPPNysR~hM40}=YzksghgCs0AA`zUIEJ9Oi#+{`X`!+WUnn+MI?^lQP$>G=cC%C zSRQNrb)s(%9iHExuMw17)mjrlzZJT`?7>U!^8X5$flvkk3+D;Tpp%GGSg(u9Dgy3u z!&@i$jfno#dZuT+16AZ`eqS!k@>hRdSrbv>CrD*DzMbj#VOz)`KgA|kWvs7 zf?U3m+wFZT_XzSRSOk1odL-C6r)2&Z+zzzm( zm(o9f)8;X5_=NV=DnZ6a4TJ%+aK-rAQHE&&E$PRa?V0j}>CDEWvt&>NX;cqXIKjWO zq3LV_zx!-Ul&rV-Sjp{(60ja!yc&luO33z_(>3nMFac1oi2j0`1ob@9{)KJ zq?CstkCq+J)IKi=_m#SnXbuI1M=lnU2uE&N&c6+6ex}dT8)=RNo+7w{sFNiU6Vq^; zH*yekU?KvKr8tgLQzn@>VauL(a4__fh7xH-mp)jYW%|QoA7|c7)k{6?PJ4QOHAbMh z8p+U_5~c`h-H5_#L)a#Ph)y@miVzVU__RDp=H@`H*6PBlBlTH70N|tIQ|kzTkJx9i ztNSxx%YgSQr_U4kSW_%xTfB#vw+;2+Fz?Um6Cf>Vfq_>F<~dKFK3ku~hYPdq`bE*! z5EOj@1eQ}g-#jO+*o+9uk29IIxCl{)&tKP>_lH!1QqG~~BKkTxfYU$_)I@V%_TcsD zhC%R!i6Be=8L@YE%O8^b3C;(oAxxP>NeM=vy3|wygQdg9v+n)l2-nSW`8=BxNHp0b zqTR>bdi#Bt?>&IrZl^)XSF0({&OtKZ_u#L^G+)!3yl@+F3ze~qT*5$!Tgu`h4gCFt zyIVN04X~aB5k0eHZD)t5jgdxdG$-TC)uw+T82}^qcZU3E!Z)YeqQ72lbupI+BP?Z! z@g1b_-k<@-Xi~yYYJ^`YkU@xrba$$xn&~x1(VN!l=~`p-%n(6nyQ^`|Gk>pvI;F!``YoCuad199X z(SMCi{1@V$`XZ(7v<;C)%s+})AbGw>xT&>XFE`jxPK8&~;`85z(E+-i{dq!zumD8* zYTj?9tgXWlOjCp6p93hwWowyA~+=;R7a+r0G7G&e&?S3(eV~D5u3Cb^Zb zVb!j@8RZ6%vugFEbdu1bVwfX`n0$YeT9!*_l8i<>q5YlLr=759b$ zH5+Ql*9<%idz#-7jDLAtizFVZ0%#iXUBmYG@G~!wGd*t3O(Cv+Eu)6j;~T zW<(U&uNpAfhjLQ5Ak6(zMYppv4e+|AM>z6yQzmnI7j7u`9JA2Ys%amc#?~D{8|c<= z)Q1fL1sl!rFL?!J762!5npCn1eh{{B9!FYw+3TVM<$$B*oO953wVzN><%#&asU_HX+sp7GuF@$QaW3n4WDx>ufK zTL3QLyKA1RFHx^ybOrO?+!Yi5a!srhK}2tH^Mmg>0xA;&S{UL3ZgwH-&(Lr&rb|3Q zUAuF&DFfO~Q8QEvf3^S`REmm-(zVX9FC~GC>%e2N!ld0m5CZbE)s2&XtR&=c?<14) z^WZG@fP9zu*d5st^S9lZF}LV;{8sP0-t#L`#7-_>iEFt91KOHfWP%xcj6*4DSk*Vz zfi1)D-a_u>RH~%p^5h+jH$F>FaOK*samsJyreVO0A1RlVKd{Mc5n-=jS28t} zd{xMZmP22cmGJ*%xsCc4$1t8bhRiJ53K0M>?%sk^BJ1b?oFbiR>p@@rwgTu4!$>Jq zd9#1uQmsudR6Yy9@m5ZEjOLZ1&~Els^e75w^LkuYf56j|4bd?Be?ll0D&{B$Z&7Zo zsyAhcf!!4~wRoC4E=}dkQ*8M7{yB=m>f{+z{%^d6RRBQae?u#JMuFY+pR0;X{~s^D zoA-|W-&g>X|MxFRU#sX01Q73Ft|F7_aS{_npD(~sh7vBzjiKSYr2xI^&3ApS!g;?> z>P`CrUR3=C4u*;xUCW`4M%CTGEj9%tL`pEhT-=bT$b1#BW=sH{Feob^1-bq-_-tEk z0su{HhF!4l$Rg7OFkfIJcmc-}1jWbe%LRdBNI~>L;@;Jpz(ZvDE4m0$>;RYl`vl_* zm^IX}Ir(H%&KiKtVj0fwXlVIf+ggsb8pSPYw6)zEty1_~^H|0*uW{p&FRq?`jf= zfS8{5eZXq?T5D#o`u?-#=ygd3B!;UG+GU?UAJ_$ zbwV)mdBA(6Ke7Rwp*;KZS>|S+W4P>+o@vR|o}WuEe||o~I>oPI@IO>0{^w3?HH7!m z8(kAc0$$+)tN%OX*BR?K?H9AJ2_Vfw?X#x;y}>z;_ zn^SCl#+Q0k*Q-kYcB4L0;ZC`4$G|B z3|oBLiY>5#RBZBZ1utH8n!MdVx_}tKS1p&3A*AtsN#Q)|y;#_K+V4C};7_G!*b4AB zr~cQGYqV7;1Z1Km9sKDe_zq)hW(RzI&Wc1p%xw=Jj*Fj>SI5IWoV=arwWIDosP%Q( zA48{@Rm`#M8Mv;~$47%akDLRa7&aKVb>Dqfnemq)zpZ;w@TT9cT%)7X3Jfl1U%R8a z-T5z7)R4yP^MQ{T=J`8IY)5uSnJmAMG6Yasz+G49`aqBj0)H$Sp2OiQ=2MuE4Wl1! zybXe|<)lSOI7KWbxT?RwetMp9DEv?+kgPLoAA0rj*i+yMO`=A<^_=oOeY<36z?hYY zx?_!ab^F;S0ooCVXds9~3f2s(4iilk_~a;x&sb}={d4q$o=pE6+hWxx0R_9BI4t_UR~fNj$K( z3XfhDZX1e8X zNh3s_lnJJnrLT6GHwMZ#e)`^aAbp>Mw=JlXSr%j+0ZO}Vm+4)7pr2@^oI<763A7UPVu_^Yj5`beZ}Ztd#BU3+S)t3W9^v=s+<1v`+aUltv|${70Msc zwjqzchOuxGqo$KL#_~)R?H#SCi12vK4oO$T>&PpY^tZ+`JihL$c@$&Ogw00LmUE3A zO`7~jU&_I;QkU6ZYIyp~)AcUBztZyBg)Y;^i+5A(2i*P@c;g`tzK92}(Bcl?w568p zxm;etnL@4?(Z5YJ3GD_ zszI~pW$twdGqvjI^6-MSQ36`#<|+~c=9^o-?sZfJz%@fxgWC&5EmS4_a4r?$c#eWJ ze0;T+rG+T=IJe)&#!~B#%A&t&88Quc{q3R4XP{qAK%Wxa&pgtU5>rE;Sx|#dL1;5k z5h#2;hV!)D%*S zM)&97grjZ09>g;YMkCiPa%RKjZq#bR2l1PsP2cS?f{IghMY}qCeE|ZWe#%v8}ARTwEx8Jz8|= z{$4%PJvKz%e19~qdIjhXJvr>ojBd2-^4JTkzbTKy-^P1LHC$e$P2K1_m%HG|c6_f) z#%RT9EmLhY89jb#iMq*wP&-bBblpN16c=vVhNC--ALs&EWpU9)swf@76+bcb_(?ilAYCkk6Wd22NIOh&?iXi$q@|y1}9`b+F zOScHKF@ip+omW&fFF0_B>AsF!TmQ=(VbeNBY-fK3$^3`i`ik(_uY=AE1~#1PqNlKD z47)T@*C7i~9w*v&jATLnwXt4-lzV#leRvbTXtsc(K$mWHo7rCMWz=Vk`+a80jAWn& z*I#*9w|S>;RLktnv^g5-Uen&excxv>YZ;NMA&TdAcjT$jfDSBEaVxQ2%np-RRMPLe zQX=r)(zYRNBDut;k;~5IM0yPRlVtdo9k~p=_0pU0@&g`}B_BWHG0dYGccwci0G3jr zuRl*g#>GT1(J#?{&1tN2F)Wic8*>VeL)Xc_PYsaz(^f^s<2s|C1L|04z7H{M!s*3R z*(l4ORHWUpaKlR3qm;^Q^=|%&^`HIYu1HZ*XqUPXdMwVV8| z%S9QRD_ump{&8PFwrv}QauMw_!QN_TQev#+u#l2XEQ%!MAoOe3^Xk6P)-nr$;}QAO z#~Mk|#jEN-UP!DB&ZbI~E7BnJs707uX-DY;)?~BS3R@`r);BLl=Lk3cKGN6~#HZK?)Ya)e;@yy#6x$64d%+gK<}>eJC<7-$o6164_tKRMgm8byFVB ztr9$vc3HU{uEf3@KI_6M&#W($<5R56s@gVL`!f*7#SZbFpL<86Q#L@lp_U|DGQ=2L zP#~g0tdXXuNf^*ryy6rp6KD)}KI;>QS3V{`_~g7CZ!dmb6-qBOmi+Ug^NmOtSz^_X1)i^}P60~=Ay?Tm zJdqjikqJM&ZDu+*5#$F}u{iD2OU62^@Vz|?s6UU!$WYFj(w|PLxROo>;_Pwg`31P0|-^seN zuMrg+O<4CeZ%6wPLWY<50AqUB2Xo~D7{i+ElxTPLyCzn}$*%$V4zSt-X!vP--t1cC zKdiW1x@cC)q_uD`!o-4;Nm~Ez1|I!gYbv(jkG(-68jj2NVv56ju$`&RSJ@Z*Fd@B^ z<7U7gYJ<(nueMKu&*x?hZ-mnXFI=>g z@!~F;LvMWbg}4YV`6a|rk3e7k$ijCI&HjwzyA8=EJfLWHe9`z^#hMo}T- zHBO6m=@Iezpe%CRAz7B}TmDeHHa*^}E6T>&&r|pD^H47q2(ruv%@@StKTt7EP}*Lr zG-%bHIU*Tg&0nlyCU_SjUM5B#BO>O{Mday~(ava4|F@hh#|0*r<3@tQWu18r7+EP|z=S|Nn zkrqRl{+5G;|y9fico$Xgz%~Br!|t77Z#<%0mxC+ zIxpZvWbuAH1A?0V?R<4MUsBGOb;1Wsz~>5|t!c&tT;2N$zfCLB2>$?jYU#P^*o@x3 zNwfq^StB+%P+`z8=E^5AZWx)$_SN^HPNbiq8aWD1!|u${FWON)#(Hy+;GJ?B^HHW! zE)$QOaiVZ|XmlZe^^R4-!rZxqBPf>7ALl}`4UAOXE@}@#Zb}$k9CfwcEy3B{QgcsR zf)ubmNc0{mU7kg{B1wJr7_C}t`QpM|^DCC_i;$(T%Jf1`G^>q*`|?^Z0|U9z>gt1FTTxHfudFHFJDZ-Ew`IYO(E6rf5Q zf!k;|I=x}MDD`HF3?SBWcWH#HdnIB6!lY=LK)IvcTn($;)Uu%~Z{KO9{tBwMNN4`Y$|J*XZzwW|J@8&kA%=(Vyi66cyt`)*$+cOl{^l{e^D^^c?35?Y ze%ESbe-L@YQUzgd0$U5z*&%$p`ks4}^yolj*#YrB-&xJw(J<>|0`SO{SPU4ue*PJr zbx_nSA0%!RSyK?uCSi8Ac5MlNm;*D1a%o-I*kMxRwE_J^QYyW-*UjNaTuQ*1@5>kC z!jI^4{l>NT0gyPqZSLln6f<%vuvJ+xFs=?_@| z^$*mw@cLa5mT#@EroAJ}_^X@D^{=bf?!UP_iqwXPA)Fk#wang~SjthY9qbRWeJ&=Q zZOM3(^HODpQH;in#NYDbhwL`r$Hkbd zH?_`3`kf&=nZ>kBuaCuE$@n9ke+%es7PR0A7MSM?T#6cCpC6ueD~VJ>^Y(KuE7_86 zz<8*wc>Y__AFNDQ)ddPW8TEts5wH109g03eN{amQSEN||e>br+;^8?%=W$G*y1{R1 z4ZobVZ-P(wiH6WP^cVBD{U2Fp+b~<-+61uA2#Lq4IaQ8~jzg|~8Gf8J^4F32S(S-d6;zsHwwX{AUAR|1_g{L1u2h_mqy}56dv8t#p z8>7C?GXd@?PCPiZ$fxKtTSy*w@12n$|BNw(54rwD4Nk78;3fTS*@e<@6MlG?OQ721 zxfZ#0zLV*2Ix-c9oAbcYoL!9`SDCIS+#;}9&$f&(*aA$h*Kq~{ZG9f#MXvDQD&`YoWuB-~vA z_YGSb4{A*NxUGqv_59CYs59sie9{5gNK@a(M}0lFxALVeZj{dX2eEV66Bm2`6x^A>83`vL!;!EpNS!x`3&sTd(3HQ5HvjI)1BNReq#h~Mh@o+qZa zq@Wi%S!kO1tdmy`j7BxSUvOG~)IhX#3=5px>!USk5sc_5hiamb z+u?M4uvLh^kZGs*2WcW(zPDIc)f&uWIP=dZmR)wLD~?6Yq#9hQwNILD_xHVVsckKTdVa8d8^9fGERenk2D%BqWT4OG4{zqt>^cz8 zt#asDL#vVQ;Hq$R;@;%G0^%g8^LZ5lo=WQ|17|e%y&L5JHId08i@*MG`De;SM`f{= zt~4ku{1x<_A3N3oB)b$az9<8&3u21V8{oE#IXHCx^r>_txBY({m(6xC>zRj z!s{=Brb~+WW))fW2v7J2Py41vad6TSop|K8wPq?sx24zWP5aCg*{XF=^lLkV0S~^e zbK(0vlOPB|0?IedNinYwe1devfU;`vG^V^Z2Fil4zY!J!V0I)5ycTu!Ht*q;dV1Dk zyQO!g%Co*|72Xu_wD%gF_z@q5l0VFt02=g9=Z|3a>QyX(x?%HrHJkGf4w5$hOb1(A zAd~ciU=#0-MCti;QRBu19;&V{Z_ooz=L8+T+kdNsT_4F!Rk!;rAJZPR2&MuLxjp{0 zzW?Vr&UctiAcUc*-%xg1cCEf>pr(~%9q$}R$2>2Q@&X4VzF7&CqiLEYA1^iSLw1tblIE~AO%UWP_q2|K)1b%KL;>4U<2C}*!z zqEgd2j2qv*JOm67^ln1j)Ql1rsbs`yJxpA7XI@U~T9@LgPCPZGP+&ROUyG1b?8S^n zWs^^`Zc6@4q*Y?{eQ@h*2z0*^jJC34v|XGNI&@V1NdMKla6$HPj?pjgK(NtT63=mZ5|jFWYuZA27b}} z#58H)JM~c^dq8P6Mcalp^Y<0BVmoFqBN9~`#Oubwt`q@_tg>n$e1Ys>%86A8X;W!N zE|FhUCYvd}K13zjsY{s;m-o)__7pa#;4*mR>3OZO_*bi}{hSkLg&Q#UzXm4JO1on1 z+~%MIX4vUfHbog>ESl&-;-sBhf}|aZ@7wl}k`s4KxGrx+$rxHhy_uemRH$Eey_%BY zf@YJw_g6z9sU9>AaG+FknCxlBIx3stc z$V*u^Yrttth>VIy4zk<#oZH%gHmqAq+C^x$c(SqtC~4D;p7lX9j2WA0vYqe$qN#|{ zTOp=-^YX@fzWj$~8a0sSb_oCA`2Lh~Ek{XMX43PEwC1ApX1iBy#c9_}igt;P-OyC6 z1$d=%8x=M9V$QbAmSs8T+~4sRJnHvmD-Id!Z>PA$qh31c`COcip4<_TR~MTYk~=N|6KVH$Bg@j-yzJ(GZFsJy9leXKU^pP^9H zrK_Zxpf~Q`R`$wxd~gU~DWceA1;RnBSB)2P8R+rB%~;?{*MN>#jwFR_3ay#*qD304wFaL^s0Zp^tw2vn5?09xCTAMLyg0T`fm<>#5%~jxMIl zi~y(?ZNZLrzC=Ie#t0*a49Jx*SDAkVl$j^uNDY4ZOU;Gn}7GKbY zi=@Q_wne1k=EHFC9KFzY{;Q}J)k4FLIgnf1&>#AoY(Wb|6*?|DASt0s3dW44qk*|l zDW;;;<&cjC@xaPV>Ty&QNjG@X(wakZP*<~~_l~(R6DRpx0oI;zB#zIfncRVdV9_uY zgLsdCCY!fYroD;i`OVbc`k^a>k-e#wA7cWd_U)9_nK0G`)Q|iLB~;|pnfg*Cqgjfz z7m4Q+%60McESTZnONN`RR}B~PcSL{KW1j+*1`WkXt;!5$FCAmraS$1ig7--kLEjMr zUKpv*7=!AH5g>?SIy2jDWKaZ|4@dNx9+OgGl!Lm0TJ|=nD!8%udRy6p({C4LBH!=r zjL7klnu5s;?_iy)*DHw&JcZWAmL6V|@xd@9-`VH^cUDT=-<*WUrf~KQ`_qy3;z*y< zoQGn_@jNS!-d2aj+}p+Fj9d6o)qo07n!97Bs74&8fU|fC*zfIR;?ZhY#0)2oipa)8 zfw#6@fM@Qnu+^u>)Yo?>M~UkSVz*v;=U($V?3Z^_HGS?HaBP1BGJ<8?M=#8Gp4xpn z8=#?JtbSJkdG90mw;~P?*hhlPEms#+7lMfXpq4-fnI}9iaResBZ1BWQ1E^xTOb*nC z?4iLmtj5E9i}$;@l8$D>a;>JZ0MZ?xeZ1aSg{IQsaS-&9#s00sD^mThe=Af@s7&;< z@L1XnI&Ne%ANY8cez(;|8(9$vtciK}Uis~}s|@ty+UkVYcH=NKEgJnZI!Ux7ur=Re z%oDvGwjdM8Rhe-gOxd=W^3I_w%F4J{X9O|gia=xkgj5 zRr#0LMe1a@#kejn%2_bwq%~)-T#bLDk)A2z$U-z2C6{Y1GOwp~qZ_IU z0MGh_Kq|FGw-yjcnn7SwzErypm+LA#67!mIyh?bO&Fj|=vYs^CO^_3;3i)_SG!#v`O&9o=PEhGg< zNGTTVf~t(3a9-yv?7a7|A^^W7jKAu%>_n|s<7)Y9OXH0A162_(-l`Kryd6AC)2Kzy zuK{Qp%`I91v_AbTLR1jYpw zOKpABGBlj$tB&5+9~EH~Ut3=ZOZur^$5wWMQ7WX#3s6Z%b0==EIkgRLrk#Mk5fs{# zW_i*_a-3eXzo;L9AI5@mpsIKyXAbo0${r(XBF2q%supy7`PVkPhvnaPE!bbSL2j=8 zi{gya(ln?#f+~_e%-0o&(t=S7F z^*XH8BnD83tnKWiY;+Ek%Mws}>d@sJ%e1A+8Uw|-(_jxzjRcvB3@|!^t>^LrgPAVJ zZR5A{=EqIYOlS|#0-?wiS+U4Gqwesb-1VJbX=wN%CryXqeOiL%Bk`%ABdTf{I1X%` z1~w`J^C6c7KJpNHkPeS|UrC1IG!QYtevSdXpTby$Kz&86MmdGD(QoT%&(<{m1c_|GvUW+|bylMuk~wkW(sC zbP(H2u_2;8guyPCP7hr;xh>FJrgz?0uL5>zyO6NGmk>{WiFM?sNL(mULc)<2p7++rsu}nh9+R-SKJKZ#m^ko@%E)g z<6HTi&@A3SOE+tugtcJ8h@r_1t<`NfZd}((3XBDdV+Inl&t8f$LH^0?VIbj zSx&+hUI)3Cr|#+4o&++)6wz=v0YVUO*^rTP3i)s<+mCH#2u{k;d-ncPh2FxebVfUS z?0+9^OcXmxN_AmxPtDaQpp47hUvk(&6EP}gCHduSOh!8=3SvXTm#ApwA_fX>Nhf)c zElRCj9KPYjX9XPurZHpHQ}!C$UeMxs>4Ms4GY(vbY?O^aAuRnQIz(s-Hln#rg1mQy zU{kt+WUjQefHS%Cjr}&7D9p%38?hg4PZAQTzm$c3dvA}-d?=oKJ>^h)MPs=kV6?iz zc@;m2DkUj56o}SiIQnRj+{Hz{yBFT$bxVsUvv%XJa`1U>@Ax%}uc4Mk&PG;J<4#>f zPBD*Tg}c_c?;ic%E$Ui4iRId5GfZ3%sEtRDsQF$T_zwmVpvO4pX13z6HdEU=Q^nIP ziq|`RAojyuG5@@H7IHOh+dD}bV}D5aa8--IRDB-thVz%btL=-xYk$#J-s}f(2QgABA=cTV92-$KEX3_L@Y`b8UMG+#u`M6QRe;MPau8 zXbbo53wF^@F%>mz7>uTHqtm=1*YeZWTHP8f z=n0FgQHv@1SL_-No3kIxT=JN3+igdGXS8aQ*r*h=JoTt1Qc-!|-+KP@eJCee&dR9) zu_SljVPNv8L0iPnE3lCP$Cx_52sd*&C9&)ZajsfMmJ#F+FVG$hChNLo{-9ZTa5|@9 zOM)MjoKPbt5DPWTDs*hNo6P@9-RUGJ<)q{4U%$asBoYZ`y3ixS)JfualaW&Rqq)#k zVnP+I>3x%;a1*mWP)xwxaih>b+n=#D_j8LJ>cRR0UE%H+5ftJl#`-5!`N~V?qgI^i zjIfr7yjDY5XxIk-Trr^%>!(!Xgrv{S66R5xUL(l$W*uscK3?<9DK)Q?xsIH_o!k7} zGFk-U_srYta(%v#Xo5rY_-+bvc;!x@-D`1;&z>4jT%Jkdxv%-7#gQFcG}i z5^y8Mv*X=h98zK6WP54GT2#sON!OEV!FwSP(?wfFfB9!cDiEM2$$^&8&@Y_#i`&(^ zeq%2(!Rt;aXv+G%&iWY9)ZB+!kLW>Bpvhbpo5)OyadZ9l%Lb_ zKfUU<4{T**d2;Q@@S-QZD|&Jp3lt)giBW5aG-M*nuF2S9S5R%ER6fr{Hjvtne%0-J z-5q(}td-lLz(Me3B@e4FW>8x(D?Zl+>)nUaT@y}<+b=#xqn)3DMsJ!aGjkFLjD_ea z*ge^PdkIx7{QMrl*vc&ak{PY@pRk5fIrK#A+!8UNNJi>TgUX~|FBXIt^ikpiv15=3 z5+XCy#^yfl*Lh@e^dk7UkrH2cFTdpD^OpQEKS4H;1fdiYHZZCXHJIgz#Uh7i^AU^KWpm0dNdk*+7c ztTlHkn)UGPtHvM1 zW15DD-*QJaVKSFUwH4?ztn++}Tdfj9Mlz+|MYbbSllVa~bUJplx-D!DPQwZ`EmRb; z9zHl$yX^b9k%-)U#)q88gZKRU*e66s{UMYUZ9ezp{DnVx=7B(aJCYgAa=%dxftX)c z_?@b16xce%=628l=BNt~tlht1C;}gv4Av_bB41=on$`=I6<63t_9L$%sPkpFQj@(} zrmCT5&CxHcQE-vZ&%0xB)+)pNQn7{7?sW4$rop8I=(abSd*x&F{ajZ5%nH$A^RZB& z?>ON)*IyN%ou5?Z1T{g!&q4D{xkTh736KOSdqAyB*Y3#w)6`W)MHQ}H8YHC~ln|s# zI;A^Dx}>GM89-5T5CLgWnn6No7#LKfQ@V%lPGK0n!@XDje-3* zKx49my^R`KwLTRdcr+3eU->N2gW+Pz4r8m#sC}nvVGRn{5`L8fx+*iP%|u;Iee-8S0ALiY$%qSG!6x$p`4OGCi}*XI+IPC3sd?!3WK3P>UJfL1v-8>b69 z%LqN?vS6~Y8v?Y6FCCDlLV<%{fcBUMrs1_tt3H1WaK=!MF6{o7e^NMbqBz{;&v^q8 z(~>@?X9M&xBsB;T^T~Nd?1mNM1==2r>^A0R_-t_TW2@hGV8r^Y02K?g7>B%>9h}U0 zt#f9i{Kf!^56YcW6FDzJ^?sx_6_Gtvs6P8v{<+L3yl^{ilG2rVe~(7X(Nwh%_YLqpIK~m$eUK$4kVBvV!m+YiBRN)o1~DoED?>wX#u!{9e9v=ci}10X`S(o^E~T z4M~0P=T7}yMS61ZqzepQ7>t&zIRE?+H}-3_qxrJf`DYsNc=3KR8>LE(m7UDy4^`qd zhm#%6jllbzgZ{{n^AQF6uz3Y)fb>?Vf6K*(r>wHDMTYKJSAwwiD;rij3`pYGN9 zrdi5-yo$5$K2Na?PJgqAHXUh8ap^Y?ThD5{L3yLlrPh0$j`ri)AoBFsh)VyeuvvwI zcqPT;;NGDem33s1Sk6yMUS53S`NvYGwxv%D|B4!T3=O!q)^2_?Zn`{|1f1ZACNp5) zl|PRE9*f?haKtRo>3h{3wO}p;{laHp>SgC)wNLVZ7=JHW@c-bv_qxcXMdbe7H7hXV z>vtId^o7}+z07?ef{=gWbB(ESeLbhNo8rDx=aqWDBpMd!u+qjL@b|_4Dp2w_1-`3q z7$u?2YzblNKw(^4rxDC!SIMD;AEM zel%$`Q6f7e+?{#wzEDGATFZ3BlQ&InYMs>#Sg>XY@+$G9D^vJUr5)-hS1; zNG+W^tZWuP_~#Jhh>9GQ=C2a_&_p0U_T)wis=QMn4No{=g z_7&qRBwZ~@yTkX`AN4%?$_J&5TBh{I_KClJ)3B;fo@EXY}Shq1y4O zTLNxwhr%swIM~>9T~^%6Iv_X=m;I5c>%)la5~RJn!}H$8g-Lz|j%z#KZmUc-ie( zi+#^)*T%r9l-}Co!cp%a3!JpeKJ2vn6|V%*5MTGAmi<~tSEtu@A@Xjq*_Za0SX74z z+YH_U*t8{lqwmm2OBk;)F_ z*|DbdBj1urkgrcvpN~q75h4lF|NOQ)3FTom3@}xFZIdMp_eu`Z8+j?gDEZnlI+<$@y(EeB97lbHdI~}(uuAiCB8Q-|f$KMg?t8>8W ziGptX{IQvnZ;p9ho6k#qi`9&Zo+@s~3AwSnX8T6*4~ab2VUST1Oa0|TjQ-!y%lY|) zjrfw0yEMQCRo=j7ucJ-MYB;5w;qd3zXo^VU!}VG*ihLE5r$Ho>pWDyf2J&C6K%bg8k~RQS{gh{%W^Z zCn(E@#U;RR(3XINVM%SxYh=c6)GwD?2xiznKbXYrJl$+d0f&U`_vLk8Td#PK<=!>L6?`ki!f4Jw$SE zX08!$=Eue|^VOwKptw{R6hOJ+^UwLnzX=|yKHf95m4FBB|0wge?kZ1o^)k2VDx$Le zg$i)!T)*_|%^IZR+XGgvDCRncbzY!f;9zpFPfK}kF~s;r+C^FgvT0-_E;_gxdOelM zg`CRS*tZ6dd-5{S>xJtxFpZmrybP>fA-(VD4D`O1Mj7_;J#Tpdt~oA`w&rH|sz>l{ zys;8rn`$UHH}^UW5xN*RL^Cc&cEtkML_Udq$4zh3CpZAKx9m2UkNRwQ@?gH-3xCoV3}(7>2B)<4Cpxw<-i3vVr{c6QPC z%Jix$kH^Y;gMj3Bz||a3KiLgxAFFaiN8(14CnzcULI?K1pAR!vp{w$pvp3C^Lcd%X za82s`LBqT z1AP#bF}4Q{{;MIv@(bw%ZJ~^$OorlI(=XQNkhuEvsWBC0Cic@@G<~;pDJb| zUtde0eEw3rJ+$cQY2awhs`&k`#@++?l_PBRrQo9sQu(^r-p2y^`tscoL5Qw@a5y}G zrBjj&(+mJgHk4Z;qhhZ8^qURS9)Am?`GfLvA=~CJ9EjXKLNutTm^iER89a%VjE61S zJ3NaH^VP?qpXc|k!!|pI1{|xG(hh=KKeAd`|YKfkRrSs{PHIDG=q^QXd$*_l&@zO=^FK?97if-e@A zL~2@LDB{+WVqM1sVQ-^tGW}OrK5I_hba`9O93Kupk9k_M^`pl|go0)Vm`gkz690YU zv=j=UW)H$ZVEeR7wfQ6}o{^DUWyFWMKa|3sw*lXR*)wkq`6-cRLiJ_rYpibbN=io(vvH!5jbyh6OR3^Bw zy%9QyODAK@zcc(o=ubc+)0fiWff0~*1PM~_>u(B|q}K6-C#pq0hU)dl^c5GY+q|4| z{tGk7ZRCpHlkdh-oLAniIII008(qDei=n2Yg;E%!(>sW6L_bjK|N%4Xpz6!dl zI}nb3s|XqXCKn5B)4Tnky7|NCD>cy1rbt#$=Hl!eq~kR!8Ri5o`c|7fdhNG5_ci16 zs(9^hT-R|$r(&ZL(F^lGnjfK+5ZekpZ{LF`0fUBsGl62q>#6=jorQTgnxk*oq?u|p zKY0eGqx0`nqijtZQh5Bl0w46nN2c&z6RuvsZ$LXoZYx^*t+8;)!%Qo19Wyp)Z*C$; z8mkvYemR2uDPRmzb5=0q9N}}O=CE!5ZVyz_SXx+Z%s7`emQp{0#PwVXp{#8ZXILC3 z^kg6VmB4Tw${A>EJaJ#n-BWLd#kphWyDDV-)cBq3J&R~J;bRCy6InU5RZ(_mT8^5z zfLDUEEy{W|F_eSmA-GR0XwCS3aZcQ-oQnL-&kP#49%+$S(x9}MpiMf>=kOVfPm_5I zRAdzxWwE=H+@*oiN!)o}BIl}6Zzoowvv)4;xBCTobEqqeJ{@8G?yz@=yYv`Oxo}P6)83n97fxd_QD`G9Lcw?#)M~%a z2VKCC-cy79Cc2DX8m-j3)E^@V@frWr6p?I)q2TG~9vq&rnXGF@!zF*ALfTHPD+zmU zxMD2N=Pd&V_4H>Fv#HPppnHidZ4hn4f;=?mZIv|>WEZDj(v;Tn7q)9-E>=8H4+w*p zwl}5IO|;rOT>6aFwKc!0X%nir?rD@eqI&%sNz(-li*KjA zq}=gaThjT1$?U0LjBi4?kkrdZZ1%#z8|A_4&f?oc=wp-L$?Z1#cO&NS^Gm%MV;S1N z^O@#YB3o}_^tlEHH@mw;Sp?D22M3nbqjdxk*uL$mse2DB$9~rb$f-r3rVqZLa>Kh( zp{Sz}E>(n6-02>2P~UPGtKngl&V|(q{6H?=_GX@KMJx%@820KGcB zLX?SdD*3h#@dX3N=CcUq7e{Yl5l7AVEbMeo6u^yXRZ@gp?3kqK~Z+3iv&T2IWX`HPEL22XdKYY8C@Q5w9@eDZKNdG zQPHScvFD;_P9Ymzt0v;;duAvpLKl}=WLq^3m%C@$>a?2*ZuxTzBHkt|*SkXQZg{!BPpCq@n#xib;X5M4f6q zXDS%;RNMn@-Wn>DXtnbk1;t^{Ls-e*!k7at|FFcS@oXORQj zf-zQ$HORr_@b-5UiBtlAD%r9kL1glT`;9)ESy0}p1>x?~E{61{WNvxStmK?2Nf=4i1$h> zFCB_mCTCuk%y_#_o^j;wUq@qSAdMO6iIglxgh|nQ6kMd_3j(V|iA=Q#k!PV8U zL3hXb!)8JJ9Wy?%rL}OylZCUE(7$^sZTe_ESPMOXl40+h{>+qZ_Xu-E{R9vX+%5kw z0h50BtiBm49(l};<|5I730-OeT^z16r=7?@X|RxSqLNn!ZJz=QbF_Loc|1!KWm2qB zlAL^cOPmQamY1}r?Bj%
pM=}V9&=8phR(j^(Nq=0@5^JM;2zZ8=d1mD=5DOaqp z5oVj&$)oUFGH5jGjx;$*jZl6!gqTVG>U~KUxvny4`jUss^4GuYc7deP)952x?R7n{ z!V=}5PmlgQYvn6Q;09u;YQP=*HOndLpntbthc}*ptFrX3GTuh`j(+@U-t^w(^Uqyz zSJ#2*dP`|nE&Lcbei*iW2r!=$%tTEM#th|=V>l-|==a@W08wQTeR!@^dI9$jK!Hzn zJqsz)nmWo8$jItHWb2LbaTKfE}hPE;nh9G?L>Jn5Rt^Wo4*eC&~ z^>+WQuG4B&jGbljswm~q(8w!5u_(EOoLM+^$XjGDCZLhEg?i3sdj3hg;rKN+8gdc% zRIEF!+l`LNL;|2W7~jcQf#EiE$DwL$uD07oABUpz^;+%StOlIiAx$TK?@~H*f<2FB zT~sl~ZYhFG{8J*B*Uz#)E&Mz==S7RMpsaqiDk0*TBQyD~cf{3k;*OnxC=Um4ev&hh zI4coD%VqF<+Mm1O%!xo63je^+z?&LAiLjNnVapLwDtjET=; z^%}QyDJNH7m)!LGW`3_hOjKV#F<3;%ou`TF?o9r+Zm)5#D-41_D6orjSZNpQO9aJL zZ34Y>Ys3ywQ*pnSoMeeZ%Wo7dR{BR-1)au?>jdc#!Nbx@(jyy%UiDvI3jmGcP7KPi zhGayG-(QFSp1pe!IOUmHq+{&I9iN$(9GQRjLC??&26;YO>aiVvwz&o8d7cX$BOk`iw}E|dWKsG zyn;V$aXeWW@Kmf^>$v0BqA4PY~9%%^I?ydie36&03 zSTyJLFO(!B^>QBXdo1277lmr8dChGqj?!G!Ltn50^b!ANE)m_dven*P2KR`?-+6HY z++P&=uy6dKFV!7>PAh+vs;eqQKaBq5%=1K%V9>$M z5DUlKNL+MHT-P~Jcptqvv1J-K3?d2E=TxpYLN>hg45_wYI)8oQNTl`BkE&qvNyMc7 zRdLtwML4sa@?B0DALSLC9{*5(2Ju4bpwz|P&rS?+6dmrMTf+hA_`>9FAe|884%F)f z@J+c?Pksif9$M-|@UUzW^7ARDYGh^^7ujY#e~mnCAY$p} zEDpfg1P~!m3>7??MJ#eu_6Dou4Yc+;+RBTC9f{oEi5uL1WqcXLQ6abiPM1}_{s1~? zYEm1P{%8u&l;4O|72B=ozU(3&jb#`PF;sG$l`KGExDZnas+{JhfmAQvbu zrpljm?VR^n^;P`~B^_|7JC57j^gnRWO_^Oy+qbHHd@b`f1;J0#L`)Xek_0r?y z9Ui62qr+Pd!3r)gJV@FpW8#!_XsB{tQ*cktJH17Iv3n(IuR%*Di`QBk0*HFi|MEGu zpwnkFgAluobANs+US2x6^|z%(j1JbMPInw{eU3bLIF&Xb&(Cp4B&?2A-gY#4D;>Ul zm;}4~u&nbNTx?2=KZQPr@n_SejB$jnQP{U%u1w3%!>>N)bcp(R=Jj&78U zO!`sj?foYcfo2+U3mJ%|tes^JzmeUbld6C4q67QbBakz!O95lZBkWKA&Yl%kdA0L( zUvdAVz{((U8D~Fst2!8uRKcSZ=m{>Kk8D5V05=_zZrCecefyy$*4>(s3I8`-yAkt~ zjSm7!uKpnAQs01UZhH0519SMtn;9eA*e9C9bgGmw_pL?IHiiD3dNUNkY7+z1q7>Ig zXv7ToT3jWwgZ>#qw?q@f#qcCe#&C_(rfYW3F}$xrSA#Mdo{#uKa6X-mklf-qt&MrZ znlu63P+tbi_6GXn=E`QCBFP>0jPXd$@yk4Yqnp$R$$Jh+pvIejw{I81)%(AYvszJm z`9L5Eum!a<9-G86ANv7q%{Oklf=p;-TR>f@xf=NVV7NTajppjyQ>lRKfjl1TWUvX= zRt}%b+xk8Bt5Tun@g(geB@uYLY36|%AYLY45##RPW74dW^RH-L8N*HhNq|@RNw2m@EB{ z@o+KS+zA(`CUU3=d_Hn5yUmvK%|5ImLFiZdX_554rP8KL-k;%G zKL8X8vb4IA7dP{w>~%Bph<2Z!k-hChz;bXsU66Vp>R3sVc()w0lOr3wk?mD>E3~=d zXi9&xjV#^b?_u@^8_g;C+avl=*J-hZ{KI#DK!2hl3n+nfy|pGkL5L_t73th&(y)t% z#|sDh2g>2!cWus7wg^%#xViq=W$B+}ccAK8A4K1MFBSqna&hZEPj4PDqn4N)zfQ1p zISPu}qj(SH5FU4Lrm5y?rdq#$B-GwoD0`4$QOxD{AtX`dd@1|%eg9Giu3B-UKRy+w zn*qnvrmgbM{@>iHBK$)v34KK6iw%Dg`_)!JDv-t`=wr6lR{w7_9SoQ)|Ek&@AG8ca%dd2^Nk4CJUs5Kc~ zut~j?P4~kQ2|$ro7syvO5HqtIPo;?sX3-GcO~zE z%XvMwPe;E#F<|+3Cg0dRRL^MpbA*s|;wu=$PLj)d1h4Iwm-5OvSYmd$qu^m>KF&&` z+1UFl%fHfpVy$vbEYK97vPe+z0Co7W{-J9$;J3Xmijl+88{3@waYNs^Tm`}RsjkvDf|xxNCJMsDFSk*w>mq^FXzY=Vfo@vA_PA6vNnuSmi>q^A!gt z?L6M}P+wNUn@&*6lLn86QB=K$B5?3bqhCWG##6mJ5Ru-fxMZq!{N&H}Lk%RwIILT2 zpq-=h!t!%;hrn|)0p*Xv6zwkm&fjkt?8$}y>NUpJG10f0{x<<7d|ojqN1np}HsYv{ zV0Zrf7ss-dI8V9aeYL>T!;lX(xmNdP;6IKy#31*`k?yp{2V`oyy2w)7p`H!?T%Cm1 zgQ#&#j8th%=QfX`N3-8Xuv~aWB?Z5w3#}IphxWQFvr+(2DdV9ep0TB_MP7;b7P`7v zI$F>W(pn&P2{ZSGzH2e(KwO9yx1GwqnUng`K0HeN1{<4#Dd;q}H7P6hxq$@AL#Kg?cORQZo{9>(*L9XX5W_R&GnSp-#dA$$ z+7GH*Qn`APjJ#`bj*mV1Ck`7P#KA};bFZ+SiU^N0{cVKQmNSAG!%jK(Z~FLNn}|cP zAxlFL5?4O=+*c;aMc>j2jO(RZI-FtBG-mrN5`Z<0w=%_}%@L%eO4o*(hnGtl`yGob zo1YpsE`AGpSBz!`Qd#Mm(HjPhE!UYCm5Tn#|=Bcz`UBk?LM6qDQ~^qh^i z5qW#^%Nm1>U-J;x@H@UW~JC zF|>A64szo+uFwj-J}J{Va^LT^&tb7eyI?!$joJZ!;4yDu5|^I$rm<_)eM$&x^HxH1 zxb^%k00Oh-(oGP)l*e8GVpy!WDA>P*TUvrcFrqEKL8+T2Dw?l8Wh(A;Dxi1>g1@y> zq%UrG{wf()s3bFlnR%c3#7~?Nqs!Yvo>eJR8sC$EzqbfV@qJL|bp2cMj&jVBV&`m# z7s%hJd-71hSp*hy!reHCuD#HD;=r3Kv{kI%F}+~d%7Xqi=L9X~n&N3fKozlY79aDm&S(j;CE z8n;x*p-jf4tzJQI-KL>68K9x3JYCqhrU2kviXmZ)Xz8-^3x8?sWQFPImlL;$mPUNV zs&F$U`Q@&PA`cnDoCe=lwZX&jU&=l{W3$TGP!ravcb}5C6XIeZYDeGS$FU!oz*Yka z0gZQx@^CA|rBVJi0Xn-md{sq$bi1g}jOKfcAk ztusqL-XtceWy+>!GBr@wK5?hQ{P2|%?^J)jC`s^6u{EOzYLC!Lv?0L=fQltK<_^ z7Ks_D3G(+4wG+Wi1)|988jXdcrlag4eLwYsW0vEB&}xUKfD}7&=#4cp$qwCvGKI8r z_5X*qWgV~6I zyHdsH{gn$R+D8Uyu(c>LMSb3N0M~a_f|qw z7|m3b<+ssAQ>SI;R^&wPPv1xu8;OYbSaw=RmT_{)R{}QtHeaDAnKmFTX)vJTk+Z2@ zRVVI2y*U=Lu9AZ?MN!$lA{O?91x%U}E#h8Hos!y0U)#qAB+(|(c7h|xmDU;1NFQNl zpyfo?krh#QX98ydXlj9WtR24trS-wC43mtj5z-WC(iRP+VO>KpLnX*wL{~SBu#F{d zBCeGh%!-Ajt<>ep2|=K(8;Gt6Z`LU(>7aKIz~0^izb$q@r*sE=JpMDp5@2F#F~lEp8 zjize5KbQ*h40N$m{3AtP5am}vZgf$s)PiY*nGS7b^F0D=?tZX(uiaw^;ERJm8S%X~ zNTa!b@-a}A)VUGNZs_i#=UaWh3+P@*`sZcGE!`qtYvej^FGPWy{*J zl7;VabaG$k#cE&AX3wb&GUkrX_9@vruO^E$gqgg4+9W8I*hT)s{A8My*J$!K%JoF` z5%EuJsgwnjAF1io@D5AdnOhO`=-F`Ajf4avcztcUJ%Su~F9Hkim*bihK^Yk3yN(DR zWlv`~8jBFIICHa-()-w65mMTTfveXyDgJCXQ%W`k zUGNVW)~;oX(vl`&yeyDdi}!L1*LP{H?Z<)F?saW7f(70gXgw?UsQbtApqwb-lY&@F zc9stu4a2pxobZwjqKSn} z%NOavLPnXwiK1YS+%xM}viBtIqb5~%V`!E2mL)r=(cZW7mhlo3fRQYf?yy=~`8l_$ zD!s>q|FNb%@kQWvYO|awWxpU>UK!_kv}1vlh6dI$3}i*6C1!*&Hwez-lFl;UA3tGvuSx$!LnCrG8RRg{Iw%6MhbdE7$!d=DCqXWJci?LRoU+wqEf^Yb z`oPZUVwSL721p>Jj)N$m>tQ2RK;sTBt@ypzr;&;$SD9*5zYD_A3zm#gxHFm8f#hD; z$1g~ONx*DtVWj_XYH?BRV@5V3iLmG6HQ!jr0;$mKB9|+6la-4}@6|UgS;*A0+SP6L#qye^7#1Lm7Rx0|t1!y~X?Q;}%WHHpZ4<^5x@ znacoOoW%7NZ=Wk0m(u#nC&nPLxLyH-GQ#}2;LkML__3OwwHmb1vR1*Nj7qeS;O(Uj~7QOl<423ZNqan4wL2gzu_IRB!VD49EF70xohB#qz# zxzk3`q?FSWyP-WR-|Y_2vBjYQx(=pvu6;4L5L;QgThEZ#+0pQEyGo#)J&Z_>(k#K% z=;C`9{60qRTue?M_3Wdx%$g-qxu`qSLvPYhvOGWG-sXwKR|QG|S}^s@4$EmSw0eJl z26!}D3+8=(S)1&K0!b;$oBbhqiDO&-RiQf|@{~5s#Oqv*!(AFNMw|^Jkj^^~QT+QI zPL*PK;wv+vDXXT`k#MhcQLvaWm&w@8g@lAiKV9p7;3#lo3=pK6q zMOA+2Y5ySS%W7%Kic+MjWBi_1g&W!R%(SNB&D@_U^@I25u94#)|6qfEeZZ;C-r}z> z|7!;N0D|1C9WhBt{a8v@j>v*@{aOC&5(@qf=?ZwMMEq-{$s)~2l%bgb{PkiDMfc1R zy8G7j5dhqP`}PIg-i)Jt%>1!lvCF|&--qe*Dezd^UT%iHWT({cR_DK=EUD`M%X;<8 zaaRL2<_AyG9nt?KytV=vzW=o;&Gh(RLTu^${}N>X|E&l2n7Mn~2Nwjvcxb>!U0Fw| J`lU_u{{iME%Ax=O diff --git a/docs/src/assets/grid.svg b/docs/src/assets/grid.svg deleted file mode 100644 index 9e4be4ce2..000000000 --- a/docs/src/assets/grid.svg +++ /dev/null @@ -1,1126 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - pi,j - pi+1,j - pi+1,j+1 - pi,j+1 - ui-½,j+1 - ui-½,j - vi,j-½ - vi+1,j-½ - vi+1,j+½ - vi,j+½ - vi,j+¾ - vi+1,j+¾ - ui+½,j - ωi+½,j+½ - ωi+½,j-½ - ωi+¾,j-½ - ωi+¾,j+½ - ωi+¾,j+¾ - ωi+½,j+¾ - ωi-½,j+¾ - ωi-½,j+½ - ωi-½,j-½ - ui+½,j+1 - ui+¾,j+1 - ui+¾,j - xi-½ - yj-½ - xi+½ - yj+½ - xi+¾ - yj+¾ - - diff --git a/docs/src/equations/ns.md b/docs/src/equations/ns.md index 0a714c6e3..326dd13e4 100644 --- a/docs/src/equations/ns.md +++ b/docs/src/equations/ns.md @@ -1,100 +1,61 @@ # Incompressible Navier-Stokes equations -Let ``d \in \{2, 3\}`` denote the spatial dimension, and ``\Omega \subset -\mathbb{R}^d`` some spatial domain. The incompressible Navier-Stokes equations -on ``\Omega`` are comprised of a mass equation and ``d`` momentum equations. In -conservative form, they are given by +The incompressible Navier-Stokes equations describe conservation of mass and +conservation of momentum, which can be written as a divergence-free constraint +and an evolution equation: ```math -\begin{align*} -\nabla \cdot u & = 0, \\ -\frac{\partial u}{\partial t} + \nabla \cdot (u u^\mathsf{T}) & = -\nabla p + -\nu \nabla^2 u + f. -\end{align*} +\begin{aligned} + \nabla \cdot u & = 0, \\ + \frac{\partial u}{\partial t} + \nabla \cdot (u u^\mathsf{T}) + & = -\nabla p + \nu \nabla^2 u + f, +\end{aligned} ``` -where ``u = (u^1, \dots, u^d)`` is the velocity field, ``p`` is the +where ``\Omega \subset \mathbb{R}^d`` is the domain, ``d \in \{2, 3\}`` is the +spatial dimension, ``u = (u^1, \dots, u^d)`` is the velocity field, ``p`` is the pressure, ``\nu`` is the kinematic viscosity, and ``f = (f^1, \dots, f^d)`` is -the body force per unit of volume. In scalar notation, the equations can be -written as - -```math -\begin{align*} -\sum_{\beta = 1}^d \frac{\partial u^\beta}{\partial x^\beta} & = 0, \\ -\frac{\partial u^\alpha}{\partial t} + \sum_{\beta = 1}^d -\frac{\partial}{\partial x^\beta} (u^\alpha u^\beta) & = -\frac{\partial -p}{\partial x^\alpha} + \nu \sum_{\beta = 1}^d \frac{\partial^2 u^\alpha}{\partial -x^\beta \partial x^\beta} + f^\alpha. -\end{align*} -``` - +the body force per unit of volume. The velocity, pressure, and body force are +functions of the spatial coordinate ``x = (x^1, \dots, x^d)`` and time ``t``. +We assume that ``\Omega`` is a rectangular domain. ## Integral form -When discretizing the Navier-Stokes equations it can be useful to integrate the -equations over an arbitrary control volume ``\mathcal{O}``. Its boundary will -be denoted ``\partial \mathcal{O}``, with normal ``n`` and surface element -``\mathrm{d} \Gamma``. - -The mass equation in integral form is given by - -```math -\frac{1}{| \mathcal{O} |} -\int_{\partial \mathcal{O}} u \cdot n \, \mathrm{d} \Gamma = 0, -``` - -where we have used the divergence theorem to convert the volume integral to a -surface integral. Similarly, the momentum equations take the form +The integral form of the Navier-Stokes equations is used as starting point to +develop a spatial discretization: ```math -\frac{\mathrm{d}}{\mathrm{d} t} -\frac{1}{| \mathcal{O} |} -\int_\mathcal{O} u \, \mathrm{d} \Omega -= -\frac{1}{| \mathcal{O} |} -\int_{\partial \mathcal{O}} -\left( - u u^\mathsf{T} - P + \nu S \right) \cdot n -\, \mathrm{d} \Gamma + -\frac{1}{| \mathcal{O} |} -\int_\mathcal{O} f \mathrm{d} \Omega +\begin{aligned} + \frac{1}{|\mathcal{O}|} \int_{\partial \mathcal{O}} + u \cdot n \, \mathrm{d} \Gamma & = 0, \\ + \frac{\mathrm{d} }{\mathrm{d} t} \frac{1}{|\mathcal{O}|} + \int_\mathcal{O} u \, \mathrm{d} \Omega + & = \frac{1}{|\mathcal{O}|} \int_{\partial \mathcal{O}} + \left( - u u^\mathsf{T} - p I + \nu \nabla u \right) \cdot n \, + \mathrm{d} \Gamma + + \frac{1}{|\mathcal{O}|}\int_\mathcal{O} f \mathrm{d} \Omega, +\end{aligned} ``` -where ``P = p \mathrm{I}`` is the hydrostatic stress tensor -and ``S = \nabla u + (\nabla u)^\mathsf{T}`` is the strain-rate tensor. Here we -have once again used the divergence theorem. - -!!! note "Strain-rate tensor" - The second term in the strain rate tensor is optional, as - - ```math - \int_{\partial \mathcal{O}} (\nabla u)^\mathsf{T} \cdot n \, \mathrm{d} \Gamma = 0 - ``` - - due to the divergence freeness of ``u`` (mass equation). - +where ``\mathcal{O} \subset \Omega`` is an arbitrary control volume with boundary +``\partial \mathcal{O}``, normal ``n``, surface element ``\mathrm{d} \Gamma``, and +volume size ``|\mathcal{O}|``. We have divided by the control volume sizes in the +integral form. ## Boundary conditions -Consider a part ``\Gamma`` of the total boundary ``\partial \Omega``, with -normal ``n``. We allow for the following types of boundary conditions on -``\Gamma``: - -- Periodic boundary conditions: ``u(x) = u(x + \tau)`` and ``p(x) = p(x + \tau)`` for ``x \in - \Gamma``, where ``\tau`` is a constant translation to another part of the - boundary ``\partial \Omega``. This usually requires ``\Gamma`` and its - periodic counterpart ``\Gamma + \tau`` to be flat and rectangular (including - in this software suite). -- Dirichlet boundary conditions: ``u = u_\text{BC}`` on ``\Gamma``. A common - situation here is that of a sticky wall, with "no slip boundary conditions. - Then ``u_\text{BC} = 0``. -- Symmetric boundary conditions: Same velocity field at each side. This implies - zero Dirichlet conditions for the normal component (``n \cdot u = 0``), and - zero Neumann conditions for the parallel component: ``n \cdot \nabla (u - (n - \cdot u) n) = 0``. -- Pressure boundary conditions: The pressure is prescribed (``p = - p_\text{BC}``), while the velocity has zero Neumann conditions: - ``n \cdot \nabla u = 0``. - +The boundary conditions on a part of the boundary +``\Gamma \subset \partial \Omega`` are one or more of the following: + +- Dirichlet: ``u = u_\text{BC}`` on ``\Gamma`` for some ``u_\text{BC}``; +- Neumann: ``\nabla u \cdot n = 0`` on ``\Gamma``; +- Periodic: ``u(x) = u(x + \tau)`` and ``p(x) = p(x + \tau)`` + for ``x \in \Gamma``, where + ``\Gamma + \tau \subset \partial \Omega`` + is another part of the boundary and + ``\tau`` is a translation vector; +- Stress free: ``\sigma \cdot n = 0`` on ``\Gamma``, + where ``\sigma = \left(- p \mathrm{I} + 2 \nu S \right)``. ## Pressure equation diff --git a/docs/src/equations/spatial.md b/docs/src/equations/spatial.md index c420156b4..e98183f68 100644 --- a/docs/src/equations/spatial.md +++ b/docs/src/equations/spatial.md @@ -1,374 +1,104 @@ # Spatial Discretization -To discretize the incompressible Navier-Stokes equations, we will use finite -volumes on a staggered Cartesian grid, as proposed by Harlow and Welsh -[Harlow1965](@cite). We will use the notation of Sanderse [Sanderse2012](@cite) -[Sanderse2013](@cite) [Sanderse2014](@cite). - -Let ``d \in \{2, 3\}`` denote the spatial dimension (2D or 3D). We will make -use of the "Cartesian" index ``I = (i, j)`` in 2D or ``I = (i, j, k)`` in 3D, -with ``I(1) = i``, ``I(2) = j``, and ``I(3) = k``. Here, the indices ``I``, -``i``, ``j``, and ``k``, represent discrete degrees of freedom, and can take -integer or half values (e.g. ``3`` or ``5/2``). To specify a -spatial dimension, we will use the symbols ``(\alpha, \beta, \gamma) \in \{1, -\dots, d\}^3``. We will use the symbol ``\delta(\alpha) = (\delta_{\alpha -\beta})_{\beta = 1}^d \in \{0, 1\}^d`` to indicate a perturbation in the -direction ``\alpha``, where ``\delta_{\alpha \beta}`` is the Kronecker symbol. -The spatial variable is ``x = (x^1, \dots, x^d) \in \Omega \subset -\mathbb{R}^d``. - - -## Finite volumes - -We here assume that ``\Omega = \prod_{\alpha = 1}^d [0, L^\alpha]`` has the -shape of a box with side lengths ``L^\alpha > 0``. This allows for partitioning -``\Omega`` into the finite volumes +the ``d`` spatial dimensions are indexed by ``\alpha \in \{1, \dots, d\}``. The +``\alpha``-th unit vector is denoted ``e_\alpha = (e_{\alpha \beta})_{\beta = +1}^d``, where the Kronecker symbol ``e_{\alpha \beta}`` is ``1`` if ``\alpha = +\beta`` and ``0`` otherwise. We note ``h_\alpha = e_\alpha / 2``. The Cartesian +index ``I = (I_1, \dots, I_d)`` is used to avoid repeating terms and equations +``d`` times, where ``I_\alpha`` is a scalar index (typically one of ``i``, +``j``, and ``k`` in common notation). This notation is dimension-agnostic, since +we can write ``u_I`` instead of ``u_{i j}`` in 2D or ``u_{i j k}`` in 3D. In our +Julia implementation of the solver we use the same Cartesian notation +(`u[I]` instead of `u[i, j]` or `u[i, j, k]`). + +For the discretization scheme, we use a staggered Cartesian grid as proposed by +Harlow and Welch [Harlow1965](@cite). Consider a rectangular domain +``\Omega = \prod_{\alpha = 1}^d [a_\alpha, b_\alpha]``, where ``a_\alpha < +b_\alpha`` are the domain boundaries and ``\prod`` is a Cartesian product. Let +``\Omega = \bigcup_{I \in \mathcal{I}} \Omega_I`` be a partitioning of ``\Omega``, +where ``\mathcal{I} = \prod_{\alpha = 1}^d \{ \frac{1}{2}, 2 - \frac{1}{2}, +\dots, N_\alpha - \frac{1}{2} \}`` are volume center indices, ``N = (N_1, \dots, +N_d) \in \mathbb{N}^d`` are the number of volumes in each dimension, +``\Omega_I = \prod_{\alpha = 1}^d \Delta^\alpha_{I_\alpha}`` is a finite +volume, ``\Gamma^\alpha_I = \Omega_{I - h_\alpha} \cap \Omega_{I + h_\alpha} = +\prod_{\beta \neq \alpha} \Delta^\beta_{I_\beta}`` is a volume face, +``\Delta^\alpha_i = \left[ x^\alpha_{i - \frac{1}{2}}, x^\alpha_{i + \frac{1}{2}} +\right]`` is a volume edge, ``x^\alpha_0, \dots, x^\alpha_{N_\alpha}`` are volume +boundary coordinates, and ``x^\alpha_i = \frac{1}{2} \left(x^\alpha_{i - +\frac{1}{2}} + x^\alpha_{i + \frac{1}{2}}\right)`` for ``i \in \{ 1 / 2, \dots, +N_\alpha - 1 / 2\}`` are volume center coordinates. We also define the operator +``\delta_\alpha`` which maps a discrete scalar field ``\varphi = (\varphi_I)_I`` to ```math -\Omega_I = \prod_{\alpha = 1}^d \left[ x^\alpha_{I(\alpha) - \frac{1}{2}}, -x^\alpha_{I(\alpha) + \frac{1}{2}} \right], \quad I \in \mathcal{I}. +(\delta_\alpha \varphi)_I = \frac{\varphi_{I + h_\alpha} - \varphi_{I - +h_\alpha}}{| \Delta^\alpha_{I_\alpha} |}. ``` -Just like ``\Omega`` itself, they represent rectangles in 2D and prisms n 3D. -They are fully defined by the vectors of volume face coordinates ``x^\alpha = -\left( x^\alpha_{i} \right)_{i = 0}^{N(\alpha)} \in \mathbb{R}^{N(\alpha) + -1}``, where ``N = (N(1), \dots, N(d)) \in \mathbb{N}^d`` are the numbers of -finite volumes in each dimension and ``\mathcal{I} = \prod_{\alpha = 1}^d -\left\{ \frac{1}{2}, 2 - \frac{1}{2}, \dots, N(\alpha) - \frac{1}{2} \right\}`` -the set of finite volume indices (note that the reference volumes are indexed -by half indices only). The components ``x^\alpha_{i}`` are not assumed to be -uniformly spaced. But we do assume that they are strictly increasing with -``i``, with ``x^\alpha_0 = 0`` and ``x^\alpha_{N(\alpha)} = L^\alpha``. - -The coordinates of the volume centers are determined from the those of the -volume boundaries by -``x^\alpha_{I(\alpha)} = \frac{1}{2} (x^\alpha_{I(\alpha) - \frac{1}{2}} + -x_{I(\alpha) + \frac{1}{2}})`` -for ``I \in \mathcal{I}``. This allows for defining shifted volumes such as -``\Omega_{I + \delta(\alpha) / 2}`` and -``\Omega_{I + \delta(\alpha) / 2 + \delta(\beta) / 2}``. The original volumes -(with indices in ``\mathcal{I}``) will be referred to as the reference finite -volumes. - -We also define the volume widths/depths/heights ``\Delta x^\alpha_i = -x^\alpha_{i + \frac{1}{2}} - x^\alpha_{i - \frac{1}{2}}``, where ``i`` can take -half values. The volume sizes are thus ``| \Omega_{I} | = \prod_{\alpha = 1}^d -\Delta x^\alpha_{I(\alpha)}``. -In addition to the finite volumes and their shifted variants, we -define the interface -``\Gamma^\alpha_I = \Omega_{I - \delta(\alpha) / 2} \cup \Omega_{I + -\delta(\alpha) / 2}``. - -In each reference finite volume ``\Omega_{I}`` (``I \in \mathcal{I}``), there -are three different types of positions in which quantities of interest can be -defined: - -- The volume center ``x_I = (x_{I(1)}, \dots, x_{I(d)})``, where the discrete - pressure ``p_I`` is defined; -- The right/rear/top volume face centers ``x_{I + \delta(\alpha) / 2}``, where - the discrete ``\alpha``-velocity component ``u^\alpha_{I + \delta(\alpha) / 2}`` is defined; -- The right-rear-top volume corner ``x_{I + \sum_{\alpha} \delta(\alpha) / - 2}``, where the discrete vorticity ``\omega_{I + \sum_{\alpha} \delta(\alpha) / - 2}`` is defined. - -The vectors of unknowns ``u^\alpha_h`` and ``p_h`` will not contain all the -half-index components, only those from their own canonical position. The -unknown discrete pressure represents the average pressure in each reference -volume, and the unknown discrete velocity components represent exchange of mass -between neighboring volumes. - -In 2D, this finite volume configuration is illustrated as follows: +It can be interpreted as a discrete equivalent of the continuous operator +``\frac{\partial}{\partial x^\alpha}``. All the above definitions are extended to +be valid in volume centers ``I \in \mathcal{I}``, volume faces +``I \in \mathcal{I} + h_\alpha``, +or volume corners ``I \in \mathcal{I} + \sum_{\alpha = 1}^d h_\alpha``. +The discretization is illustrated below: ![Grid](../assets/grid.png) -## Interpolation - -When a quantity is required *outside* of its native point, we will use interpolation. Examples: - -- To compute ``u^\alpha`` at the pressure point ``x_I``, ``I \in \mathcal{I}``: - ```math - \begin{split} - u^\alpha_I & = \frac{x^\alpha_{I(\alpha) + 1 / 2} - - x^\alpha_{I(\alpha)}}{x^\alpha_{I(\alpha) + 1 / 2} - x^\alpha_{I(\alpha) - 1 / 2}} - u_{I - \delta(\alpha) / 2} - + \frac{x^\alpha_{I(\alpha)} - x^\alpha_{I(\alpha) - 1 / 2}}{x^\alpha_{I(\alpha) + 1 / 2} - x^\alpha_{I(\alpha) - 1 / 2}} - u_{I + \delta(\alpha) / 2} \\ - & = \frac{1}{2} \left( u_{I - \delta(\alpha) / 2} + u_{I + \delta(\alpha) / 2} \right). - \end{split} - ``` - Interpolation weights from volume faces to volume centers are always - ``\frac{1}{2}``. -- To compute ``u^\alpha`` at center of edge between ``\alpha``-face and - ``\beta``-face ``x_{I + \delta(\alpha) / 2 + \delta(\beta) / 2}``: - ```math - u^\alpha_{I + \delta(\alpha) / 2 + \delta(\beta) / 2} = - \frac{x^\beta_{I(\beta) + 1} - x^\beta_{I(\beta) + 1 / 2}}{x^\beta_{I(\beta) + 1} - x^\beta_{I(\beta)}} - u^\alpha_{I + \delta(\alpha) / 2} - + \frac{x^\beta_{I(\beta) + 1 / 2} - x^\beta_{I(\beta)}}{x^\beta_{I(\beta) + 1} - x^\beta_{I(\beta)}} - u^\alpha_{I + \delta(\alpha) / 2 + \delta(\beta)}. - ``` - Note that the grid is allowed to be non-uniform, so the interpolation weights - may unequal and different from ``\frac{1}{2}``. -- To compute ``p`` at ``u^\alpha``-points: - ```math - p_{I + \delta(\alpha) / 2} = - \frac{x^\alpha_{I(\alpha) + 1} - x^\alpha_{I(\alpha) + 1 / 2}}{x^\alpha_{I(\alpha) + 1} - x^\alpha_{I(\alpha)}} - p_{I} - + \frac{x^\alpha_{I(\alpha) + 1 / 2} - x^\alpha_{I(\alpha)}}{x^\alpha_{I(\alpha) + 1} - x^\alpha_{I(\alpha)}} - p_{I + \delta(\alpha)} - ``` - - ## Finite volume discretization of the Navier-Stokes equations -We will consider the integral form of the Navier-Stokes equations. This has the -advantage that some of the spatial derivatives disappear, reducing the amount -of finite difference approximations we need to perform. - -We define the finite difference operator ``\partial_\alpha`` equivalent to -the continuous operator ``\frac{\partial}{\partial x^\alpha}``. For all fields -discrete fields ``\varphi``, it is given by - -```math -(\partial_\alpha \varphi)_I = \frac{\varphi_{I + \delta(\alpha) / 2} - \varphi_{I - -\delta(\alpha) / 2}}{\Delta^\alpha_{I(\alpha)}}, -``` - -where ``\varphi`` is interpolated first if necessary. - - -### Mass equation - -The mass equation takes the form - -```math -\frac{1}{| \mathcal{O} |} -\int_{\partial \mathcal{O}} u \cdot n \, \mathrm{d} \Gamma = 0, -\quad \forall \mathcal{O} \subset \Omega. -``` - -Using the pressure volume ``\mathcal{O} = \Omega_{I}``, we get +We now define the unknown degrees of freedom. The average pressure in +``\Omega_I``, ``I \in \mathcal{I}`` is approximated by the quantity ``p_I(t)``. The +average ``\alpha``-velocity on the face ``\Gamma^\alpha_I``, ``I \in \mathcal{I} + +h_\alpha`` is approximated by the quantity ``u^\alpha_I(t)``. Note how the pressure +``p`` and the ``d`` velocity fields ``u^\alpha`` are each defined in their own +canonical positions ``x_I`` and ``x_{I + h_\alpha}`` for ``I \in \mathcal{I}``. +In the following, we derive equations for these unknowns. -```math -\sum_{\alpha = 1}^d -\frac{1}{| \Omega_I |} -\left( \int_{\Gamma^\alpha_{I + \delta(\alpha) / 2}} -u^\alpha \, \mathrm{d} \Gamma - \int_{\Gamma_{I - \delta(\alpha) / 2}^\alpha} -u^\alpha \, \mathrm{d} \Gamma -\right) = 0. -``` - -Assuming that the flow is fully resolved, meaning that ``\Omega_{I}`` is is -sufficiently small such that ``u`` is locally linear, we can perform the -local approximation (quadrature) - -```math -\int_{\Gamma^\alpha_I} u^\alpha \, \mathrm{d} \Gamma \approx | \Gamma^\alpha_I -| u^\alpha_{I}. -``` - -This yields the discrete mass equation +Using the pressure control volume ``\mathcal{O} = \Omega_I`` with ``I \in +\mathcal{I}`` in the mass integral constraint and +approximating the face integrals with the mid-point quadrature rule +``\int_{\Gamma_I} u \, \mathrm{d} \Gamma \approx | \Gamma_I | u_I`` results in the +discrete divergence-free constraint ```math -\sum_{\alpha = 1}^d (\partial_\alpha u^\alpha)_{I} = 0. +\sum_{\alpha = 1}^d (\delta_\alpha u^\alpha)_I = 0. ``` -!!! note "Approximation error" - For the mass equation, the only approximation we have performed is - quadrature. No interpolation or finite difference error is present. - +Note how dividing by the volume size results in a discrete equation resembling +the continuous one (since ``| \Omega_I | = | \Gamma^\alpha_I | | +\Delta^\alpha_{I_\alpha} |``). -### Momentum equations - -Grouping the convection, pressure gradient, diffusion, and body force terms in -each of their own integrals, we get, for all ``\mathcal{O} \subset \Omega``: +Similarly, choosing an ``\alpha``-velocity control volume ``\mathcal{O} = +\Omega_{I}`` with ``I \in \mathcal{I} + h_\alpha`` in the integral momentum +equation, approximating the volume- and face integrals using +the mid-point quadrature rule, and replacing remaining spatial derivatives in +the diffusive term with a finite difference approximation gives the discrete +momentum equations ```math -\begin{split} -\frac{\mathrm{d}}{\mathrm{d} t} -\frac{1}{| \mathcal{O} |} -\int_\mathcal{O} u^\alpha \, \mathrm{d} \Omega -= -& - \sum_{\beta = 1}^d -\frac{1}{| \mathcal{O} |} -\int_{\partial \mathcal{O}} -u^\alpha u^\beta n^\beta -\, \mathrm{d} \Gamma \\ -& + \nu \sum_{\beta = 1}^d -\frac{1}{| \mathcal{O} |} -\int_{\partial \mathcal{O}} -\frac{\partial u^\alpha}{\partial x^\beta} n^\beta -\, \mathrm{d} \Gamma \\ -& + \frac{1}{| \mathcal{O} |} -\int_\mathcal{O} f^\alpha \mathrm{d} \Omega \\ -& - \frac{1}{| \mathcal{O} |} -\int_{\partial \mathcal{O}} p n^\alpha \, \mathrm{d} \Gamma, -\end{split} +\frac{\mathrm{d}}{\mathrm{d} t} u^\alpha_{I} = +- \sum_{\beta = 1}^d +(\delta_\beta (u^\alpha u^\beta))_{I} ++ \nu \sum_{\beta = 1}^d +(\delta_\beta \delta_\beta u^\alpha)_{I} ++ f^\alpha(x_{I}) +- (\delta_\alpha p)_{I}. ``` -where ``n = (n^1, \dots, n^d)`` is the surface normal vector to ``\partial -\mathcal{O}``. - -This time, we will not let ``\mathcal{O}`` be the reference finite volume -``\Omega_{I}`` (the ``p``-volume), but rather the shifted ``u^\alpha``-volume. -Setting ``\mathcal{O} = \Omega_{I + \delta(\alpha) / 2}`` (with right/rear/top -``\beta``-faces ``\Gamma^\beta_{I + \delta(\alpha) / 2 + \delta(\beta) / 2}``) -gives - -```math -\begin{split} - \frac{\mathrm{d}}{\mathrm{d} t} - \frac{1}{| \Omega_{I + \delta(\alpha) / 2} |} - \int_{\Omega_{I + \delta(\alpha) / 2}} - \! \! \! - \! \! \! - \! \! \! - \! \! \! - u^\alpha \, \mathrm{d} \Omega - = - & - - \sum_{\beta = 1}^d - \frac{1}{| \Omega_{I + \delta(\alpha) / 2} |} - \left( - \int_{\Gamma^{\beta}_{I + \delta(\alpha) / 2 + \delta(\beta) / 2}} - \! \! \! - \! \! \! - \! \! \! - \! \! \! - u^\alpha u^\beta \, \mathrm{d} \Gamma - - \int_{\Gamma^{\beta}_{I + \delta(\alpha) / 2 - \delta(\beta) / 2}} - \! \! \! - \! \! \! - \! \! \! - \! \! \! - \! \! \! - \! \! \! - u^\alpha u^\beta \, \mathrm{d} \Gamma - \right) \\ - & + \nu \sum_{\beta = 1}^d - \frac{1}{| \Omega_{I + \delta(\alpha) / 2} |} - \left( - \int_{\Gamma^{\beta}_{I + \delta(\alpha) / 2 + \delta(\beta) / 2}} - \frac{\partial u^\alpha}{\partial x^\beta} \, \mathrm{d} \Gamma - - \int_{\Gamma^{\beta}_{I + \delta(\alpha) / 2 - \delta(\beta) / 2}} - \frac{\partial u^\alpha}{\partial x^\beta} \, \mathrm{d} \Gamma - \right) \\ - & + - \frac{1}{| \Omega_{I + \delta(\alpha) / 2} |} - \int_{\Omega_{I + \delta(\alpha) / 2}} - f^\alpha \, \mathrm{d} \Omega \\ - & - - \frac{1}{| \Omega_{I + \delta(\alpha) / 2} |} - \left( - \int_{\Gamma^{\alpha}_{I + \delta(\alpha)}} p \, \mathrm{d} \Gamma - - \int_{\Gamma^{\alpha}_{I}} p \, \mathrm{d} \Gamma - \right). -\end{split} -``` - -This equation is still exact. We now introduce some approximations on -``\Omega_{I + \delta(\alpha) / 2}`` and its boundaries to remove all unknown -continuous quantities. - -1. We replace the integrals with a mid-point quadrature rule. -1. The mid-point values of derivatives are approximated using a central-like - finite difference: - ```math - \frac{\partial u^\alpha}{\partial x^\beta}(x_I) \approx (\partial_\beta u^\alpha)_I - ``` -1. Quantities outside their canonical positions are obtained through - interpolation. - -Finally, the discrete ``\alpha``-momentum equations are given by - -```math -\begin{split} - \frac{\mathrm{d} }{\mathrm{d} t} u^\alpha_{I + \delta(\alpha) / 2} = - - & \sum_{\beta = 1}^d - (\partial_\beta (u^\alpha u^\beta))_{I + \delta(\alpha) / 2} \\ - + & \nu \sum_{\beta = 1}^d - (\partial_\beta \partial_\beta u^\alpha)_{I + \delta(\alpha) / 2} \\ - + & f^\alpha(x_{I + \delta(\alpha) / 2}, t) - - (\partial_\alpha p)_{I + \delta(\alpha) / 2}. -\end{split} -``` +where we made the assumption that ``f`` is constant in time for simplicity. +The outer discrete derivative in ``(\delta_\beta \delta_\beta u^\alpha)_{I}`` is +required at the position ``I``, which means that the inner derivative is evaluated +as ``(\delta_\beta u^\alpha)_{I + h_\beta}`` and ``(\delta_\beta u^\alpha)_{I - +h_\beta}``, thus requiring ``u^\alpha_{I - 2 h_\beta}``, ``u^\alpha_{I}``, and +``u^\alpha_{I + 2 h_\beta}``, which are all in their canonical positions. The two +velocity components in the convective term ``u^\alpha u^\beta`` are required at +the positions ``I - h_\beta`` and ``I + h_\beta``, which are outside the canonical +positions. Their value at the required position is obtained using averaging with +weights ``1 / 2`` for the ``\alpha``-component and with linear interpolation for the +``\beta``-component. This preserves the skew-symmetry of the convection operator, +such that energy is conserved (in the convective term) [Verstappen2003](@cite). ## Boundary conditions -Depending on the type of boundary conditions, certain indices used in the left- -and right hand sides of the equations may not be part of the solution vectors, -even if they are indeed at their canonical positions. Consider for example the -``\alpha``-left/front/bottom boundary ``\{ x \in \Omega | x^{\alpha} = -x^\alpha_{0} \}``. Let ``\Omega_I`` be one of the reference finite volumes -touching this boundary, i.e. ``I(\alpha) = \frac{1}{2}``. - -- For periodic boundary conditions, we also consider the opposite boundary. - We add two "ghost" reference volumes, one at each side of ``\Omega``: - 1. The ghost volume ``\Omega_{I - \delta(\alpha)}`` has the same shape as - ``\Omega_{I + (N(\alpha) - 1) \delta(\alpha)}`` and contains the same - components: - - ``u^\alpha_{I - \delta(\alpha) / 2} = u^\alpha_{I + (N(\alpha) - 1 / 2) - \delta(\alpha)}``, - - ``u^\beta_{I + \delta(\beta) / 2 - \delta(\alpha)} = - u^\beta_{I + \delta(\beta) / 2 + N(\alpha) \delta(\alpha)}`` for ``\beta - \neq \alpha``, - - ``p_{I - \delta(\alpha)} = p_{I + (N(\alpha) - 1) \delta(\alpha)}``. - 2. The ghost volume ``\Omega_{I + N(\alpha) \delta(\alpha)}`` has the same - shape as ``\Omega_I`` and contains the same pressure and velocity - components. -- For Dirichlet boundary conditions, all the veloctity components are - prescribed. For the normal velocity component, this is straightforward: - ```math - u^\alpha_{I - \delta(\alpha) / 2} = u^\alpha(x_{I - \delta(\alpha) / 2}). - ``` - The parallel (``\beta \neq \alpha``) velocity components - ``u^\beta_{I + \delta(\beta) / 2 - \delta(\alpha)}`` - appear in some of the right hand side expressions. Their ``\alpha`` - position ``x^\alpha_{- 1 / 2}`` has actually never been defined, - so we simply define it to be on the boundary itself: - ``x^\alpha_{- 1 / 2} = x^\alpha_0``. The value can then be - prescribed: - ```math - u^\beta_{I + \delta(\beta) / 2 - \delta(\alpha)} = u^\beta \left( x_{I + - \delta(\beta) / 2 - \delta(\alpha)} \right) = u^\beta \left( x_{I + - \delta(\beta) / 2 - \delta(\alpha) / 2} \right). - ``` - The pressure does not require any boundary modifications. -- For a symmetric boundary, the normal component has zero Dirichlet - conditions, while the parallel components have zero Neumann conditions. For - this, we add a ghost volume ``\Omega_{I - \delta(\alpha)}`` which has the - same shape as ``\Omega_{I}``, meaning that ``x^\alpha_{-1} = x^\alpha_0 - - (x^\alpha_1 - x^\alpha_0) = - x^\alpha_1``. The pressure in this volume is - never used, but we set ``u^\alpha_{I - \delta(\alpha) / 2} = 0`` and - ``u^\beta_{I + \delta(\beta) / 2 - \delta(\alpha)} = u^\beta_{I + - \delta(\beta) / 2}`` for ``\beta \neq \alpha``. -- For a pressure boundary, the value of the pressure is prescribed, while the - velocity has zero Neumann boundary conditions. For this, we add an - infinitely thin ghost volume ``\Omega_{I - \delta(\alpha)}``, by setting - ``x^\alpha_{-1} = x^\alpha_0 - \epsilon`` for some epsilon. We then let this - thickness go to zero *after* substituting the boundary conditions in - the momentum equations. The pressure is prescribed by - ```math - p_{I - \delta(\alpha)} = p(x_{I - \delta(\alpha)}) \underset{\epsilon \to 0}{\to} p(x_{I - \delta(\alpha) / 2}). - ``` - The Neumann boundary conditions are obtained by setting - ``u^\alpha_{I - 3 / 2 \delta(\alpha)} = u^\alpha_{I - \delta(\alpha) / 2}`` - and - ``u^\beta_{I - \delta(\alpha) + \delta(\beta) / 2} = u^\beta_{I + - \delta(\beta) / 2}`` for ``\beta \neq \alpha``. - Note that the normal velocity component at the boundary ``u^\alpha_{I - - \delta(\alpha) / 2}`` is now a degree of freedom, and we need to - observe that the first normal derivative in the diffusion term of its - corresponding momentum equation is zero before taking the limit as ``\epsilon - \to 0``, to avoid dividing by zero. - -It should now be clear from the above cases which components of the discrete -velocity and pressure are unknown degrees of freedom, and which components are -prescribed or obtained otherwise. The *unknown* degrees of freedom are stored -in the vectors ``u_h = (u^1_h, \dots, u^d_h)`` and ``p_h`` using the -column-major convention. Note that the ``d`` discrete velocity fields ``u^1_h, -\dots u^d_h`` may have different numbers of elements. - !!! note "Storage convention" We use the column-major convention (Julia, MATLAB, Fortran), and not the row-major convention (Python, C). Thus the ``x^1``-index ``i`` will vary for @@ -389,45 +119,43 @@ three times coarser control volume ```math \Omega^3_I = -\bigcup_{\alpha = 1}^d \Omega_{I - \delta(\alpha)} \cup -\Omega_I \cup \Omega_{I + \delta(\alpha)}, +\bigcup_{\alpha = 1}^d \Omega_{I - e_\alpha} \cup +\Omega_I \cup \Omega_{I + e_\alpha}, ``` while the momentum equation is -derived for its shifted variant ``\Omega^3_{I + \delta(\alpha) / 2}``. +derived for its shifted variant ``\Omega^3_{I + h_\alpha}``. The resulting fourth order accurate equations are given by ```math \sum_{\alpha = 1}^d -(\partial_\alpha u^\alpha)_I +(\delta_\alpha u^\alpha)_I - \frac{| \Omega^3_I |}{3^{2 + d} | \Omega_I |} \sum_{\alpha = 1}^d -(\partial^3_\alpha u^\alpha)_I +(\delta^3_\alpha u^\alpha)_I = 0 ``` and ```math -\begin{split} - \frac{\mathrm{d} }{\mathrm{d} t} u^\alpha_{I + \delta(\alpha) / 2} = - - & \sum_{\beta = 1}^d - (\partial_\beta (u^\alpha u^\beta))_{I + \delta(\alpha) / 2} \\ - + & \nu \sum_{\beta = 1}^d - (\partial_\beta \partial_\beta u^\alpha)_{I + \delta(\alpha) / 2} \\ - + & f^\alpha(x_{I + \delta(\alpha) / 2}, t) - - (\partial_\alpha p)_{I + \delta(\alpha) / 2}, \\ - + & \text{fourth order} -\end{split} +\frac{\mathrm{d}}{\mathrm{d} t} u^\alpha_{I} = +- \sum_{\beta = 1}^d +(\delta_\beta (u^\alpha u^\beta))_{I} ++ \nu \sum_{\beta = 1}^d +(\delta_\beta \delta_\beta u^\alpha)_{I} ++ f^\alpha(x_{I}) +- (\delta_\alpha p)_{I} ++ \text{fourth order}, ``` where ```math -(\partial^3_\alpha \varphi)_I = -\frac{\varphi_{I + 3 \delta(\alpha) / 2} - -\varphi_{I - 3 \delta(\alpha) / 2}}{\Delta^\alpha_{I(\alpha) - 1} + -\Delta^\alpha_{I(\alpha)} + \Delta^\alpha_{I(\alpha) + 1}}. +(\delta^3_\alpha \varphi)_I = +\frac{\varphi_{I + 3 h_\alpha} - +\varphi_{I - 3 h_\alpha}}{\Delta^\alpha_{I_\alpha - 1} + +\Delta^\alpha_{I_\alpha} + \Delta^\alpha_{I_\alpha + 1}}. ``` ## Matrix representation @@ -540,66 +268,20 @@ the squares. ### Vorticity -In 2D, the vorticity is a scalar. Integrating the vorticity ``\omega = --\frac{\partial u^1}{\partial x^2} + \frac{\partial u^2}{\partial x^1}`` over -the vorticity volume ``\Omega_{I + \delta(1) / 2 + \delta(2) / 2}`` gives +In 2D, the vorticity is a scalar. We define it as ```math -\begin{split} -\int_{\Omega_{I + \delta(1) / 2 + \delta(2) / 2}} \omega \, \mathrm{d} \Omega -= & - \left( -\int_{\Gamma^2_{I + \delta(1) / 2 + \delta(2)}} u^1 \, \mathrm{d} \Gamma -- \int_{\Gamma^2_{I + \delta(1) / 2}} u^1 \, \mathrm{d} \Gamma -\right) \\ -& + \left( -\int_{\Gamma^1_{I + \delta(1) + \delta(2) / 2}} u^2 \, \mathrm{d} \Gamma -- \int_{\Gamma^1_{I + \delta(2) / 2}} u^2 \, \mathrm{d} \Gamma -\right) -\end{split}. -``` - -Using quadrature, and dividing by the vorticity volume -``| \Omega_{I + \delta(1) / 2 + \delta(2) / 2} |``, -the discrete vorticity in the corner is given by - -```math -\omega_{I + \delta(1) / 2 + \delta(2) / 2} = -- \frac{u^1_{I + \delta(1) / 2 + \delta(2)} - -u^1_{I + \delta(1) / 2}}{\Delta^2_{I(2) + 1 / 2}} -+ \frac{u^2_{I + \delta(1) + \delta(2) / 2} - -u^2_{I + \delta(2) / 2}}{\Delta^1_{I(1) + 1 / 2}}. +\omega = -\delta^2 u^1 + \delta^1 u^2. ``` The 3D vorticity is a vector field ``(\omega^1, \omega^2, \omega^3)``. Noting ``\alpha^+ = \operatorname{mod}_3(\alpha + 1)`` and -``\alpha^- = \operatorname{mod}_3(\alpha - 1)``, the vorticity is obtained +``\alpha^- = \operatorname{mod}_3(\alpha - 1)``, the vorticity is defined as through ```math -\begin{split} -\int_{\Omega_{I + \delta(\alpha^+) / 2 + \delta(\alpha^-) / 2}} \omega \, \mathrm{d} \Omega -= & - \left( -\int_{\Gamma^{\alpha^-}_{I + \delta(\alpha^+) / 2 + \delta(\alpha^-)}} u^{\alpha^+} \, \mathrm{d} \Gamma -- \int_{\Gamma^{\alpha^-}_{I + \delta(\alpha^+) / 2}} u^{\alpha^+} \, \mathrm{d} \Gamma -\right) \\ -& + \left( -\int_{\Gamma^{\alpha^+}_{I + \delta(\alpha^+) + \delta(\alpha^-) / 2}} u^{\alpha^-} \, \mathrm{d} \Gamma -- \int_{\Gamma^{\alpha^+}_{I + \delta(\alpha^-) / 2}} u^{\alpha^-} \, \mathrm{d} \Gamma -\right) -\end{split}. -``` - -Using quadrature, and dividing by the vorticity volume -``| \Omega_{I + \delta(\alpha^+) / 2 + \delta(\alpha^-) / 2} |``, -the discrete vorticity around the ``\alpha``-edge is given by - -```math -\omega_{I + \delta(\alpha^+) / 2 + \delta(\alpha^-) / 2} = -- \frac{u^{\alpha^+}_{I + \delta(\alpha^+) / 2 + \delta(\alpha^-)} - -u^{\alpha^+}_{I + \delta(\alpha^+) / 2}}{\Delta^{\alpha^-}_{I(\alpha^-) + 1 / 2}} -+ \frac{u^{\alpha^-}_{I + \delta(\alpha^+) + \delta(\alpha^-) / 2} - -u^{\alpha^-}_{I + \delta(\alpha^-) / 2}}{\Delta^{\alpha^+}_{I(\alpha^+) + 1 / -2}}. +\omega^\alpha = - \delta^{\alpha^-} u^{\alpha^+} + +\delta^{\alpha^+} u^{\alpha^-}. ``` ## Stream function @@ -610,16 +292,16 @@ yields ```math \begin{split} -- \int_{\Omega_{I + \delta(1) / 2 + \delta(2) / 2}} \omega \, \mathrm{d} \Omega -& = \int_{\Omega_{I + \delta(1) / 2 + \delta(2) / 2}} \nabla^2 \psi \, +- \int_{\Omega_{I + h_1 + h_2}} \omega \, \mathrm{d} \Omega +& = \int_{\Omega_{I + h_1 + h_2}} \nabla^2 \psi \, \mathrm{d} \Omega \\ -& = \int_{\Gamma^1_{I + \delta(1) + \delta(2) / 2}} \frac{\partial \psi}{\partial x^1} +& = \int_{\Gamma^1_{I + e_1 + h_2}} \frac{\partial \psi}{\partial x^1} \, \mathrm{d} \Gamma -- \int_{\Gamma^1_{I + \delta(2) / 2}} \frac{\partial \psi}{\partial x^1} +- \int_{\Gamma^1_{I + h_2}} \frac{\partial \psi}{\partial x^1} \, \mathrm{d} \Gamma \\ -& + \int_{\Gamma^2_{I + \delta(1) / 2 + \delta(2)}} \frac{\partial \psi}{\partial x^2} +& + \int_{\Gamma^2_{I + h_1 + e_2}} \frac{\partial \psi}{\partial x^2} \, \mathrm{d} \Gamma -- \int_{\Gamma^2_{I + \delta(1) / 2}} \frac{\partial \psi}{\partial x^2} +- \int_{\Gamma^2_{I + h_1}} \frac{\partial \psi}{\partial x^2} \, \mathrm{d} \Gamma. \end{split} ``` @@ -630,17 +312,17 @@ equation for the stream function at the vorticity point: ```math \begin{split} -\left| \Gamma^1_{I + \delta(1) / 2 + \delta(2) / 2} \right| +\left| \Gamma^1_{I + h_1 + h_2} \right| \left( - \frac{\psi_{I + 3 / 2 \delta(1) + \delta(2) / 2} - \psi_{I + \delta(1) / 2 + \delta(2) / 2}}{x^1_{I(1) + 3 / 2} - x^1_{I(1) + 1 /2}} -- \frac{\psi_{I + \delta(1) / 2 + \delta(2) / 2} - \psi_{I - \delta(1) / 2 + \delta(2) / 2}}{x^1_{I(1) + 1 / 2} - x^1_{I(1) - 1 / 2}} + \frac{\psi_{I + 3 / h_1 + h_2} - \psi_{I + h_1 + h_2}}{x^1_{I_1 + 3 / 2} - x^1_{I_1 + 1 /2}} +- \frac{\psi_{I + h_1 + h_2} - \psi_{I - h_1 + h_2}}{x^1_{I_1 + 1 / 2} - x^1_{I_1 - 1 / 2}} \right) & + \\ -\left| \Gamma^2_{I + \delta(1) / 2 + \delta(2) / 2} \right| +\left| \Gamma^2_{I + h_1 + h_2} \right| \left( -\frac{\psi_{I + \delta(1) / 2 + 3 / 2 \delta(2)} - \psi_{I + \delta(1) / 2 + \delta(2) / 2}}{x^2_{I(1) + 3 / 2} - x^2_{I(1) + 1 / 2}} --\frac{\psi_{I + \delta(1) / 2 + \delta(2) / 2} - \psi_{I + \delta(1) / 2 - \delta(2) / 2}}{x^2_{I(2) + 1 / 2} - x^2_{I(2) - 1 / 2}} +\frac{\psi_{I + h_1 + 3 h_2} - \psi_{I + h_1 + h_2}}{x^2_{I_1 + 3 / 2} - x^2_{I_1 + 1 / 2}} +-\frac{\psi_{I + h_1 + h_2} - \psi_{I + h_1 - h_2}}{x^2_{I_2 + 1 / 2} - x^2_{I_2 - 1 / 2}} \right) & = \\ -\left| \Omega_{I + \delta(1) / 2 + \delta(2) / 2} \right| -\omega_{I + \delta(1) / 2 + \delta(2) / 2} & +\left| \Omega_{I + h_1 + h_2} \right| +\omega_{I + h_1 + h_2} & \end{split} ``` diff --git a/docs/src/features/closure.md b/docs/src/features/closure.md index 115edce44..f2ec0f84f 100644 --- a/docs/src/features/closure.md +++ b/docs/src/features/closure.md @@ -1,5 +1,13 @@ # Neural closure models +!!! note "`NeuralClorusure`" + These features are experimental, and require cloning + IncompressibleNavierStokes from GitHub: + ```sh + git clone https://github.com/agdestein/IncompressibleNavierStokes.jl + cd IncompressibleNavierStokes/lib/NeuralClosure + ``` + For [large eddy simulation (LES)](../features/les.md), a closure model is required. With IncompressibleNavierStokes, a neural closure model can be trained on filtered DNS data. The discrete DNS equations are given by @@ -34,14 +42,25 @@ M \bar{v} & = 0, \\ \end{split} ``` +## NeuralClosure module + +IncompressibleNavierStokes provides a NeuralClosure module. + +```@docs +NeuralClosure.NeuralClosure +``` + +The following filters are available: + ```@docs NeuralClosure.FaceAverage NeuralClosure.VolumeAverage NeuralClosure.reconstruct! ``` +The following functions create data. + ```@docs -NeuralClosure.NeuralClosure NeuralClosure.filtersaver NeuralClosure.create_les_data NeuralClosure.create_io_arrays diff --git a/docs/src/features/gpu.md b/docs/src/features/gpu.md index d566a96e6..b3f6ac5a2 100644 --- a/docs/src/features/gpu.md +++ b/docs/src/features/gpu.md @@ -1,16 +1,28 @@ # GPU Support IncompressibleNavierStokes supports various array types. The desired array type -only has to be passed to the [`Setup`](@ref) function. All operators have been +only has to be passed to the [`Setup`](@ref) function: + +```julia +using CUDA +setup = Setup(x...; kwargs..., ArrayType = CuArray) +``` + +All operators have been made are backend agnostic by using [KernelAbstractions.jl](https://github.com/JuliaGPU/KernelAbstractions.jl/). -Even if a GPU is not available, the operators are multithreaded if Julia is started with multiple threads (e.g. `julia -t 4`) - -Limitations: +Even if a GPU is not available, the operators are multithreaded if +Julia is started with multiple threads (e.g. `julia -t 4`) +- This has been tested with CUDA compatible GPUs. - This has not been tested with other GPU interfaces, such as - [AMDGPU.jl](https://github.com/JuliaGPU/AMDGPU.jl) - [Metal.jl](https://github.com/JuliaGPU/Metal.jl) - [oneAPI.jl](https://github.com/JuliaGPU/oneAPI.jl) If they start supporting sparse matrices and fast Fourier transforms they could also be used. + +!!! note "`psolver_direct` on CUDA" + To use a specialized linear solver for CUDA, make sure to install and + `using` CUDA.jl and CUDSS.jl. Then `psolver_direct` will automatically use + the CUDSS solver. diff --git a/docs/src/features/temperature.md b/docs/src/features/temperature.md index 53efc0f45..5bfe77ce7 100644 --- a/docs/src/features/temperature.md +++ b/docs/src/features/temperature.md @@ -19,6 +19,8 @@ setup = Setup( ) ``` +where `temperature_equation` can be configured as follows: + ```@docs temperature_equation ``` From 38f627fe9813f11ce6bf1bae07a0864210f5fb9d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Syver=20D=C3=B8ving=20Agdestein?= Date: Mon, 6 May 2024 23:37:42 +0200 Subject: [PATCH 379/379] Fix test --- examples/LidDrivenCavity2D.jl | 2 +- src/IncompressibleNavierStokes.jl | 1 - test/operators.jl | 2 +- 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/examples/LidDrivenCavity2D.jl b/examples/LidDrivenCavity2D.jl index fa61deef2..e020b7edf 100644 --- a/examples/LidDrivenCavity2D.jl +++ b/examples/LidDrivenCavity2D.jl @@ -111,7 +111,7 @@ processors = ( # By default, a standard fourth order Runge-Kutta method is used. If we don't # provide the time step explicitly, an adaptive time step is used. tlims = (T(0), T(10)) -state, outputs = solve_unsteady(; setup, ustart, tlims; Δt = T(1e-3), processors); +state, outputs = solve_unsteady(; setup, ustart, tlims, Δt = T(1e-3), processors); # ## Post-process # diff --git a/src/IncompressibleNavierStokes.jl b/src/IncompressibleNavierStokes.jl index 849751054..5e4322870 100644 --- a/src/IncompressibleNavierStokes.jl +++ b/src/IncompressibleNavierStokes.jl @@ -20,7 +20,6 @@ using StaticArrays using Statistics using WriteVTK: CollectionFile, paraview_collection, vtk_grid, vtk_save - # # Easily retrieve value from Val # (::Val{x})() where {x} = x diff --git a/test/operators.jl b/test/operators.jl index d644a455f..1bca2e7c4 100644 --- a/test/operators.jl +++ b/test/operators.jl @@ -117,7 +117,7 @@ testops(dim) = @testset "Operators $(dim())D" begin end @testset "Momentum" begin - m = momentum(u, temp, T(1), setup) + m = momentum(u, nothing, T(1), setup) @test m isa Tuple @test m[1] isa Array{T} @test all(all.(!isnan, m))