From a1e350851de4f2a31636214fb728461ddc78309a Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Thu, 29 Feb 2024 19:13:02 -0500 Subject: [PATCH 01/11] first commit --- test/test_distributed_hydrostatic_model.jl | 94 ++++++++++++---------- 1 file changed, 50 insertions(+), 44 deletions(-) diff --git a/test/test_distributed_hydrostatic_model.jl b/test/test_distributed_hydrostatic_model.jl index 8511a4c8ad..6dd53942e0 100644 --- a/test/test_distributed_hydrostatic_model.jl +++ b/test/test_distributed_hydrostatic_model.jl @@ -78,49 +78,55 @@ Ny = 32 for arch in archs @testset "Testing distributed solid body rotation" begin - grid = LatitudeLongitudeGrid(arch, size = (Nx, Ny, 1), - halo = (3, 3, 3), - latitude = (-80, 80), - longitude = (-160, 160), - z = (-1, 0), - radius = 1, - topology=(Bounded, Bounded, Bounded)) - - global_grid = reconstruct_global_grid(grid) - - # "s" for "serial" computation - us, vs, ws, cs, ηs = solid_body_rotation_test(global_grid) - - us = Array(interior(us)) - vs = Array(interior(vs)) - ws = Array(interior(ws)) - cs = Array(interior(cs)) - ηs = Array(interior(ηs)) - - @info " Testing distributed solid body rotation with architecture $arch" - u, v, w, c, η = solid_body_rotation_test(grid) - - u = Array(interior(u)) - v = Array(interior(v)) - w = Array(interior(w)) - c = Array(interior(c)) - η = Array(interior(η)) - - cpu_arch = cpu_architecture(arch) - - us = partition_global_array(cpu_arch, us, size(u)) - vs = partition_global_array(cpu_arch, vs, size(v)) - ws = partition_global_array(cpu_arch, ws, size(w)) - cs = partition_global_array(cpu_arch, cs, size(c)) - ηs = partition_global_array(cpu_arch, ηs, size(η)) - - atol = eps(eltype(grid)) - rtol = sqrt(eps(eltype(grid))) - - @test all(isapprox(u, us; atol, rtol)) - @test all(isapprox(v, vs; atol, rtol)) - @test all(isapprox(w, ws; atol, rtol)) - @test all(isapprox(c, cs; atol, rtol)) - @test all(isapprox(η, ηs; atol, rtol)) + underlying_grid = LatitudeLongitudeGrid(arch, size = (Nx, Ny, 1), + halo = (3, 3, 3), + latitude = (-80, 80), + longitude = (-160, 160), + z = (-1, 0), + radius = 1, + topology=(Bounded, Bounded, Bounded)) + + bottom(λ, φ) = -30 < λ < 30 && -40 < φ < 20 ? 0 : - 1 + + immersed_grid = ImmersedBoundaryGrid(underlying_grid, GridFittedBottom(bottom); active_cells_map = true) + + for grid in (underlying_grid, immersed_grid) + global_grid = reconstruct_global_grid(grid) + + # "s" for "serial" computation + us, vs, ws, cs, ηs = solid_body_rotation_test(global_grid) + + us = Array(interior(us)) + vs = Array(interior(vs)) + ws = Array(interior(ws)) + cs = Array(interior(cs)) + ηs = Array(interior(ηs)) + + @info " Testing distributed solid body rotation with architecture $arch" + u, v, w, c, η = solid_body_rotation_test(grid) + + u = Array(interior(u)) + v = Array(interior(v)) + w = Array(interior(w)) + c = Array(interior(c)) + η = Array(interior(η)) + + cpu_arch = cpu_architecture(arch) + + us = partition_global_array(cpu_arch, us, size(u)) + vs = partition_global_array(cpu_arch, vs, size(v)) + ws = partition_global_array(cpu_arch, ws, size(w)) + cs = partition_global_array(cpu_arch, cs, size(c)) + ηs = partition_global_array(cpu_arch, ηs, size(η)) + + atol = eps(eltype(grid)) + rtol = sqrt(eps(eltype(grid))) + + @test all(isapprox(u, us; atol, rtol)) + @test all(isapprox(v, vs; atol, rtol)) + @test all(isapprox(w, ws; atol, rtol)) + @test all(isapprox(c, cs; atol, rtol)) + @test all(isapprox(η, ηs; atol, rtol)) + end end end From 0602848d70d2c4fd65419b6a5633671032412ff9 Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Thu, 29 Feb 2024 19:15:18 -0500 Subject: [PATCH 02/11] change the info message --- test/test_distributed_hydrostatic_model.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_distributed_hydrostatic_model.jl b/test/test_distributed_hydrostatic_model.jl index 6dd53942e0..cce7805c1d 100644 --- a/test/test_distributed_hydrostatic_model.jl +++ b/test/test_distributed_hydrostatic_model.jl @@ -102,7 +102,7 @@ for arch in archs cs = Array(interior(cs)) ηs = Array(interior(ηs)) - @info " Testing distributed solid body rotation with architecture $arch" + @info " Testing distributed solid body rotation with architecture $arch on $(typeof(grid).name.wrapper)" u, v, w, c, η = solid_body_rotation_test(grid) u = Array(interior(u)) From 8e5691b10622d4c60b27dbc5de0e37713056be91 Mon Sep 17 00:00:00 2001 From: "Navid C. Constantinou" Date: Fri, 1 Mar 2024 10:19:48 +0200 Subject: [PATCH 03/11] central -> new-central --- .buildkite/distributed/JuliaProject.toml | 15 --------------- .buildkite/distributed/pipeline.yml | 9 +++------ 2 files changed, 3 insertions(+), 21 deletions(-) delete mode 100644 .buildkite/distributed/JuliaProject.toml diff --git a/.buildkite/distributed/JuliaProject.toml b/.buildkite/distributed/JuliaProject.toml deleted file mode 100644 index fff554e83a..0000000000 --- a/.buildkite/distributed/JuliaProject.toml +++ /dev/null @@ -1,15 +0,0 @@ -[extras] -CUDA_Runtime_jll = "76a88914-d11a-5bdc-97e0-2f5a05c973a2" -CUDA_Driver_jll = "4ee394cb-3365-5eb0-8335-949819d2adfc" -MPIPreferences = "3da0fdf6-3ccc-4f1b-acd9-58baa6c99267" - -[preferences.CUDA_Driver_jll] -compat = false - -[preferences.CUDA_Runtime_jll] -version = "12.2" -local = "true" - -[preferences.MPIPreferences] -_format = "1.0" -binary = "MPItrampoline_jll" diff --git a/.buildkite/distributed/pipeline.yml b/.buildkite/distributed/pipeline.yml index 27325c245a..bc6dd5c456 100644 --- a/.buildkite/distributed/pipeline.yml +++ b/.buildkite/distributed/pipeline.yml @@ -1,17 +1,14 @@ agents: - queue: central + queue: new-central slurm_mem: 8G - modules: julia/1.10.0 cuda/12.2 ucx/1.14.1_cuda-12.2 openmpi/4.1.5_cuda-12.2 nsight-systems/2023.2.1 + modules: climacommon/2024_02_27 + env: JULIA_LOAD_PATH: "${JULIA_LOAD_PATH}:${BUILDKITE_BUILD_CHECKOUT_PATH}/.buildkite/distributed" OPENBLAS_NUM_THREADS: 1 - JULIA_NVTX_CALLBACKS: gc OMPI_MCA_opal_warn_on_missing_libcuda: 0 JULIA_CPU_TARGET: 'broadwell;skylake' - JULIA_CUDA_MEMORY_POOL: none - MPITRAMPOLINE_LIB: /groups/esm/software/MPIwrapper/ompi4.1.5_cuda-12.2/lib64/libmpiwrapper.so - MPITRAMPOLINE_MPIEXEC: /groups/esm/software/MPIwrapper/ompi4.1.5_cuda-12.2/bin/mpiwrapperexec steps: - label: "initialize" From ce6099b3513336cc6a2515dbea934de2f38b776a Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Fri, 1 Mar 2024 10:00:04 -0500 Subject: [PATCH 04/11] just testing an hypothesis here --- test/test_distributed_hydrostatic_model.jl | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/test/test_distributed_hydrostatic_model.jl b/test/test_distributed_hydrostatic_model.jl index cce7805c1d..a3a0671967 100644 --- a/test/test_distributed_hydrostatic_model.jl +++ b/test/test_distributed_hydrostatic_model.jl @@ -90,8 +90,10 @@ for arch in archs immersed_grid = ImmersedBoundaryGrid(underlying_grid, GridFittedBottom(bottom); active_cells_map = true) - for grid in (underlying_grid, immersed_grid) - global_grid = reconstruct_global_grid(grid) + global_underlying_grid = reconstruct_global_grid(underlying_grid) + global_immersed_grid = ImmersedBoundaryGrid(global_underlying_grid, GridFittedBottom(bottom); active_cells_map = true) + + for (grid, global_grid) in zip((underlying_grid, immersed_grid), (global_underlying_grid, global_immersed_grid)) # "s" for "serial" computation us, vs, ws, cs, ηs = solid_body_rotation_test(global_grid) From b31d84a5d9ef7e961ce9093818f67753f06599f9 Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Fri, 1 Mar 2024 15:50:15 -0500 Subject: [PATCH 05/11] bugfix --- test/test_distributed_hydrostatic_model.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_distributed_hydrostatic_model.jl b/test/test_distributed_hydrostatic_model.jl index a3a0671967..2dcfb14779 100644 --- a/test/test_distributed_hydrostatic_model.jl +++ b/test/test_distributed_hydrostatic_model.jl @@ -79,7 +79,7 @@ Ny = 32 for arch in archs @testset "Testing distributed solid body rotation" begin underlying_grid = LatitudeLongitudeGrid(arch, size = (Nx, Ny, 1), - halo = (3, 3, 3), + halo = (4, 4, 4), latitude = (-80, 80), longitude = (-160, 160), z = (-1, 0), From 384128933022e326435854bdbc11603fd8295cc8 Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Thu, 7 Mar 2024 12:04:55 -0500 Subject: [PATCH 06/11] on architecture --- test/dependencies_for_runtests.jl | 1 + test/test_distributed_hydrostatic_model.jl | 20 ++++++++++---------- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/test/dependencies_for_runtests.jl b/test/dependencies_for_runtests.jl index 1e7b206af9..6f5816e813 100644 --- a/test/dependencies_for_runtests.jl +++ b/test/dependencies_for_runtests.jl @@ -39,6 +39,7 @@ using Oceananigans.Units using Oceananigans.Utils using Oceananigans.MultiRegion using Oceananigans.Architectures: device, array_type # to resolve conflict with CUDA.device +using Oceananigans.Architectures: on_architecture using Oceananigans: Clock using Dates: DateTime, Nanosecond diff --git a/test/test_distributed_hydrostatic_model.jl b/test/test_distributed_hydrostatic_model.jl index 2dcfb14779..43d5d46e41 100644 --- a/test/test_distributed_hydrostatic_model.jl +++ b/test/test_distributed_hydrostatic_model.jl @@ -98,20 +98,20 @@ for arch in archs # "s" for "serial" computation us, vs, ws, cs, ηs = solid_body_rotation_test(global_grid) - us = Array(interior(us)) - vs = Array(interior(vs)) - ws = Array(interior(ws)) - cs = Array(interior(cs)) - ηs = Array(interior(ηs)) + us = interior(on_architecture(CPU(), us)) + vs = interior(on_architecture(CPU(), vs)) + ws = interior(on_architecture(CPU(), ws)) + cs = interior(on_architecture(CPU(), cs)) + ηs = interior(on_architecture(CPU(), ηs)) @info " Testing distributed solid body rotation with architecture $arch on $(typeof(grid).name.wrapper)" u, v, w, c, η = solid_body_rotation_test(grid) - u = Array(interior(u)) - v = Array(interior(v)) - w = Array(interior(w)) - c = Array(interior(c)) - η = Array(interior(η)) + u = interior(on_architecture(CPU(), u)) + v = interior(on_architecture(CPU(), v)) + w = interior(on_architecture(CPU(), w)) + c = interior(on_architecture(CPU(), c)) + η = interior(on_architecture(CPU(), η)) cpu_arch = cpu_architecture(arch) From c45617bf7cc711b715828e26ef946f209b1b19c5 Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Fri, 8 Mar 2024 08:40:26 -0500 Subject: [PATCH 07/11] bugfix --- src/Fields/field.jl | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/Fields/field.jl b/src/Fields/field.jl index 27bf1c2c3c..0c19664430 100644 --- a/src/Fields/field.jl +++ b/src/Fields/field.jl @@ -420,12 +420,12 @@ total_size(f::Field) = total_size(f.grid, location(f), f.indices) on_architecture(arch, field::AbstractField{LX, LY, LZ}) where {LX, LY, LZ} = Field{LX, LY, LZ}(on_architecture(arch, field.grid), - on_architecture(arch, data), - on_architecture(arch, bcs), - on_architecture(arch, indices), - on_architecture(arch, op), - on_architecture(arch, status), - on_architecture(arch, buffers)) + on_architecture(arch, field.data), + on_architecture(arch, field.bcs), + on_architecture(arch, field.indices), + on_architecture(arch, field.op), + on_architecture(arch, field.status), + on_architecture(arch, field.buffers)) ##### ##### Interface for field computations From 824edd87e58c8e0fe37662c2cf76ee2ef6884460 Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Fri, 8 Mar 2024 09:51:22 -0500 Subject: [PATCH 08/11] bugfix --- src/Fields/field.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Fields/field.jl b/src/Fields/field.jl index 0c19664430..f32f70fb8a 100644 --- a/src/Fields/field.jl +++ b/src/Fields/field.jl @@ -421,11 +421,11 @@ total_size(f::Field) = total_size(f.grid, location(f), f.indices) on_architecture(arch, field::AbstractField{LX, LY, LZ}) where {LX, LY, LZ} = Field{LX, LY, LZ}(on_architecture(arch, field.grid), on_architecture(arch, field.data), - on_architecture(arch, field.bcs), + on_architecture(arch, field.boundary_conditions), on_architecture(arch, field.indices), on_architecture(arch, field.op), on_architecture(arch, field.status), - on_architecture(arch, field.buffers)) + on_architecture(arch, field.boundary_buffers)) ##### ##### Interface for field computations From 0043d9cec106b047accb9309a1be3d8bf601b93b Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Fri, 8 Mar 2024 10:53:49 -0500 Subject: [PATCH 09/11] last bugfix --- src/Fields/field.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Fields/field.jl b/src/Fields/field.jl index f32f70fb8a..d0b82dcedd 100644 --- a/src/Fields/field.jl +++ b/src/Fields/field.jl @@ -423,7 +423,7 @@ on_architecture(arch, field::AbstractField{LX, LY, LZ}) where {LX, LY, LZ} = on_architecture(arch, field.data), on_architecture(arch, field.boundary_conditions), on_architecture(arch, field.indices), - on_architecture(arch, field.op), + on_architecture(arch, field.operand), on_architecture(arch, field.status), on_architecture(arch, field.boundary_buffers)) From 67ec92e4bb006e7f3001fd48c9c59ecd6b63ab76 Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Fri, 8 Mar 2024 17:14:41 -0500 Subject: [PATCH 10/11] last bugfix --- test/test_distributed_hydrostatic_model.jl | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/test/test_distributed_hydrostatic_model.jl b/test/test_distributed_hydrostatic_model.jl index 43d5d46e41..7c75436d3f 100644 --- a/test/test_distributed_hydrostatic_model.jl +++ b/test/test_distributed_hydrostatic_model.jl @@ -107,14 +107,14 @@ for arch in archs @info " Testing distributed solid body rotation with architecture $arch on $(typeof(grid).name.wrapper)" u, v, w, c, η = solid_body_rotation_test(grid) - u = interior(on_architecture(CPU(), u)) - v = interior(on_architecture(CPU(), v)) - w = interior(on_architecture(CPU(), w)) - c = interior(on_architecture(CPU(), c)) - η = interior(on_architecture(CPU(), η)) - cpu_arch = cpu_architecture(arch) + u = interior(on_architecture(cpu_arch, u)) + v = interior(on_architecture(cpu_arch, v)) + w = interior(on_architecture(cpu_arch, w)) + c = interior(on_architecture(cpu_arch, c)) + η = interior(on_architecture(cpu_arch, η)) + us = partition_global_array(cpu_arch, us, size(u)) vs = partition_global_array(cpu_arch, vs, size(v)) ws = partition_global_array(cpu_arch, ws, size(w)) From 361b219c9207d5e3b019f3a675f5ee13f0e331c0 Mon Sep 17 00:00:00 2001 From: "Navid C. Constantinou" Date: Mon, 11 Mar 2024 23:41:11 +0200 Subject: [PATCH 11/11] bump patch release --- Project.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Project.toml b/Project.toml index 2353752cc1..7438f1c929 100644 --- a/Project.toml +++ b/Project.toml @@ -1,7 +1,7 @@ name = "Oceananigans" uuid = "9e8cae18-63c1-5223-a75c-80ca9d6e9a09" authors = ["Climate Modeling Alliance and contributors"] -version = "0.90.10" +version = "0.90.11" [deps] Adapt = "79e6a3ab-5dfb-504d-930d-738a2a938a0e"