(0.90.11) Tests for a distributed Immersed boundary grid + fix some `…

…on_architecture` bugs (#3487)

* first commit

* change the info message

* central -> new-central

* just testing an hypothesis here

* bugfix

* on architecture

* bugfix

* bugfix

* last bugfix

* last bugfix

* bump patch release

---------

Co-authored-by: Navid C. Constantinou <[email protected]>

Project.toml

@@ -1,7 +1,7 @@
 name = "Oceananigans"
 uuid = "9e8cae18-63c1-5223-a75c-80ca9d6e9a09"
 authors = ["Climate Modeling Alliance and contributors"]
-version = "0.90.10"
+version = "0.90.11"
 
 [deps]
 Adapt = "79e6a3ab-5dfb-504d-930d-738a2a938a0e"

src/Fields/field.jl

@@ -420,12 +420,12 @@ total_size(f::Field) = total_size(f.grid, location(f), f.indices)
 
 on_architecture(arch, field::AbstractField{LX, LY, LZ}) where {LX, LY, LZ} = 
     Field{LX, LY, LZ}(on_architecture(arch, field.grid), 
-                      on_architecture(arch, data),
-                      on_architecture(arch, bcs),
-                      on_architecture(arch, indices),
-                      on_architecture(arch, op),
-                      on_architecture(arch, status),
-                      on_architecture(arch, buffers))
+                      on_architecture(arch, field.data),
+                      on_architecture(arch, field.boundary_conditions),
+                      on_architecture(arch, field.indices),
+                      on_architecture(arch, field.operand),
+                      on_architecture(arch, field.status),
+                      on_architecture(arch, field.boundary_buffers))
 
 #####
 ##### Interface for field computations

test/dependencies_for_runtests.jl

@@ -39,6 +39,7 @@ using Oceananigans.Units
 using Oceananigans.Utils
 using Oceananigans.MultiRegion
 using Oceananigans.Architectures: device, array_type # to resolve conflict with CUDA.device
+using Oceananigans.Architectures: on_architecture
 
 using Oceananigans: Clock
 using Dates: DateTime, Nanosecond

test/test_distributed_hydrostatic_model.jl

@@ -78,49 +78,57 @@ Ny = 32
 
 for arch in archs
     @testset "Testing distributed solid body rotation" begin
-        grid = LatitudeLongitudeGrid(arch, size = (Nx, Ny, 1),
-                                     halo = (4, 4, 4),
-                                     latitude = (-80, 80),
-                                     longitude = (-160, 160),
-                                     z = (-1, 0),
-                                     radius = 1,
-                                     topology=(Bounded, Bounded, Bounded))
-
-        global_grid = reconstruct_global_grid(grid)
-
-        # "s" for "serial" computation
-        us, vs, ws, cs, ηs = solid_body_rotation_test(global_grid)
-
-        us = Array(interior(us))
-        vs = Array(interior(vs))
-        ws = Array(interior(ws))
-        cs = Array(interior(cs))
-        ηs = Array(interior(ηs))
-
-        @info "  Testing distributed solid body rotation with architecture $arch"
-        u, v, w, c, η = solid_body_rotation_test(grid)
-
-        u = Array(interior(u))
-        v = Array(interior(v))
-        w = Array(interior(w))
-        c = Array(interior(c))
-        η = Array(interior(η))
-
-        cpu_arch = cpu_architecture(arch)
-
-        us = partition_global_array(cpu_arch, us, size(u))
-        vs = partition_global_array(cpu_arch, vs, size(v))
-        ws = partition_global_array(cpu_arch, ws, size(w))
-        cs = partition_global_array(cpu_arch, cs, size(c))
-        ηs = partition_global_array(cpu_arch, ηs, size(η))
-
-        atol = eps(eltype(grid))
-        rtol = sqrt(eps(eltype(grid)))
-
-        @test all(isapprox(u, us; atol, rtol))
-        @test all(isapprox(v, vs; atol, rtol))
-        @test all(isapprox(w, ws; atol, rtol))
-        @test all(isapprox(c, cs; atol, rtol))
-        @test all(isapprox(η, ηs; atol, rtol))
+        underlying_grid = LatitudeLongitudeGrid(arch, size = (Nx, Ny, 1),
+                                                halo = (4, 4, 4),
+                                                latitude = (-80, 80),
+                                                longitude = (-160, 160),
+                                                z = (-1, 0),
+                                                radius = 1,
+                                                topology=(Bounded, Bounded, Bounded))
+
+        bottom(λ, φ) = -30 < λ < 30 && -40 < φ < 20 ? 0 : - 1
+
+        immersed_grid = ImmersedBoundaryGrid(underlying_grid, GridFittedBottom(bottom); active_cells_map = true)
+
+        global_underlying_grid = reconstruct_global_grid(underlying_grid)
+        global_immersed_grid   = ImmersedBoundaryGrid(global_underlying_grid, GridFittedBottom(bottom); active_cells_map = true)
+
+        for (grid, global_grid) in zip((underlying_grid, immersed_grid), (global_underlying_grid, global_immersed_grid))
+
+            # "s" for "serial" computation
+            us, vs, ws, cs, ηs = solid_body_rotation_test(global_grid)
+
+            us = interior(on_architecture(CPU(), us))
+            vs = interior(on_architecture(CPU(), vs))
+            ws = interior(on_architecture(CPU(), ws))
+            cs = interior(on_architecture(CPU(), cs))
+            ηs = interior(on_architecture(CPU(), ηs))
+
+            @info "  Testing distributed solid body rotation with architecture $arch on $(typeof(grid).name.wrapper)"
+            u, v, w, c, η = solid_body_rotation_test(grid)
+
+            cpu_arch = cpu_architecture(arch)
+
+            u = interior(on_architecture(cpu_arch, u))
+            v = interior(on_architecture(cpu_arch, v))
+            w = interior(on_architecture(cpu_arch, w))
+            c = interior(on_architecture(cpu_arch, c))
+            η = interior(on_architecture(cpu_arch, η))
+
+            us = partition_global_array(cpu_arch, us, size(u))
+            vs = partition_global_array(cpu_arch, vs, size(v))
+            ws = partition_global_array(cpu_arch, ws, size(w))
+            cs = partition_global_array(cpu_arch, cs, size(c))
+            ηs = partition_global_array(cpu_arch, ηs, size(η))
+
+            atol = eps(eltype(grid))
+            rtol = sqrt(eps(eltype(grid)))
+
+            @test all(isapprox(u, us; atol, rtol))
+            @test all(isapprox(v, vs; atol, rtol))
+            @test all(isapprox(w, ws; atol, rtol))
+            @test all(isapprox(c, cs; atol, rtol))
+            @test all(isapprox(η, ηs; atol, rtol))
+        end
     end
 end