From e06d78ed8b8f18110dd4b04177cdce6331080dd0 Mon Sep 17 00:00:00 2001 From: Gregory Wagner Date: Fri, 15 Mar 2024 13:37:11 -0600 Subject: [PATCH 01/25] Generalize file splitting for JLD2OutputWriter so that alternative criterion to file size may be used --- src/OutputWriters/OutputWriters.jl | 1 + src/OutputWriters/jld2_output_writer.jl | 48 +++++++++++++++--------- src/OutputWriters/output_writer_utils.jl | 28 ++++++++++++++ 3 files changed, 60 insertions(+), 17 deletions(-) diff --git a/src/OutputWriters/OutputWriters.jl b/src/OutputWriters/OutputWriters.jl index 1d5de0f6b5..810fe3e88a 100644 --- a/src/OutputWriters/OutputWriters.jl +++ b/src/OutputWriters/OutputWriters.jl @@ -15,6 +15,7 @@ using Oceananigans.Models using Oceananigans: AbstractOutputWriter using Oceananigans.Grids: interior_indices using Oceananigans.Utils: TimeInterval, IterationInterval, WallTimeInterval, instantiate +using Oceananigans.Utils: pretty_filesize using OffsetArrays diff --git a/src/OutputWriters/jld2_output_writer.jl b/src/OutputWriters/jld2_output_writer.jl index 37eeaee203..550694f8f1 100644 --- a/src/OutputWriters/jld2_output_writer.jl +++ b/src/OutputWriters/jld2_output_writer.jl @@ -2,14 +2,22 @@ using Printf using JLD2 using Oceananigans.Utils using Oceananigans.Models -using Oceananigans.Utils: TimeInterval, pretty_filesize, prettykeys +using Oceananigans.Utils: TimeInterval, prettykeys using Oceananigans.Fields: boundary_conditions, indices default_included_properties(::NonhydrostaticModel) = [:grid, :coriolis, :buoyancy, :closure] default_included_properties(::ShallowWaterModel) = [:grid, :coriolis, :closure] default_included_properties(::HydrostaticFreeSurfaceModel) = [:grid, :coriolis, :buoyancy, :closure] -mutable struct JLD2OutputWriter{O, T, D, IF, IN, KW} <: AbstractOutputWriter +update_file_splitting_schedule!(schedule, filepath) = nothing +update_file_splitting_schedule!(schedule::FileSizeLimit, filepath) = schedule.path = filepath + +struct NoFileSplitting end +(::NoFileSplitting)(model) = false +Base.summary(::NoFileSplitting) = "NoFileSplitting" +Base.show(io::IO, nfs::NoFileSplitting) = print(io, summary(nfs)) + +mutable struct JLD2OutputWriter{O, T, D, IF, IN, FS, KW} <: AbstractOutputWriter filepath :: String outputs :: O schedule :: T @@ -17,7 +25,7 @@ mutable struct JLD2OutputWriter{O, T, D, IF, IN, KW} <: AbstractOutputWriter init :: IF including :: IN part :: Int - max_filesize :: Float64 + file_splitting :: FS overwrite_existing :: Bool verbose :: Bool jld2_kw :: KW @@ -32,7 +40,7 @@ ext(::Type{JLD2OutputWriter}) = ".jld2" indices = (:, :, :), with_halos = false, array_type = Array{Float64}, - max_filesize = Inf, + file_splitting = NoFileSplitting(), overwrite_existing = false, init = noinit, including = [:grid, :coriolis, :buoyancy, :closure], @@ -80,10 +88,12 @@ Keyword arguments ## File management -- `max_filesize`: The writer will stop writing to the output file once the file size exceeds `max_filesize`, - and write to a new one with a consistent naming scheme ending in `part1`, `part2`, etc. - Defaults to `Inf`. - +- `file_splitting`: Schedule for splitting the output file. The new files will be suffixed with + `_part1`, `_part2`, etc. For example `file_splitting = FileSizeLimit(sz)` will + split the output file when its size exceeds `sz`. Another example is + `file_splitting = TimeInterval(30days)`, which will split files every 30 days of + simulation time. The default incurs no splitting. + - `overwrite_existing`: Remove existing files if their filenames conflict. Default: `false`. @@ -100,7 +110,7 @@ Keyword arguments - `verbose`: Log what the output writer is doing with statistics on compute/write times and file sizes. Default: `false`. -- `part`: The starting part number used if `max_filesize` is finite. +- `part`: The starting part number used when file splitting. Default: 1. - `jld2_kw`: Dict of kwargs to be passed to `jldopen` when data is written. @@ -163,7 +173,7 @@ function JLD2OutputWriter(model, outputs; filename, schedule, indices = (:, :, :), with_halos = false, array_type = Array{Float64}, - max_filesize = Inf, + file_splitting = NoFileSplitting(), overwrite_existing = false, init = noinit, including = default_included_properties(model), @@ -174,6 +184,7 @@ function JLD2OutputWriter(model, outputs; filename, schedule, mkpath(dir) filename = auto_extension(filename, ".jld2") filepath = joinpath(dir, filename) + update_file_splitting_schedule!(schedule, filepath) overwrite_existing && isfile(filepath) && rm(filepath, force=true) outputs = NamedTuple(Symbol(name) => construct_output(outputs[name], model.grid, indices, with_halos) @@ -185,7 +196,7 @@ function JLD2OutputWriter(model, outputs; filename, schedule, initialize_jld2_file!(filepath, init, jld2_kw, including, outputs, model) return JLD2OutputWriter(filepath, outputs, schedule, array_type, init, - including, part, max_filesize, overwrite_existing, verbose, jld2_kw) + including, part, file_splitting, overwrite_existing, verbose, jld2_kw) end function initialize_jld2_file!(filepath, init, jld2_kw, including, outputs, model) @@ -271,9 +282,9 @@ function write_output!(writer::JLD2OutputWriter, model) verbose && @info "Fetching time: $(prettytime(tc))" - # Start a new file if the filesize exceeds max_filesize - filesize(path) >= writer.max_filesize && start_next_file(model, writer) - path = writer.filepath # we might have a new path... + # Start a new file if the file_splitting(model) is true + writer.file_splitting(model) && start_next_file(model, writer) + update_file_splitting_schedule!(schedule, writer.filepath) # Write output from `data` verbose && @info "Writing JLD2 output $(keys(writer.outputs)) to $path..." @@ -311,9 +322,10 @@ end function start_next_file(model, writer::JLD2OutputWriter) verbose = writer.verbose - sz = filesize(writer.filepath) + verbose && @info begin - "Filesize $(pretty_filesize(sz)) has exceeded maximum file size $(pretty_filesize(writer.max_filesize))." + schedule_type = summary(writer.file_splitting) + "Splitting output because $(schedule_type) has been actuated." end if writer.part == 1 @@ -346,5 +358,7 @@ function Base.show(io::IO, ow::JLD2OutputWriter) "├── $Noutputs outputs: ", prettykeys(ow.outputs), show_averaging_schedule(averaging_schedule), "\n", "├── array type: ", show_array_type(ow.array_type), "\n", "├── including: ", ow.including, "\n", - "└── max filesize: ", pretty_filesize(ow.max_filesize)) + "├── file_splitting: ", summary(ow.file_splitting), "\n", + "└── file size: ", pretty_filesize(filesize(ow.filepath))) end + diff --git a/src/OutputWriters/output_writer_utils.jl b/src/OutputWriters/output_writer_utils.jl index 7225d69182..061197ab2d 100644 --- a/src/OutputWriters/output_writer_utils.jl +++ b/src/OutputWriters/output_writer_utils.jl @@ -6,11 +6,39 @@ using Oceananigans.Fields: AbstractField, indices, boundary_conditions, instanti using Oceananigans.BoundaryConditions: bc_str, FieldBoundaryConditions, ContinuousBoundaryFunction, DiscreteBoundaryFunction using Oceananigans.TimeSteppers: QuasiAdamsBashforth2TimeStepper, RungeKutta3TimeStepper using Oceananigans.Models.LagrangianParticleTracking: LagrangianParticles +using Oceananigans.Utils: AbstractSchedule ##### ##### Output writer utilities ##### +mutable struct FileSizeLimit <: AbstractSchedule + size_limit :: Float64 + path :: String +end + +""" + FileSizeLimit(size_limit [, path=""]) + +Return a schedule that actuates when the file at `path` exceeds +the `size_limit`. + +The `path` is automatically added and updated when `FileSizeLimit` is +used with an output writer, and should not be provided manually. +""" +FileSizeLimit(size_limit) = FileSizeLimit(size_limit, "") + +(fsl::FileSizeLimit)(model) = filesize(fsl.path) >= fsl.size_limit + +function Base.summary(fsl::FileSizeLimit) + current_size_str = pretty_filesize(filesize(fsl.path)) + size_limit_str = pretty_filesize(fsl.size_limit) + return string("FileSizeLimit(size_limit=", size_limit_str, + ", path=", fsl.path, " (", current_size_str, ")") +end + +Base.show(io::IO, fsl::FileSizeLimit) = print(io, summary(fsl)) + """ ext(ow) From b1ad9b6f40c2f0ad18a6b983d0bd9e7d7772687f Mon Sep 17 00:00:00 2001 From: "Gregory L. Wagner" Date: Fri, 15 Mar 2024 18:47:46 -0600 Subject: [PATCH 02/25] Update src/OutputWriters/jld2_output_writer.jl Co-authored-by: Navid C. Constantinou --- src/OutputWriters/jld2_output_writer.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/OutputWriters/jld2_output_writer.jl b/src/OutputWriters/jld2_output_writer.jl index 550694f8f1..175be32529 100644 --- a/src/OutputWriters/jld2_output_writer.jl +++ b/src/OutputWriters/jld2_output_writer.jl @@ -92,7 +92,7 @@ Keyword arguments `_part1`, `_part2`, etc. For example `file_splitting = FileSizeLimit(sz)` will split the output file when its size exceeds `sz`. Another example is `file_splitting = TimeInterval(30days)`, which will split files every 30 days of - simulation time. The default incurs no splitting. + simulation time. The default incurs no splitting (`NoFileSplitting()`). - `overwrite_existing`: Remove existing files if their filenames conflict. Default: `false`. From 183e729cf6824cc105d896cca45ff0793b085967 Mon Sep 17 00:00:00 2001 From: Gregory Wagner Date: Fri, 15 Mar 2024 18:50:03 -0600 Subject: [PATCH 03/25] Export FileSizeLimit and update JLD2OutputWriter test --- src/Oceananigans.jl | 2 +- test/test_jld2_output_writer.jl | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Oceananigans.jl b/src/Oceananigans.jl index 6e74ae3180..743f53a512 100644 --- a/src/Oceananigans.jl +++ b/src/Oceananigans.jl @@ -103,7 +103,7 @@ export # Output writers NetCDFOutputWriter, JLD2OutputWriter, Checkpointer, TimeInterval, IterationInterval, AveragedTimeInterval, SpecifiedTimes, - AndSchedule, OrSchedule, written_names, + FileSizeLimit, AndSchedule, OrSchedule, written_names, # Output readers FieldTimeSeries, FieldDataset, InMemory, OnDisk, diff --git a/test/test_jld2_output_writer.jl b/test/test_jld2_output_writer.jl index 905a2fa087..92b543644d 100644 --- a/test/test_jld2_output_writer.jl +++ b/test/test_jld2_output_writer.jl @@ -58,7 +58,7 @@ function test_jld2_file_splitting(arch) including = [:grid], array_type = Array{Float64}, with_halos = true, - max_filesize = 200KiB, + file_splitting = FileSizeLimit(200KiB), overwrite_existing = true) push!(simulation.output_writers, ow) From 0f5fedb38626040293e6a28439e4b30f760e2e38 Mon Sep 17 00:00:00 2001 From: josuemtzmo Date: Sat, 16 Mar 2024 11:01:59 +0100 Subject: [PATCH 04/25] implementing file splitting in netcdf --- src/OutputWriters/OutputWriters.jl | 3 +- src/OutputWriters/netcdf_output_writer.jl | 58 ++++++++++++++++------- test/test_jld2_output_writer.jl | 4 +- test/test_netcdf_output_writer.jl | 4 +- 4 files changed, 48 insertions(+), 21 deletions(-) diff --git a/src/OutputWriters/OutputWriters.jl b/src/OutputWriters/OutputWriters.jl index 810fe3e88a..25ba0a0563 100644 --- a/src/OutputWriters/OutputWriters.jl +++ b/src/OutputWriters/OutputWriters.jl @@ -3,7 +3,8 @@ module OutputWriters export JLD2OutputWriter, NetCDFOutputWriter, written_names, Checkpointer, WindowedTimeAverage, - TimeInterval, IterationInterval, WallTimeInterval, AveragedTimeInterval + TimeInterval, IterationInterval, WallTimeInterval, + AveragedTimeInterval, FileSizeLimit using CUDA diff --git a/src/OutputWriters/netcdf_output_writer.jl b/src/OutputWriters/netcdf_output_writer.jl index 0c02e6f627..e7d6cae77f 100644 --- a/src/OutputWriters/netcdf_output_writer.jl +++ b/src/OutputWriters/netcdf_output_writer.jl @@ -10,7 +10,15 @@ using Oceananigans.Utils: versioninfo_with_gpu, oceananigans_versioninfo, pretty using Oceananigans.TimeSteppers: float_or_date_time using Oceananigans.Fields: reduced_dimensions, reduced_location, location, validate_indices -mutable struct NetCDFOutputWriter{D, O, T, A} <: AbstractOutputWriter +update_file_splitting_schedule!(schedule, filepath) = nothing +update_file_splitting_schedule!(schedule::FileSizeLimit, filepath) = schedule.path = filepath + +struct NoFileSplitting end +(::NoFileSplitting)(model) = false +Base.summary(::NoFileSplitting) = "NoFileSplitting" +Base.show(io::IO, nfs::NoFileSplitting) = print(io, summary(nfs)) + +mutable struct NetCDFOutputWriter{D, O, T, A, FS} <: AbstractOutputWriter filepath :: String dataset :: D outputs :: O @@ -24,7 +32,7 @@ mutable struct NetCDFOutputWriter{D, O, T, A} <: AbstractOutputWriter overwrite_existing :: Bool deflatelevel :: Int part :: Int - max_filesize :: Float64 + file_splitting :: FS verbose :: Bool end @@ -173,7 +181,7 @@ end overwrite_existing = false, deflatelevel = 0, part = 1, - max_filesize = Inf, + file_splitting = NoFileSplitting(), verbose = false) Construct a `NetCDFOutputWriter` that writes `(label, output)` pairs in `outputs` (which should @@ -222,15 +230,19 @@ Keyword arguments and 9 means maximum compression). See [NCDatasets.jl documentation](https://alexander-barth.github.io/NCDatasets.jl/stable/variables/#Creating-a-variable) for more information. -- `part`: The starting part number used if `max_filesize` is finite. - Default: 1. - -- `max_filesize`: The writer will stop writing to the output file once the file size exceeds `max_filesize`, - and write to a new one with a consistent naming scheme ending in `part1`, `part2`, etc. - Defaults to `Inf`. +- `file_splitting`: Schedule for splitting the output file. The new files will be suffixed with + `_part1`, `_part2`, etc. For example `file_splitting = FileSizeLimit(sz)` will + split the output file when its size exceeds `sz`. Another example is + `file_splitting = TimeInterval(30days)`, which will split files every 30 days of + simulation time. The default incurs no splitting (`NoFileSplitting()`). ## Miscellaneous keywords +- `verbose`: Log what the output writer is doing with statistics on compute/write times and file sizes. + Default: `false`. + +- `part`: The starting part number used when file splitting. + - `global_attributes`: Dict of model properties to save with every file. Default: `Dict()`. - `output_attributes`: Dict of attributes to be saved with each field variable (reasonable @@ -354,12 +366,16 @@ function NetCDFOutputWriter(model, outputs; filename, schedule, overwrite_existing = nothing, deflatelevel = 0, part = 1, - max_filesize = Inf, + file_splitting = NoFileSplitting(), verbose = false) mkpath(dir) filename = auto_extension(filename, ".nc") filepath = joinpath(dir, filename) + update_file_splitting_schedule!(schedule, filepath) + + @info file_splitting + if isnothing(overwrite_existing) if isfile(filepath) overwrite_existing = false @@ -415,7 +431,7 @@ function NetCDFOutputWriter(model, outputs; filename, schedule, overwrite_existing, deflatelevel, part, - max_filesize, + file_splitting, verbose) end @@ -485,9 +501,10 @@ Write output to netcdf file `output_writer.filepath` at specified intervals. Inc every time an output is written to the file. """ function write_output!(ow::NetCDFOutputWriter, model) - # TODO allow user to split by number of snapshots, rathern than filesize. - # Start a new file if the filesize exceeds max_filesize - filesize(ow.filepath) ≥ ow.max_filesize && start_next_file(model, ow) + # Start a new file if the file_splitting(model) is true + @info ow.file_splitting(model) + ow.file_splitting(model) && start_next_file(model, ow) + update_file_splitting_schedule!(schedule, ow.filepath) ow.dataset = open(ow) @@ -556,7 +573,9 @@ function Base.show(io::IO, ow::NetCDFOutputWriter) "├── filepath: ", ow.filepath, "\n", "├── dimensions: $dims", "\n", "├── $Noutputs outputs: ", prettykeys(ow.outputs), show_averaging_schedule(averaging_schedule), "\n", - "└── array type: ", show_array_type(ow.array_type)) + "└── array type: ", show_array_type(ow.array_type), + "├── file_splitting: ", summary(ow.file_splitting), "\n", + "└── file size: ", pretty_filesize(filesize(ow.filepath))) end ##### @@ -577,11 +596,16 @@ dictify(outputs::LagrangianParticles) = Dict("particles" => outputs) default_dimensions(outputs::Dict{String,<:LagrangianParticles}, grid, indices, with_halos) = Dict("particle_id" => collect(1:length(outputs["particles"]))) +##### +##### File splitting +##### + function start_next_file(model, ow::NetCDFOutputWriter) verbose = ow.verbose - sz = filesize(ow.filepath) + verbose && @info begin - "Filesize $(pretty_filesize(sz)) has exceeded maximum file size $(pretty_filesize(ow.max_filesize))." + schedule_type = summary(ow.file_splitting) + "Splitting output because $(schedule_type) has been actuated." end if ow.part == 1 diff --git a/test/test_jld2_output_writer.jl b/test/test_jld2_output_writer.jl index 92b543644d..c1d5c0afd2 100644 --- a/test/test_jld2_output_writer.jl +++ b/test/test_jld2_output_writer.jl @@ -50,6 +50,8 @@ function test_jld2_file_splitting(arch) file["boundary_conditions/fake"] = π end + max_filesize = FileSizeLimit(200KiB) + ow = JLD2OutputWriter(model, (; u=model.velocities.u); dir = ".", filename = "test.jld2", @@ -58,7 +60,7 @@ function test_jld2_file_splitting(arch) including = [:grid], array_type = Array{Float64}, with_halos = true, - file_splitting = FileSizeLimit(200KiB), + file_splitting = max_filesize, overwrite_existing = true) push!(simulation.output_writers, ow) diff --git a/test/test_netcdf_output_writer.jl b/test/test_netcdf_output_writer.jl index d7a59143ba..a53241d731 100644 --- a/test/test_netcdf_output_writer.jl +++ b/test/test_netcdf_output_writer.jl @@ -52,7 +52,7 @@ function test_netcdf_file_splitting(arch) fake_attributes = Dict("fake_attribute"=>"fake_attribute") - max_filesize = 200KiB + max_filesize = FileSizeLimit(200KiB) ow = NetCDFOutputWriter(model, (; u=model.velocities.u); dir = ".", @@ -61,7 +61,7 @@ function test_netcdf_file_splitting(arch) array_type = Array{Float64}, with_halos = true, global_attributes = fake_attributes, - max_filesize, + file_splitting = max_filesize, overwrite_existing = true) push!(simulation.output_writers, ow) From 9276ba62858ae39ed1f3469a87aef8af47147c1b Mon Sep 17 00:00:00 2001 From: Gregory Wagner Date: Sat, 16 Mar 2024 17:22:45 -0600 Subject: [PATCH 05/25] Properly export FileSizeLimit --- src/OutputWriters/OutputWriters.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/OutputWriters/OutputWriters.jl b/src/OutputWriters/OutputWriters.jl index 810fe3e88a..0f66cee682 100644 --- a/src/OutputWriters/OutputWriters.jl +++ b/src/OutputWriters/OutputWriters.jl @@ -2,7 +2,7 @@ module OutputWriters export JLD2OutputWriter, NetCDFOutputWriter, written_names, - Checkpointer, WindowedTimeAverage, + Checkpointer, WindowedTimeAverage, FileSizeLimit, TimeInterval, IterationInterval, WallTimeInterval, AveragedTimeInterval using CUDA From 6c6e07e9de6949fa94809101247b311b38012f01 Mon Sep 17 00:00:00 2001 From: josuemtzmo Date: Sun, 17 Mar 2024 21:56:05 +0100 Subject: [PATCH 06/25] fix handeling of path writer.filepath with the file_splitting --- src/OutputWriters/jld2_output_writer.jl | 23 ++++++++++++----------- test/test_jld2_output_writer.jl | 4 +++- 2 files changed, 15 insertions(+), 12 deletions(-) diff --git a/src/OutputWriters/jld2_output_writer.jl b/src/OutputWriters/jld2_output_writer.jl index 175be32529..8e8b09ac91 100644 --- a/src/OutputWriters/jld2_output_writer.jl +++ b/src/OutputWriters/jld2_output_writer.jl @@ -10,7 +10,10 @@ default_included_properties(::ShallowWaterModel) = [:grid, :coriolis, :closure] default_included_properties(::HydrostaticFreeSurfaceModel) = [:grid, :coriolis, :buoyancy, :closure] update_file_splitting_schedule!(schedule, filepath) = nothing -update_file_splitting_schedule!(schedule::FileSizeLimit, filepath) = schedule.path = filepath + +function update_file_splitting_schedule!(schedule::FileSizeLimit, filepath) + schedule.path = filepath +end struct NoFileSplitting end (::NoFileSplitting)(model) = false @@ -184,7 +187,7 @@ function JLD2OutputWriter(model, outputs; filename, schedule, mkpath(dir) filename = auto_extension(filename, ".jld2") filepath = joinpath(dir, filename) - update_file_splitting_schedule!(schedule, filepath) + update_file_splitting_schedule!(file_splitting, filepath) overwrite_existing && isfile(filepath) && rm(filepath, force=true) outputs = NamedTuple(Symbol(name) => construct_output(outputs[name], model.grid, indices, with_halos) @@ -258,18 +261,17 @@ end function write_output!(writer::JLD2OutputWriter, model) verbose = writer.verbose - path = writer.filepath current_iteration = model.clock.iteration # Some logic to handle writing to existing files - if iteration_exists(path, current_iteration) + if iteration_exists(writer.filepath, current_iteration) if writer.overwrite_existing # Something went wrong, so we remove the file and re-initialize it. - rm(path, force=true) + rm(writer.filepath, force=true) initialize_jld2_file!(writer, model) else # nothing we can do since we were asked not to overwrite_existing, so we skip output writing - @warn "Iteration $current_iteration was found in $path. Skipping output writing (for now...)" + @warn "Iteration $current_iteration was found in $writer.filepath. Skipping output writing (for now...)" end else # ok let's do this @@ -284,14 +286,13 @@ function write_output!(writer::JLD2OutputWriter, model) # Start a new file if the file_splitting(model) is true writer.file_splitting(model) && start_next_file(model, writer) - update_file_splitting_schedule!(schedule, writer.filepath) - + update_file_splitting_schedule!(writer.file_splitting, writer.filepath) # Write output from `data` verbose && @info "Writing JLD2 output $(keys(writer.outputs)) to $path..." - start_time, old_filesize = time_ns(), filesize(path) - jld2output!(path, model.clock.iteration, model.clock.time, data, writer.jld2_kw) - end_time, new_filesize = time_ns(), filesize(path) + start_time, old_filesize = time_ns(), filesize(writer.filepath) + jld2output!(writer.filepath, model.clock.iteration, model.clock.time, data, writer.jld2_kw) + end_time, new_filesize = time_ns(), filesize(writer.filepath) verbose && @info @sprintf("Writing done: time=%s, size=%s, Δsize=%s", prettytime((end_time - start_time) / 1e9), diff --git a/test/test_jld2_output_writer.jl b/test/test_jld2_output_writer.jl index 92b543644d..7a14164fc8 100644 --- a/test/test_jld2_output_writer.jl +++ b/test/test_jld2_output_writer.jl @@ -50,6 +50,8 @@ function test_jld2_file_splitting(arch) file["boundary_conditions/fake"] = π end + filesizelimit = FileSizeLimit(200KiB) + ow = JLD2OutputWriter(model, (; u=model.velocities.u); dir = ".", filename = "test.jld2", @@ -58,7 +60,7 @@ function test_jld2_file_splitting(arch) including = [:grid], array_type = Array{Float64}, with_halos = true, - file_splitting = FileSizeLimit(200KiB), + file_splitting = filesizelimit, overwrite_existing = true) push!(simulation.output_writers, ow) From 2937dc14e2a1bd78fbe5f46483416f53c982a63c Mon Sep 17 00:00:00 2001 From: josuemtzmo Date: Sun, 17 Mar 2024 22:23:37 +0100 Subject: [PATCH 07/25] add support to file splitting by size in netCDFs --- test/test_jld2_output_writer.jl | 6 +++--- test/test_netcdf_output_writer.jl | 10 +++++----- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/test/test_jld2_output_writer.jl b/test/test_jld2_output_writer.jl index 7a14164fc8..8cadcecbec 100644 --- a/test/test_jld2_output_writer.jl +++ b/test/test_jld2_output_writer.jl @@ -69,9 +69,9 @@ function test_jld2_file_splitting(arch) run!(simulation) # Test that files has been split according to size as expected. - @test filesize("test_part1.jld2") > 200KiB - @test filesize("test_part2.jld2") > 200KiB - @test filesize("test_part3.jld2") < 200KiB + @test filesize("test_part1.jld2") > filesizelimit.size_limit + @test filesize("test_part2.jld2") > filesizelimit.size_limit + @test filesize("test_part3.jld2") < filesizelimit.size_limit for n in string.(1:3) filename = "test_part$n.jld2" diff --git a/test/test_netcdf_output_writer.jl b/test/test_netcdf_output_writer.jl index a53241d731..9beccf8ecc 100644 --- a/test/test_netcdf_output_writer.jl +++ b/test/test_netcdf_output_writer.jl @@ -52,7 +52,7 @@ function test_netcdf_file_splitting(arch) fake_attributes = Dict("fake_attribute"=>"fake_attribute") - max_filesize = FileSizeLimit(200KiB) + filesizelimit = FileSizeLimit(200KiB) ow = NetCDFOutputWriter(model, (; u=model.velocities.u); dir = ".", @@ -61,7 +61,7 @@ function test_netcdf_file_splitting(arch) array_type = Array{Float64}, with_halos = true, global_attributes = fake_attributes, - file_splitting = max_filesize, + file_splitting = filesizelimit, overwrite_existing = true) push!(simulation.output_writers, ow) @@ -70,9 +70,9 @@ function test_netcdf_file_splitting(arch) run!(simulation) # Test that files has been split according to size as expected. - @test filesize("test_part1.nc") > max_filesize - @test filesize("test_part2.nc") > max_filesize - @test filesize("test_part3.nc") < max_filesize + @test filesize("test_part1.nc") > filesizelimit.size_limit + @test filesize("test_part2.nc") > filesizelimit.size_limit + @test filesize("test_part3.nc") < filesizelimit.size_limit @test !isfile("test_part4.nc") for n in string.(1:3) From 4f9af7e5d39d5fc300d63d86662874fc818f3818 Mon Sep 17 00:00:00 2001 From: josuemtzmo Date: Sun, 17 Mar 2024 22:23:47 +0100 Subject: [PATCH 08/25] add support to file splitting by size in netCDFs --- src/OutputWriters/jld2_output_writer.jl | 11 ----------- src/OutputWriters/netcdf_output_writer.jl | 15 ++------------- src/OutputWriters/output_writer_utils.jl | 12 ++++++++++++ 3 files changed, 14 insertions(+), 24 deletions(-) diff --git a/src/OutputWriters/jld2_output_writer.jl b/src/OutputWriters/jld2_output_writer.jl index 8e8b09ac91..9771e7a509 100644 --- a/src/OutputWriters/jld2_output_writer.jl +++ b/src/OutputWriters/jld2_output_writer.jl @@ -9,17 +9,6 @@ default_included_properties(::NonhydrostaticModel) = [:grid, :coriolis, :buoyanc default_included_properties(::ShallowWaterModel) = [:grid, :coriolis, :closure] default_included_properties(::HydrostaticFreeSurfaceModel) = [:grid, :coriolis, :buoyancy, :closure] -update_file_splitting_schedule!(schedule, filepath) = nothing - -function update_file_splitting_schedule!(schedule::FileSizeLimit, filepath) - schedule.path = filepath -end - -struct NoFileSplitting end -(::NoFileSplitting)(model) = false -Base.summary(::NoFileSplitting) = "NoFileSplitting" -Base.show(io::IO, nfs::NoFileSplitting) = print(io, summary(nfs)) - mutable struct JLD2OutputWriter{O, T, D, IF, IN, FS, KW} <: AbstractOutputWriter filepath :: String outputs :: O diff --git a/src/OutputWriters/netcdf_output_writer.jl b/src/OutputWriters/netcdf_output_writer.jl index e7d6cae77f..c2b7b43d52 100644 --- a/src/OutputWriters/netcdf_output_writer.jl +++ b/src/OutputWriters/netcdf_output_writer.jl @@ -10,14 +10,6 @@ using Oceananigans.Utils: versioninfo_with_gpu, oceananigans_versioninfo, pretty using Oceananigans.TimeSteppers: float_or_date_time using Oceananigans.Fields: reduced_dimensions, reduced_location, location, validate_indices -update_file_splitting_schedule!(schedule, filepath) = nothing -update_file_splitting_schedule!(schedule::FileSizeLimit, filepath) = schedule.path = filepath - -struct NoFileSplitting end -(::NoFileSplitting)(model) = false -Base.summary(::NoFileSplitting) = "NoFileSplitting" -Base.show(io::IO, nfs::NoFileSplitting) = print(io, summary(nfs)) - mutable struct NetCDFOutputWriter{D, O, T, A, FS} <: AbstractOutputWriter filepath :: String dataset :: D @@ -372,9 +364,7 @@ function NetCDFOutputWriter(model, outputs; filename, schedule, filename = auto_extension(filename, ".nc") filepath = joinpath(dir, filename) - update_file_splitting_schedule!(schedule, filepath) - - @info file_splitting + update_file_splitting_schedule!(file_splitting, filepath) if isnothing(overwrite_existing) if isfile(filepath) @@ -502,9 +492,8 @@ every time an output is written to the file. """ function write_output!(ow::NetCDFOutputWriter, model) # Start a new file if the file_splitting(model) is true - @info ow.file_splitting(model) ow.file_splitting(model) && start_next_file(model, ow) - update_file_splitting_schedule!(schedule, ow.filepath) + update_file_splitting_schedule!(ow.file_splitting, ow.filepath) ow.dataset = open(ow) diff --git a/src/OutputWriters/output_writer_utils.jl b/src/OutputWriters/output_writer_utils.jl index 061197ab2d..42bf19537b 100644 --- a/src/OutputWriters/output_writer_utils.jl +++ b/src/OutputWriters/output_writer_utils.jl @@ -39,6 +39,18 @@ end Base.show(io::IO, fsl::FileSizeLimit) = print(io, summary(fsl)) +# Update schedule based on user input +update_file_splitting_schedule!(schedule, filepath) = nothing + +function update_file_splitting_schedule!(schedule::FileSizeLimit, filepath) + schedule.path = filepath +end + +struct NoFileSplitting end +(::NoFileSplitting)(model) = false +Base.summary(::NoFileSplitting) = "NoFileSplitting" +Base.show(io::IO, nfs::NoFileSplitting) = print(io, summary(nfs)) + """ ext(ow) From 73a94efa5207e7b1c85546a43e46475fde9640c2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Josu=C3=A9=20Mart=C3=ADnez=20Moreno?= Date: Mon, 18 Mar 2024 10:02:08 +0100 Subject: [PATCH 09/25] update warning to properly print variable. Co-authored-by: Gregory L. Wagner --- src/OutputWriters/jld2_output_writer.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/OutputWriters/jld2_output_writer.jl b/src/OutputWriters/jld2_output_writer.jl index 9771e7a509..e55298e356 100644 --- a/src/OutputWriters/jld2_output_writer.jl +++ b/src/OutputWriters/jld2_output_writer.jl @@ -260,7 +260,7 @@ function write_output!(writer::JLD2OutputWriter, model) rm(writer.filepath, force=true) initialize_jld2_file!(writer, model) else # nothing we can do since we were asked not to overwrite_existing, so we skip output writing - @warn "Iteration $current_iteration was found in $writer.filepath. Skipping output writing (for now...)" + @warn "Iteration $current_iteration was found in $(writer.filepath). Skipping output writing (for now...)" end else # ok let's do this From cd30f3e3adb2f1a4c6fbec73dccd331c156c9278 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Josu=C3=A9=20Mart=C3=ADnez=20Moreno?= Date: Mon, 18 Mar 2024 10:07:53 +0100 Subject: [PATCH 10/25] return to the use of FileSizeLimit(200KiB) in the test to make it easier to read --- test/test_jld2_output_writer.jl | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/test/test_jld2_output_writer.jl b/test/test_jld2_output_writer.jl index 8cadcecbec..201d95e2a5 100644 --- a/test/test_jld2_output_writer.jl +++ b/test/test_jld2_output_writer.jl @@ -49,8 +49,6 @@ function test_jld2_file_splitting(arch) function fake_bc_init(file, model) file["boundary_conditions/fake"] = π end - - filesizelimit = FileSizeLimit(200KiB) ow = JLD2OutputWriter(model, (; u=model.velocities.u); dir = ".", @@ -60,7 +58,7 @@ function test_jld2_file_splitting(arch) including = [:grid], array_type = Array{Float64}, with_halos = true, - file_splitting = filesizelimit, + file_splitting = FileSizeLimit(200KiB), overwrite_existing = true) push!(simulation.output_writers, ow) From 2f0e4f6e1c39b49092047b248b28060cdca07e4b Mon Sep 17 00:00:00 2001 From: josuemtzmo Date: Mon, 18 Mar 2024 10:17:15 +0100 Subject: [PATCH 11/25] update netcdf to match jld2, and add return in update_file_splitting_schedule --- src/OutputWriters/output_writer_utils.jl | 1 + test/test_netcdf_output_writer.jl | 4 +--- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/src/OutputWriters/output_writer_utils.jl b/src/OutputWriters/output_writer_utils.jl index 42bf19537b..fd9f8c943a 100644 --- a/src/OutputWriters/output_writer_utils.jl +++ b/src/OutputWriters/output_writer_utils.jl @@ -44,6 +44,7 @@ update_file_splitting_schedule!(schedule, filepath) = nothing function update_file_splitting_schedule!(schedule::FileSizeLimit, filepath) schedule.path = filepath + return nothing end struct NoFileSplitting end diff --git a/test/test_netcdf_output_writer.jl b/test/test_netcdf_output_writer.jl index 9beccf8ecc..ffa11484e8 100644 --- a/test/test_netcdf_output_writer.jl +++ b/test/test_netcdf_output_writer.jl @@ -52,8 +52,6 @@ function test_netcdf_file_splitting(arch) fake_attributes = Dict("fake_attribute"=>"fake_attribute") - filesizelimit = FileSizeLimit(200KiB) - ow = NetCDFOutputWriter(model, (; u=model.velocities.u); dir = ".", filename = "test.nc", @@ -61,7 +59,7 @@ function test_netcdf_file_splitting(arch) array_type = Array{Float64}, with_halos = true, global_attributes = fake_attributes, - file_splitting = filesizelimit, + file_splitting = FileSizeLimit(200KiB), overwrite_existing = true) push!(simulation.output_writers, ow) From 09c4abb269d9ae88e76dca5d18f42fdb53a90061 Mon Sep 17 00:00:00 2001 From: josuemtzmo Date: Mon, 18 Mar 2024 10:23:15 +0100 Subject: [PATCH 12/25] fix tests filesize tests --- test/test_jld2_output_writer.jl | 6 +++--- test/test_netcdf_output_writer.jl | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/test/test_jld2_output_writer.jl b/test/test_jld2_output_writer.jl index 201d95e2a5..cf81c987b3 100644 --- a/test/test_jld2_output_writer.jl +++ b/test/test_jld2_output_writer.jl @@ -67,9 +67,9 @@ function test_jld2_file_splitting(arch) run!(simulation) # Test that files has been split according to size as expected. - @test filesize("test_part1.jld2") > filesizelimit.size_limit - @test filesize("test_part2.jld2") > filesizelimit.size_limit - @test filesize("test_part3.jld2") < filesizelimit.size_limit + @test filesize("test_part1.jld2") > 200KiB + @test filesize("test_part2.jld2") > 200KiB + @test filesize("test_part3.jld2") < 200KiB for n in string.(1:3) filename = "test_part$n.jld2" diff --git a/test/test_netcdf_output_writer.jl b/test/test_netcdf_output_writer.jl index ffa11484e8..4bef67d293 100644 --- a/test/test_netcdf_output_writer.jl +++ b/test/test_netcdf_output_writer.jl @@ -68,9 +68,9 @@ function test_netcdf_file_splitting(arch) run!(simulation) # Test that files has been split according to size as expected. - @test filesize("test_part1.nc") > filesizelimit.size_limit - @test filesize("test_part2.nc") > filesizelimit.size_limit - @test filesize("test_part3.nc") < filesizelimit.size_limit + @test filesize("test_part1.nc") > 200KiB + @test filesize("test_part2.nc") > 200KiB + @test filesize("test_part3.nc") < 200KiB @test !isfile("test_part4.nc") for n in string.(1:3) From db94858e49e6242e1cdeb8646ce54d260853c54c Mon Sep 17 00:00:00 2001 From: "Navid C. Constantinou" Date: Fri, 22 Mar 2024 10:31:23 +0200 Subject: [PATCH 13/25] Apply suggestions from code review --- src/OutputWriters/output_writer_utils.jl | 2 +- test/test_jld2_output_writer.jl | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/src/OutputWriters/output_writer_utils.jl b/src/OutputWriters/output_writer_utils.jl index fd9f8c943a..b053e84e55 100644 --- a/src/OutputWriters/output_writer_utils.jl +++ b/src/OutputWriters/output_writer_utils.jl @@ -28,7 +28,7 @@ used with an output writer, and should not be provided manually. """ FileSizeLimit(size_limit) = FileSizeLimit(size_limit, "") -(fsl::FileSizeLimit)(model) = filesize(fsl.path) >= fsl.size_limit +(fsl::FileSizeLimit)(model) = filesize(fsl.path) ≥ fsl.size_limit function Base.summary(fsl::FileSizeLimit) current_size_str = pretty_filesize(filesize(fsl.path)) diff --git a/test/test_jld2_output_writer.jl b/test/test_jld2_output_writer.jl index cf81c987b3..9bc692c983 100644 --- a/test/test_jld2_output_writer.jl +++ b/test/test_jld2_output_writer.jl @@ -49,7 +49,6 @@ function test_jld2_file_splitting(arch) function fake_bc_init(file, model) file["boundary_conditions/fake"] = π end - ow = JLD2OutputWriter(model, (; u=model.velocities.u); dir = ".", filename = "test.jld2", From 4804fc2d123a6c2188d2c3f9acf78f17dbc897f6 Mon Sep 17 00:00:00 2001 From: "Navid C. Constantinou" Date: Fri, 22 Mar 2024 10:32:33 +0200 Subject: [PATCH 14/25] Update test_jld2_output_writer.jl --- test/test_jld2_output_writer.jl | 1 + 1 file changed, 1 insertion(+) diff --git a/test/test_jld2_output_writer.jl b/test/test_jld2_output_writer.jl index 9bc692c983..0fdeb670c6 100644 --- a/test/test_jld2_output_writer.jl +++ b/test/test_jld2_output_writer.jl @@ -69,6 +69,7 @@ function test_jld2_file_splitting(arch) @test filesize("test_part1.jld2") > 200KiB @test filesize("test_part2.jld2") > 200KiB @test filesize("test_part3.jld2") < 200KiB + @test !isfile("test_part4.jld2") for n in string.(1:3) filename = "test_part$n.jld2" From 4c1db46f9414db557c4a0fea98a15ebfaf72bd95 Mon Sep 17 00:00:00 2001 From: "Navid C. Constantinou" Date: Sat, 23 Mar 2024 13:03:51 +0200 Subject: [PATCH 15/25] fix show for NetCDFOutputWriter --- src/OutputWriters/netcdf_output_writer.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/OutputWriters/netcdf_output_writer.jl b/src/OutputWriters/netcdf_output_writer.jl index c2b7b43d52..3c7dc92476 100644 --- a/src/OutputWriters/netcdf_output_writer.jl +++ b/src/OutputWriters/netcdf_output_writer.jl @@ -562,7 +562,7 @@ function Base.show(io::IO, ow::NetCDFOutputWriter) "├── filepath: ", ow.filepath, "\n", "├── dimensions: $dims", "\n", "├── $Noutputs outputs: ", prettykeys(ow.outputs), show_averaging_schedule(averaging_schedule), "\n", - "└── array type: ", show_array_type(ow.array_type), + "└── array type: ", show_array_type(ow.array_type), "\n", "├── file_splitting: ", summary(ow.file_splitting), "\n", "└── file size: ", pretty_filesize(filesize(ow.filepath))) end From f9b082f8c6339d97b4778a466a1c3be49a7611d1 Mon Sep 17 00:00:00 2001 From: "Navid C. Constantinou" Date: Sat, 23 Mar 2024 13:03:51 +0200 Subject: [PATCH 16/25] fix doctests --- docs/src/model_setup/output_writers.md | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/docs/src/model_setup/output_writers.md b/docs/src/model_setup/output_writers.md index a9e6d72536..8880290f10 100644 --- a/docs/src/model_setup/output_writers.md +++ b/docs/src/model_setup/output_writers.md @@ -70,6 +70,8 @@ NetCDFOutputWriter scheduled on TimeInterval(1 minute): ├── dimensions: zC(16), zF(17), xC(16), yF(16), xF(16), yC(16), time(0) ├── 2 outputs: (c, u) └── array type: Array{Float64} +├── file_splitting: NoFileSplitting +└── file size: 14.9 KiB ``` ```jldoctest netcdf1 @@ -83,6 +85,8 @@ NetCDFOutputWriter scheduled on TimeInterval(1 minute): ├── dimensions: zC(1), zF(1), xC(16), yF(16), xF(16), yC(16), time(0) ├── 2 outputs: (c, u) └── array type: Array{Float64} +├── file_splitting: NoFileSplitting +└── file size: 14.9 KiB ``` ```jldoctest netcdf1 @@ -98,6 +102,8 @@ NetCDFOutputWriter scheduled on TimeInterval(1 minute): ├── dimensions: zC(16), zF(17), xC(1), yF(1), xF(1), yC(1), time(0) ├── 2 outputs: (c, u) averaged on AveragedTimeInterval(window=20 seconds, stride=1, interval=1 minute) └── array type: Array{Float64} +├── file_splitting: NoFileSplitting +└── file size: 17.6 KiB ``` `NetCDFOutputWriter` also accepts output functions that write scalars and arrays to disk, @@ -148,6 +154,8 @@ NetCDFOutputWriter scheduled on IterationInterval(1): ├── dimensions: zC(16), zF(17), xC(16), yF(16), xF(16), yC(16), time(0) ├── 3 outputs: (profile, slice, scalar) └── array type: Array{Float64} +├── file_splitting: NoFileSplitting +└── file size: 17.8 KiB ``` See [`NetCDFOutputWriter`](@ref) for more information. @@ -196,7 +204,8 @@ JLD2OutputWriter scheduled on TimeInterval(20 minutes): ├── 3 outputs: (u, v, w) ├── array type: Array{Float64} ├── including: [:grid, :coriolis, :buoyancy, :closure] -└── max filesize: Inf YiB +├── file_splitting: NoFileSplitting +└── file size: 27.4 KiB ``` and a time- and horizontal-average of tracer `c` every 20 minutes of simulation time @@ -213,7 +222,8 @@ JLD2OutputWriter scheduled on TimeInterval(20 minutes): ├── 1 outputs: c averaged on AveragedTimeInterval(window=5 minutes, stride=1, interval=20 minutes) ├── array type: Array{Float64} ├── including: [:grid, :coriolis, :buoyancy, :closure] -└── max filesize: Inf YiB +├── file_splitting: NoFileSplitting +└── file size: 17.5 KiB ``` @@ -239,7 +249,6 @@ time `interval`. The ``t_i`` specify both the end of the averaging window and th Building an `AveragedTimeInterval` that averages over a 1 day window, every 4 days, ```jldoctest averaged_time_interval -using Oceananigans.OutputWriters: AveragedTimeInterval using Oceananigans.Units schedule = AveragedTimeInterval(4days, window=1day) @@ -253,7 +262,6 @@ to time-average its outputs before writing them to disk: ```jldoctest averaged_time_interval using Oceananigans -using Oceananigans.OutputWriters: JLD2OutputWriter using Oceananigans.Units model = NonhydrostaticModel(grid=RectilinearGrid(size=(1, 1, 1), extent=(1, 1, 1))) @@ -270,5 +278,6 @@ JLD2OutputWriter scheduled on TimeInterval(4 days): ├── 3 outputs: (u, v, w) averaged on AveragedTimeInterval(window=1 day, stride=2, interval=4 days) ├── array type: Array{Float64} ├── including: [:grid, :coriolis, :buoyancy, :closure] -└── max filesize: Inf YiB +├── file_splitting: NoFileSplitting +└── file size: 26.7 KiB ``` From 527ac3184dbfc1aeb2104ba1cf4b5e8fb03cf518 Mon Sep 17 00:00:00 2001 From: "Navid C. Constantinou" Date: Sat, 23 Mar 2024 18:18:56 +0200 Subject: [PATCH 17/25] Update src/OutputWriters/netcdf_output_writer.jl Co-authored-by: Gregory L. Wagner --- src/OutputWriters/netcdf_output_writer.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/OutputWriters/netcdf_output_writer.jl b/src/OutputWriters/netcdf_output_writer.jl index 3c7dc92476..29cd078fe3 100644 --- a/src/OutputWriters/netcdf_output_writer.jl +++ b/src/OutputWriters/netcdf_output_writer.jl @@ -594,7 +594,7 @@ function start_next_file(model, ow::NetCDFOutputWriter) verbose && @info begin schedule_type = summary(ow.file_splitting) - "Splitting output because $(schedule_type) has been actuated." + "Splitting output because $(schedule_type) is activated." end if ow.part == 1 From 78998bc4518ef7a599a45b20396299e2bfd0f8ce Mon Sep 17 00:00:00 2001 From: "Navid C. Constantinou" Date: Sat, 23 Mar 2024 18:19:38 +0200 Subject: [PATCH 18/25] Update src/OutputWriters/jld2_output_writer.jl Co-authored-by: Gregory L. Wagner --- src/OutputWriters/jld2_output_writer.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/OutputWriters/jld2_output_writer.jl b/src/OutputWriters/jld2_output_writer.jl index e55298e356..6a4f96aa02 100644 --- a/src/OutputWriters/jld2_output_writer.jl +++ b/src/OutputWriters/jld2_output_writer.jl @@ -315,7 +315,7 @@ function start_next_file(model, writer::JLD2OutputWriter) verbose && @info begin schedule_type = summary(writer.file_splitting) - "Splitting output because $(schedule_type) has been actuated." + "Splitting output because $(schedule_type) is activated." end if writer.part == 1 From 60de913e2ce0553da0d1a9be8b644ab24df80872 Mon Sep 17 00:00:00 2001 From: "Navid C. Constantinou" Date: Sat, 23 Mar 2024 21:22:46 +0200 Subject: [PATCH 19/25] fix doctests --- src/OutputWriters/jld2_output_writer.jl | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/src/OutputWriters/jld2_output_writer.jl b/src/OutputWriters/jld2_output_writer.jl index e55298e356..3d41d6751f 100644 --- a/src/OutputWriters/jld2_output_writer.jl +++ b/src/OutputWriters/jld2_output_writer.jl @@ -54,7 +54,7 @@ Keyword arguments ## Filenaming - `filename` (required): Descriptive filename. `".jld2"` is appended to `filename` in the file path - if `filename` does not end in `".jld2"`. + if `filename` does not end in `".jld2"`. - `dir`: Directory to save output to. Default: `"."` (current working directory). @@ -140,7 +140,8 @@ JLD2OutputWriter scheduled on TimeInterval(20 minutes): ├── 3 outputs: (u, v, w) ├── array type: Array{Float64} ├── including: [:grid, :coriolis, :buoyancy, :closure] -└── max filesize: Inf YiB +├── file_splitting: NoFileSplitting +└── file size: 27.4 KiB ``` and a time- and horizontal-average of tracer ``c`` every 20 minutes of simulation time @@ -157,7 +158,8 @@ JLD2OutputWriter scheduled on TimeInterval(20 minutes): ├── 1 outputs: c averaged on AveragedTimeInterval(window=5 minutes, stride=1, interval=20 minutes) ├── array type: Array{Float64} ├── including: [:grid, :coriolis, :buoyancy, :closure] -└── max filesize: Inf YiB +├── file_splitting: NoFileSplitting +└── file size: 17.5 KiB ``` """ function JLD2OutputWriter(model, outputs; filename, schedule, @@ -178,7 +180,7 @@ function JLD2OutputWriter(model, outputs; filename, schedule, filepath = joinpath(dir, filename) update_file_splitting_schedule!(file_splitting, filepath) overwrite_existing && isfile(filepath) && rm(filepath, force=true) - + outputs = NamedTuple(Symbol(name) => construct_output(outputs[name], model.grid, indices, with_halos) for name in keys(outputs)) @@ -186,7 +188,7 @@ function JLD2OutputWriter(model, outputs; filename, schedule, schedule, outputs = time_average_outputs(schedule, outputs, model) initialize_jld2_file!(filepath, init, jld2_kw, including, outputs, model) - + return JLD2OutputWriter(filepath, outputs, schedule, array_type, init, including, part, file_splitting, overwrite_existing, verbose, jld2_kw) end @@ -312,7 +314,7 @@ end function start_next_file(model, writer::JLD2OutputWriter) verbose = writer.verbose - + verbose && @info begin schedule_type = summary(writer.file_splitting) "Splitting output because $(schedule_type) has been actuated." @@ -331,7 +333,7 @@ function start_next_file(model, writer::JLD2OutputWriter) verbose && @info "Now writing to: $(writer.filepath)" initialize_jld2_file!(writer, model) - + return nothing end @@ -351,4 +353,3 @@ function Base.show(io::IO, ow::JLD2OutputWriter) "├── file_splitting: ", summary(ow.file_splitting), "\n", "└── file size: ", pretty_filesize(filesize(ow.filepath))) end - From d8233b0df19d3f55a623a6f1aef4acb80b95f97c Mon Sep 17 00:00:00 2001 From: "Navid C. Constantinou" Date: Sat, 23 Mar 2024 21:27:48 +0200 Subject: [PATCH 20/25] fix doctests --- src/OutputWriters/netcdf_output_writer.jl | 8 ++++++++ src/OutputWriters/windowed_time_average.jl | 6 +++--- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/src/OutputWriters/netcdf_output_writer.jl b/src/OutputWriters/netcdf_output_writer.jl index 29cd078fe3..f175cb7309 100644 --- a/src/OutputWriters/netcdf_output_writer.jl +++ b/src/OutputWriters/netcdf_output_writer.jl @@ -267,6 +267,8 @@ NetCDFOutputWriter scheduled on TimeInterval(1 minute): ├── dimensions: zC(16), zF(17), xC(16), yF(16), xF(16), yC(16), time(0) ├── 2 outputs: (c, u) └── array type: Array{Float64} +├── file_splitting: NoFileSplitting +└── file size: 14.9 KiB ``` ```jldoctest netcdf1 @@ -280,6 +282,8 @@ NetCDFOutputWriter scheduled on TimeInterval(1 minute): ├── dimensions: zC(1), zF(1), xC(16), yF(16), xF(16), yC(16), time(0) ├── 2 outputs: (c, u) └── array type: Array{Float64} +├── file_splitting: NoFileSplitting +└── file size: 14.9 KiB ``` ```jldoctest netcdf1 @@ -295,6 +299,8 @@ NetCDFOutputWriter scheduled on TimeInterval(1 minute): ├── dimensions: zC(16), zF(17), xC(1), yF(1), xF(1), yC(1), time(0) ├── 2 outputs: (c, u) averaged on AveragedTimeInterval(window=20 seconds, stride=1, interval=1 minute) └── array type: Array{Float64} +├── file_splitting: NoFileSplitting +└── file size: 17.6 KiB ``` `NetCDFOutputWriter` also accepts output functions that write scalars and arrays to disk, @@ -345,6 +351,8 @@ NetCDFOutputWriter scheduled on IterationInterval(1): ├── dimensions: zC(16), zF(17), xC(16), yF(16), xF(16), yC(16), time(0) ├── 3 outputs: (profile, slice, scalar) └── array type: Array{Float64} +├── file_splitting: NoFileSplitting +└── file size: 17.8 KiB ``` """ function NetCDFOutputWriter(model, outputs; filename, schedule, diff --git a/src/OutputWriters/windowed_time_average.jl b/src/OutputWriters/windowed_time_average.jl index 0a5c29b372..1a49615f48 100644 --- a/src/OutputWriters/windowed_time_average.jl +++ b/src/OutputWriters/windowed_time_average.jl @@ -61,8 +61,7 @@ to time-average its outputs before writing them to disk: ```jldoctest averaged_time_interval using Oceananigans -using Oceananigans.OutputWriters: JLD2OutputWriter -using Oceananigans.Utils: minutes +using Oceananigans.Units model = NonhydrostaticModel(grid=RectilinearGrid(size=(1, 1, 1), extent=(1, 1, 1))) @@ -78,7 +77,8 @@ JLD2OutputWriter scheduled on TimeInterval(4 days): ├── 3 outputs: (u, v, w) averaged on AveragedTimeInterval(window=2 days, stride=2, interval=4 days) ├── array type: Array{Float64} ├── including: [:grid, :coriolis, :buoyancy, :closure] -└── max filesize: Inf YiB +├── file_splitting: NoFileSplitting +└── file size: 26.7 KiB ``` """ function AveragedTimeInterval(interval; window=interval, stride=1) From 28cc203d9e3a90eb0599b339fce165da8c9b2845 Mon Sep 17 00:00:00 2001 From: "Navid C. Constantinou" Date: Sun, 24 Mar 2024 07:14:01 +0200 Subject: [PATCH 21/25] fix doctest --- src/OutputWriters/netcdf_output_writer.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/OutputWriters/netcdf_output_writer.jl b/src/OutputWriters/netcdf_output_writer.jl index f175cb7309..919e6ba0a0 100644 --- a/src/OutputWriters/netcdf_output_writer.jl +++ b/src/OutputWriters/netcdf_output_writer.jl @@ -283,7 +283,7 @@ NetCDFOutputWriter scheduled on TimeInterval(1 minute): ├── 2 outputs: (c, u) └── array type: Array{Float64} ├── file_splitting: NoFileSplitting -└── file size: 14.9 KiB +└── file size: 14.8 KiB ``` ```jldoctest netcdf1 From 97fa9bbb4c88f39f5fb93e359b2de5e818db6711 Mon Sep 17 00:00:00 2001 From: "Navid C. Constantinou" Date: Sun, 24 Mar 2024 07:14:01 +0200 Subject: [PATCH 22/25] cleanup unecessary imports --- benchmark/benchmark_multi_GPU.jl | 1 - validation/barotropic_gyre/barotropic_gyre.jl | 8 +------- validation/solid_body_rotation/solid_body_rotation.jl | 1 - .../solid_body_rotation/solid_body_tracer_advection.jl | 1 - 4 files changed, 1 insertion(+), 10 deletions(-) diff --git a/benchmark/benchmark_multi_GPU.jl b/benchmark/benchmark_multi_GPU.jl index 0abc8b21ca..39886efa3c 100644 --- a/benchmark/benchmark_multi_GPU.jl +++ b/benchmark/benchmark_multi_GPU.jl @@ -12,7 +12,6 @@ using Oceananigans.Models.HydrostaticFreeSurfaceModels: ExplicitFreeSurface using Oceananigans.Utils: prettytime, hours -using Oceananigans.OutputWriters: JLD2OutputWriter, TimeInterval, IterationInterval using Oceananigans.MultiRegion using Oceananigans.TurbulenceClosures: VerticallyImplicitTimeDiscretization diff --git a/validation/barotropic_gyre/barotropic_gyre.jl b/validation/barotropic_gyre/barotropic_gyre.jl index db641b63c5..c522983a7a 100644 --- a/validation/barotropic_gyre/barotropic_gyre.jl +++ b/validation/barotropic_gyre/barotropic_gyre.jl @@ -3,8 +3,6 @@ using Oceananigans using Oceananigans.Grids -using Oceananigans.Coriolis: HydrostaticSphericalCoriolis - using Oceananigans.Advection: EnergyConserving, EnstrophyConserving using Oceananigans.Models.HydrostaticFreeSurfaceModels: @@ -13,11 +11,7 @@ using Oceananigans.Models.HydrostaticFreeSurfaceModels: ExplicitFreeSurface, ImplicitFreeSurface - -using Oceananigans.Utils: prettytime, hours, day, days, years -using Oceananigans.OutputWriters: JLD2OutputWriter, TimeInterval, IterationInterval - -using Oceananigans.ImmersedBoundaries: ImmersedBoundaryGrid, GridFittedBoundary, GridFittedBottom +using Oceananigans.Units using Statistics using JLD2 diff --git a/validation/solid_body_rotation/solid_body_rotation.jl b/validation/solid_body_rotation/solid_body_rotation.jl index 98e52ed16f..c3811cd4b7 100644 --- a/validation/solid_body_rotation/solid_body_rotation.jl +++ b/validation/solid_body_rotation/solid_body_rotation.jl @@ -27,7 +27,6 @@ using Oceananigans.Models.HydrostaticFreeSurfaceModels: ExplicitFreeSurface using Oceananigans.Utils: prettytime, hours -using Oceananigans.OutputWriters: JLD2OutputWriter, TimeInterval, IterationInterval using Statistics using JLD2 diff --git a/validation/solid_body_rotation/solid_body_tracer_advection.jl b/validation/solid_body_rotation/solid_body_tracer_advection.jl index 397aaccc54..fe05e17a4a 100644 --- a/validation/solid_body_rotation/solid_body_tracer_advection.jl +++ b/validation/solid_body_rotation/solid_body_tracer_advection.jl @@ -27,7 +27,6 @@ using Oceananigans.Models.HydrostaticFreeSurfaceModels: PrescribedVelocityFields using Oceananigans.Utils: prettytime, hours -using Oceananigans.OutputWriters: JLD2OutputWriter, TimeInterval, IterationInterval using JLD2 using Printf From 71dab3a1981436ea815daff7586b0ecbe383c6ff Mon Sep 17 00:00:00 2001 From: "Navid C. Constantinou" Date: Sun, 24 Mar 2024 07:14:50 +0200 Subject: [PATCH 23/25] fix doctest --- src/OutputWriters/netcdf_output_writer.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/OutputWriters/netcdf_output_writer.jl b/src/OutputWriters/netcdf_output_writer.jl index 919e6ba0a0..0fb468a117 100644 --- a/src/OutputWriters/netcdf_output_writer.jl +++ b/src/OutputWriters/netcdf_output_writer.jl @@ -268,7 +268,7 @@ NetCDFOutputWriter scheduled on TimeInterval(1 minute): ├── 2 outputs: (c, u) └── array type: Array{Float64} ├── file_splitting: NoFileSplitting -└── file size: 14.9 KiB +└── file size: 14.8 KiB ``` ```jldoctest netcdf1 From 4c1cb928c70e65dfd0fdf421755f9bb949009a5e Mon Sep 17 00:00:00 2001 From: "Navid C. Constantinou" Date: Sun, 24 Mar 2024 07:15:55 +0200 Subject: [PATCH 24/25] fix doctests --- docs/src/model_setup/output_writers.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/src/model_setup/output_writers.md b/docs/src/model_setup/output_writers.md index 8880290f10..8a68ba52be 100644 --- a/docs/src/model_setup/output_writers.md +++ b/docs/src/model_setup/output_writers.md @@ -71,7 +71,7 @@ NetCDFOutputWriter scheduled on TimeInterval(1 minute): ├── 2 outputs: (c, u) └── array type: Array{Float64} ├── file_splitting: NoFileSplitting -└── file size: 14.9 KiB +└── file size: 14.8 KiB ``` ```jldoctest netcdf1 @@ -86,7 +86,7 @@ NetCDFOutputWriter scheduled on TimeInterval(1 minute): ├── 2 outputs: (c, u) └── array type: Array{Float64} ├── file_splitting: NoFileSplitting -└── file size: 14.9 KiB +└── file size: 14.8 KiB ``` ```jldoctest netcdf1 From 6c0e974e17dc05ea22c96d1ee291cd7227261026 Mon Sep 17 00:00:00 2001 From: "Navid C. Constantinou" Date: Sun, 24 Mar 2024 11:31:06 +0200 Subject: [PATCH 25/25] fix doctests --- docs/src/model_setup/output_writers.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/src/model_setup/output_writers.md b/docs/src/model_setup/output_writers.md index 8a68ba52be..20ce934d17 100644 --- a/docs/src/model_setup/output_writers.md +++ b/docs/src/model_setup/output_writers.md @@ -249,6 +249,7 @@ time `interval`. The ``t_i`` specify both the end of the averaging window and th Building an `AveragedTimeInterval` that averages over a 1 day window, every 4 days, ```jldoctest averaged_time_interval +using Oceananigans using Oceananigans.Units schedule = AveragedTimeInterval(4days, window=1day)