One .pdf instead of 36 .png long_runs artifacts

Long runs where producing 36 artifacts (.png files), this commit
tidy this up by gathering them into one .pdf instead.

.buildkite/Project.toml

@@ -29,6 +29,7 @@ MPI = "da04e1cc-30fd-572f-bb4f-1f8673147195"
 NCDatasets = "85f8d34a-cbdd-5861-8df4-14fed0d494ab"
 NLsolve = "2774e3e8-f4cf-5e23-947b-6d7e65073b56"
 Plots = "91a5bcdd-55d7-5caf-9e0b-520d859cae80"
+Poppler_jll = "9c32591e-4766-534b-9725-b71a8799265b"
 ProfileCanvas = "efd6af41-a80b-495e-886c-e51b0c7d77a3"
 RootSolvers = "7181ea78-2dcb-4de3-ab41-2b8ab5a31e74"
 SafeTestsets = "1bc83da4-3b8d-516f-aca4-4fe02f6d838f"

.buildkite/longruns_gpu/pipeline.yml

@@ -34,7 +34,7 @@ steps:
       - label: ":seedling: Soil-Canopy"
         command:
           - julia --color=yes --project=.buildkite experiments/long_runs/land.jl
-        artifact_paths: "land_longrun_gpu/*png"
+        artifact_paths: "land_longrun_gpu/*pdf"
         agents:
           slurm_gpus: 1
           slurm_time: 03:00:00
@@ -44,7 +44,7 @@ steps:
       - label: ":sun: California regional simulation"
         command:
           - julia --color=yes --project=.buildkite experiments/long_runs/land_region.jl
-        artifact_paths: "california_longrun_gpu/*png"
+        artifact_paths: "california_longrun_gpu/*pdf"
         agents:
           slurm_gpus: 1
           slurm_time: 00:30:00
@@ -54,7 +54,7 @@ steps:
       - label: "Soil"
         command:
           - julia --color=yes --project=.buildkite experiments/long_runs/soil.jl
-        artifact_paths: "soil_longrun_gpu/*png"
+        artifact_paths: "soil_longrun_gpu/*pdf"
         agents:
           slurm_gpus: 1
           slurm_time: 01:30:00
@@ -64,7 +64,7 @@ steps:
       - label: "Global bucket simulation"
         command:
           - julia --color=yes --project=.buildkite experiments/long_runs/bucket.jl
-        artifact_paths: "bucket_longrun_gpu/*png"
+        artifact_paths: "bucket_longrun_gpu/*pdf"
         agents:
           slurm_gpus: 1
           slurm_time: 00:30:00

experiments/long_runs/bucket.jl

@@ -1,7 +1,7 @@
 # # Global bucket run
 
 # The code sets up and runs the bucket model  on a spherical domain,
-# using ERA5 data. 
+# using ERA5 data.
 
 # Simulation Setup
 # Number of spatial elements: 101 in horizontal, 5 in vertical
@@ -38,6 +38,8 @@ using CairoMakie
 using Dates
 import NCDatasets
 
+using Poppler_jll: pdfunite
+
 const FT = Float64;
 time_interpolation_method = LinearInterpolation(PeriodicCalendar())
 regridder_type = :InterpolationsRegridder
@@ -167,21 +169,27 @@ setup_and_solve_problem(; greet = true);
 simdir = ClimaAnalysis.SimDir(outdir)
 short_names =
     ["swa", "rn", "tsfc", "qsfc", "lhf", "shf", "wsoil", "wsfc", "ssfc"]
-for short_name in short_names
-    var = get(simdir; short_name)
-    times = ClimaAnalysis.times(var)
-    for t in times
+mktempdir(root_path) do tmpdir
+    for short_name in short_names
         var = get(simdir; short_name)
-        fig = CairoMakie.Figure(size = (800, 600))
-        kwargs = ClimaAnalysis.has_altitude(var) ? Dict(:z => 1) : Dict()
-        viz.heatmap2D_on_globe!(
-            fig,
-            ClimaAnalysis.slice(var, time = t; kwargs...),
-            mask = viz.oceanmask(),
-            more_kwargs = Dict(
-                :mask => ClimaAnalysis.Utils.kwargs(color = :white),
-            ),
-        )
-        CairoMakie.save(joinpath(root_path, "$(short_name)_$t.png"), fig)
+        times = ClimaAnalysis.times(var)
+        for t in times
+            var = get(simdir; short_name)
+            fig = CairoMakie.Figure(size = (800, 600))
+            kwargs = ClimaAnalysis.has_altitude(var) ? Dict(:z => 1) : Dict()
+            viz.heatmap2D_on_globe!(
+                fig,
+                ClimaAnalysis.slice(var, time = t; kwargs...),
+                mask = viz.oceanmask(),
+                more_kwargs = Dict(
+                    :mask => ClimaAnalysis.Utils.kwargs(color = :white),
+                ),
+            )
+            CairoMakie.save(joinpath(tmpdir, "$(short_name)_$t.pdf"), fig)
+        end
+    end
+    figures = readdir(tmpdir, join = true)
+    pdfunite() do unite
+        run(Cmd([unite, figures..., "figures.pdf"]))
     end
 end

experiments/long_runs/land.jl

@@ -46,6 +46,8 @@ import GeoMakie
 using Dates
 import NCDatasets
 
+using Poppler_jll: pdfunite
+
 const FT = Float64;
 time_interpolation_method = LinearInterpolation(PeriodicCalendar())
 regridder_type = :InterpolationsRegridder
@@ -596,20 +598,26 @@ setup_and_solve_problem(; greet = true);
 simdir = ClimaAnalysis.SimDir(outdir)
 short_names =
     ["gpp", "ct", "lai", "swc", "si", "swa", "lwu", "et", "er", "sr", "sif"]
-for short_name in short_names
-    var = get(simdir; short_name)
-    times = ClimaAnalysis.times(var)
-    for t in times
-        fig = CairoMakie.Figure(size = (800, 600))
-        kwargs = ClimaAnalysis.has_altitude(var) ? Dict(:z => 1) : Dict()
-        viz.heatmap2D_on_globe!(
-            fig,
-            ClimaAnalysis.slice(var, time = t; kwargs...),
-            mask = viz.oceanmask(),
-            more_kwargs = Dict(
-                :mask => ClimaAnalysis.Utils.kwargs(color = :white),
-            ),
-        )
-        CairoMakie.save(joinpath(root_path, "$(short_name)_$t.png"), fig)
+mktempdir(root_path) do tmpdir
+    for short_name in short_names
+        var = get(simdir; short_name)
+        times = ClimaAnalysis.times(var)
+        for t in times
+            fig = CairoMakie.Figure(size = (800, 600))
+            kwargs = ClimaAnalysis.has_altitude(var) ? Dict(:z => 1) : Dict()
+            viz.heatmap2D_on_globe!(
+                fig,
+                ClimaAnalysis.slice(var, time = t; kwargs...),
+                mask = viz.oceanmask(),
+                more_kwargs = Dict(
+                    :mask => ClimaAnalysis.Utils.kwargs(color = :white),
+                ),
+            )
+            CairoMakie.save(joinpath(tmpdir, "$(short_name)_$t.pdf"), fig)
+        end
+    end
+    figures = readdir(tmpdir, join = true)
+    pdfunite() do unite
+        run(Cmd([unite, figures..., "figures.pdf"]))
     end
 end

experiments/long_runs/land_region.jl

@@ -45,6 +45,8 @@ using CairoMakie
 using Dates
 import NCDatasets
 
+using Poppler_jll: pdfunite
+
 const FT = Float64;
 time_interpolation_method = LinearInterpolation(PeriodicCalendar())
 regridder_type = :InterpolationsRegridder
@@ -599,16 +601,22 @@ setup_and_solve_problem(; greet = true);
 #### ClimaAnalysis ####
 simdir = ClimaAnalysis.SimDir(outdir)
 short_names = ["gpp", "ct", "swc", "si"]
-for short_name in short_names
-    var = get(simdir; short_name)
-    times = ClimaAnalysis.times(var)
-    for t in times
-        fig = CairoMakie.Figure(size = (800, 600))
-        kwargs = ClimaAnalysis.has_altitude(var) ? Dict(:z => 1) : Dict()
-        tmp = ClimaAnalysis.slice(var, time = t; kwargs...)
-        if !all(isnan.(tmp.data))
-            viz.heatmap2D!(fig, tmp)
-            CairoMakie.save(joinpath(root_path, "$(short_name)_$t.png"), fig)
+mktempdir(root_path) do tmpdir
+    for short_name in short_names
+        var = get(simdir; short_name)
+        times = ClimaAnalysis.times(var)
+        for t in times
+            fig = CairoMakie.Figure(size = (800, 600))
+            kwargs = ClimaAnalysis.has_altitude(var) ? Dict(:z => 1) : Dict()
+            tmp = ClimaAnalysis.slice(var, time = t; kwargs...)
+            if !all(isnan.(tmp.data))
+                viz.heatmap2D!(fig, tmp)
+                CairoMakie.save(joinpath(tmpdir, "$(short_name)_$t.pdf"), fig)
+            end
         end
     end
+    figures = readdir(tmpdir, join = true)
+    pdfunite() do unite
+        run(Cmd([unite, figures..., "figures.pdf"]))
+    end
 end

experiments/long_runs/soil.jl

@@ -44,6 +44,8 @@ using CairoMakie
 using Dates
 import NCDatasets
 
+using Poppler_jll: pdfunite
+
 const FT = Float64;
 time_interpolation_method = LinearInterpolation(PeriodicCalendar())
 regridder_type = :InterpolationsRegridder
@@ -365,21 +367,27 @@ setup_and_solve_problem(; greet = true);
 #### ClimaAnalysis ####
 simdir = ClimaAnalysis.SimDir(outdir)
 short_names = ["swc", "si", "sie"]
-for short_name in short_names
-    var = get(simdir; short_name)
-    times = ClimaAnalysis.times(var)
-    for t in times
+mktempdir(root_path) do tmpdir
+    for short_name in short_names
         var = get(simdir; short_name)
-        fig = CairoMakie.Figure(size = (800, 600))
-        kwargs = ClimaAnalysis.has_altitude(var) ? Dict(:z => 1) : Dict()
-        viz.heatmap2D_on_globe!(
-            fig,
-            ClimaAnalysis.slice(var, time = t; kwargs...),
-            mask = viz.oceanmask(),
-            more_kwargs = Dict(
-                :mask => ClimaAnalysis.Utils.kwargs(color = :white),
-            ),
-        )
-        CairoMakie.save(joinpath(root_path, "$(short_name)_$t.png"), fig)
+        times = ClimaAnalysis.times(var)
+        for t in times
+            var = get(simdir; short_name)
+            fig = CairoMakie.Figure(size = (800, 600))
+            kwargs = ClimaAnalysis.has_altitude(var) ? Dict(:z => 1) : Dict()
+            viz.heatmap2D_on_globe!(
+                fig,
+                ClimaAnalysis.slice(var, time = t; kwargs...),
+                mask = viz.oceanmask(),
+                more_kwargs = Dict(
+                    :mask => ClimaAnalysis.Utils.kwargs(color = :white),
+                ),
+            )
+            CairoMakie.save(joinpath(tmpdir, "$(short_name)_$t.pdf"), fig)
+        end
+    end
+    figures = readdir(tmpdir, join = true)
+    pdfunite() do unite
+        run(Cmd([unite, figures..., "figures.pdf"]))
     end
 end

test/Project.toml

@@ -24,6 +24,7 @@ NCDatasets = "85f8d34a-cbdd-5861-8df4-14fed0d494ab"
 NLsolve = "2774e3e8-f4cf-5e23-947b-6d7e65073b56"
 Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"
 SafeTestsets = "1bc83da4-3b8d-516f-aca4-4fe02f6d838f"
+SciMLBase = "0bca4576-84f4-4d90-8ffe-ffa030f20462"
 StaticArrays = "90137ffa-7385-5640-81b9-e52037218182"
 Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2"
 StatsBase = "2913bbd2-ae8a-5f71-8c99-4fb6c76f3a91"

test/diagnostics/diagnostics_tests.jl

@@ -1,14 +1,17 @@
 using Test
 using ClimaLand
 using ClimaLand.Diagnostics: @with_error
+import ClimaComms
+ClimaComms.@import_required_backends
+import ClimaParams
+import SciMLBase
+import ClimaTimeSteppers
+import ClimaDiagnostics
+using Dates
+using Statistics
 
 @test isdefined(ClimaLand.Diagnostics, :compute_sw_albedo!)
 
-@test !hasmethod(
-    ClimaLand.Diagnostics.compute_sw_albedo!,
-    (Any, Any, Any, Any, Any),
-)
-
 # Define some diagnostics for a DummyModel
 
 @test ClimaLand.Diagnostics.ALL_DIAGNOSTICS isa Dict
@@ -30,17 +33,91 @@ ClimaLand.Diagnostics.add_diagnostic_variable!(
 
 @test length(ClimaLand.Diagnostics.ALL_DIAGNOSTICS) == 1
 
-ClimaLand.Diagnostics.define_diagnostics!(DummyModel())
+# First, run a simulation for 1 hour
+FT = Float32
+seconds = 1.0;
+minutes = 60seconds;
+hours = 60minutes;
+t0 = 0.0
+Δt = 1minutes
+tf = 2hours
+bucket_domain = ClimaLand.SphericalShell(;
+    radius = FT(100),
+    depth = FT(3.5),
+    nelements = (1, 10),
+    npolynomial = 1,
+)
+
+bucket_atmos, bucket_rad = ClimaLand.prescribed_analytic_forcing(FT)
+τc = FT(1.0)
+α_bareground_func = (coordinate_point) -> 0.2
+α_snow = FT(0.8)
+z_0m = FT(1e-2)
+z_0b = FT(1e-3)
+albedo = ClimaLand.Bucket.PrescribedBaregroundAlbedo{FT}(
+    α_snow,
+    α_bareground_func,
+    bucket_domain.space.surface,
+)
+bucket_parameters =
+    ClimaLand.Bucket.BucketModelParameters(FT; albedo, z_0m, z_0b, τc)
+
+model = ClimaLand.Bucket.BucketModel(
+    parameters = bucket_parameters,
+    domain = bucket_domain,
+    atmosphere = bucket_atmos,
+    radiation = bucket_rad,
+)
+
+Y, p, coords = ClimaLand.initialize(model)
+Y.bucket.T .= 280.0
+Y.bucket.W .= 0.5 # no moisture
+Y.bucket.Ws .= 0.5 # no runoff
+Y.bucket.σS .= 0.0
 
-# Just to trigger the error
-out = Y = p = t = land_model = nothing
+exp_tendency! = ClimaLand.make_exp_tendency(model)
+set_initial_cache! = ClimaLand.make_set_initial_cache(model)
+set_initial_cache!(p, Y, t0)
 
-@test_throws ErrorException("Cannot compute albedo with model = Nothing") ClimaLand.Diagnostics.compute_albedo!(
-    out,
+prob = SciMLBase.ODEProblem(
+    ClimaTimeSteppers.ClimaODEFunction((T_exp!) = exp_tendency!),
     Y,
+    (t0, tf),
     p,
-    t,
-    land_model,
 )
 
-@test_throws ErrorException ClimaLand.Diagnostics.get_diagnostic_variable("Foo")
+# ClimaDiagnostics
+
+ClimaLand.Diagnostics.define_diagnostics!(model)
+diags = ["rn", "lhf"]
+
+tmpdir = mktempdir(".")
+nc_writer =
+    ClimaDiagnostics.Writers.NetCDFWriter(bucket_domain.space.surface, tmpdir)
+
+out = ClimaLand.Diagnostics.hourly_averages(
+    FT,
+    diags...;
+    output_writer = nc_writer,
+    reference_date = DateTime(2005),
+)
+
+diagnostic_handler = ClimaDiagnostics.DiagnosticsHandler(out, Y, p, t0; dt = Δt)
+
+diag_cb = ClimaDiagnostics.DiagnosticsCallback(diagnostic_handler)
+
+updateat = collect(t0:Δt:tf);
+drivers = ClimaLand.get_drivers(model)
+updatefunc = ClimaLand.make_update_drivers(drivers)
+driver_cb = ClimaLand.DriverUpdateCallback(updateat, updatefunc)
+cb = SciMLBase.CallbackSet(driver_cb, diag_cb)
+timestepper = ClimaTimeSteppers.RK4()
+ode_algo = ClimaTimeSteppers.ExplicitAlgorithm(timestepper)
+SciMLBase.solve(prob, ode_algo; dt = Δt, callback = cb)
+
+using ClimaAnalysis
+simdir = ClimaAnalysis.SimDir(tmpdir)
+rn = get(simdir; short_name = "rn")
+@test readdir(tmpdir) == ["lhf_1h_average.nc", "rn_1h_average.nc"]
+@test length(rn.dims) == 3
+@test mean(rn.data) != 0.0

test/runtests.jl

@@ -108,3 +108,8 @@ end
 @safetestset "Integrated soil and snow" begin
     include("integrated/soil_snow.jl")
 end
+
+# Diagnostics
+@safetestset "Diagnostics" begin
+    include("diagnostics/diagnostics_tests.jl")
+end