Merge remote-tracking branch 'origin/main' into improve-html

Author: AnHeuermann

src/compare.jl

@@ -294,12 +294,17 @@ function compare_with_reference(
     isempty(times) && return 0, 0, 0, ""
 
     # Determine which signals to compare: prefer comparisonSignals.txt
-    sig_file = joinpath(dirname(ref_csv_path), "comparisonSignals.txt")
-    signals  = if isfile(sig_file)
-        filter(s -> lowercase(s) != "time" && !isempty(s), strip.(readlines(sig_file)))
+    sig_file           = joinpath(dirname(ref_csv_path), "comparisonSignals.txt")
+    using_sig_file     = isfile(sig_file)
+    signals = if using_sig_file
+        sigs = filter(s -> lowercase(s) != "time" && !isempty(s), strip.(readlines(sig_file)))
+        sigs_missing = filter(s -> !haskey(ref_data, s), sigs)
+        isempty(sigs_missing) || error("Signal(s) listed in comparisonSignals.txt not present in reference CSV: $(join(sigs_missing, ", "))")
+        sigs
     else
         filter(k -> lowercase(k) != "time", collect(keys(ref_data)))
     end
+    n_total     = length(signals)
 
     # ── Build variable accessor map ──────────────────────────────────────────────
     # var_access: normalized name → Int (state index) or MTK symbolic (observed).
@@ -321,18 +326,27 @@ function compare_with_reference(
         @warn "Could not enumerate observed variables: $(sprint(showerror, e))"
     end
 
-    # Clip reference time to the simulation interval
+    # Verify the simulation covers the expected reference time interval.
+    # A large gap means the solver stopped early or started late.
+    isempty(sol.t) && return n_total, 0, 0, ""
     t_start    = sol.t[1]
     t_end      = sol.t[end]
+    ref_t_start = times[1]
+    ref_t_end   = times[end]
+    if t_start > ref_t_start || t_end < ref_t_end
+        @error "Simulation interval [$(t_start), $(t_end)] does not cover " *
+               "reference interval [$(ref_t_start), $(ref_t_end)]"
+        return n_total, 0, 0, ""
+    end
+
+    # Clip reference time to the simulation interval
     valid_mask = (times .>= t_start) .& (times .<= t_end)
     t_ref      = times[valid_mask]
-    isempty(t_ref) && return 0, 0, 0, ""
+    isempty(t_ref) && return n_total, 0, 0, ""
 
-    n_total     = 0
     n_pass      = 0
     pass_sigs   = String[]
     fail_sigs   = String[]
-    skip_sigs   = String[]
     pass_max_abs_error    = Dict{String, Float64}()
     pass_max_rel_error    = Dict{String, Float64}()
     fail_ref_vals         = Dict{String, Vector{Float64}}()
@@ -341,25 +355,32 @@ function compare_with_reference(
     fail_scaled_rel_error = Dict{String, Vector{Float64}}()
 
     for sig in signals
-        haskey(ref_data, sig) || continue   # signal absent from ref CSV entirely
+        signal_name = _normalize_var(sig)
+        ref_vals    = ref_data[sig][valid_mask]
+
+        nan_vec = fill(NaN, length(t_ref))
 
-        norm = _normalize_var(sig)
-        if !haskey(var_access, norm)
-            push!(skip_sigs, sig)
+        if !haskey(var_access, signal_name)
+            push!(fail_sigs, sig)
+            fail_ref_vals[sig]         = ref_vals
+            fail_sim_vals[sig]         = nan_vec
+            fail_abs_error[sig]        = nan_vec
+            fail_scaled_rel_error[sig] = nan_vec
             continue
         end
 
-        accessor  = var_access[norm]
-        ref_vals  = ref_data[sig][valid_mask]
-        n_total  += 1
+        accessor = var_access[signal_name]
 
         # Interpolate simulation at reference time points.
         sim_vals = [_eval_sim(sol, accessor, t) for t in t_ref]
 
-        # If evaluation returned NaN (observed-var access failed), treat as skip.
+        # If evaluation returned NaN (observed-var access failed), treat as fail.
         if any(isnan, sim_vals)
-            n_total -= 1
-            push!(skip_sigs, sig)
+            push!(fail_sigs, sig)
+            fail_ref_vals[sig]         = ref_vals
+            fail_sim_vals[sig]         = sim_vals
+            fail_abs_error[sig]        = nan_vec
+            fail_scaled_rel_error[sig] = nan_vec
             continue
         end
 
@@ -401,15 +422,14 @@ function compare_with_reference(
     end
 
     # ── Write detail HTML whenever there is anything worth showing ───────────────
-    if !isempty(fail_sigs) || !isempty(skip_sigs)
+    if !isempty(fail_sigs)
         write_diff_html(model_dir, model;
                         diff_csv_path      = diff_csv,
                         pass_sigs          = pass_sigs,
-                        skip_sigs          = skip_sigs,
                         pass_max_abs_error = pass_max_abs_error,
                         pass_max_rel_error = pass_max_rel_error,
                         settings           = settings)
     end
 
-    return n_total, n_pass, length(skip_sigs), diff_csv
+    return n_total, n_pass, 0, diff_csv
 end

src/pipeline.jl

@@ -194,11 +194,23 @@ function main(;
             result = test_model(omc, model, results_root, ref_root; csv_max_size_mb, settings)
             push!(results, result)
 
-            phase = result.sim_success    ? "SIM OK"     :
-                    result.parse_success  ? "SIM FAIL"   :
-                    result.export_success ? "PARSE FAIL" : "EXPORT FAIL"
-            cmp_info = result.cmp_total > 0 ?
-                "  cmp=$(result.cmp_pass)/$(result.cmp_total)" : ""
+            phase = if result.sim_success && result.cmp_total > 0
+                result.cmp_pass == result.cmp_total ? "CMP OK" : "CMP FAIL"
+            elseif result.sim_success
+                "SIM OK"
+            elseif result.parse_success
+                "SIM FAIL"
+            elseif result.export_success
+                "PARSE FAIL"
+            else
+                "EXPORT FAIL"
+            end
+            cmp_info = if result.cmp_total > 0
+                skip_note = result.cmp_skip > 0 ? " skip=$(result.cmp_skip)" : ""
+                "  cmp=$(result.cmp_pass)/$(result.cmp_total)$skip_note"
+            else
+                ""
+            end
             @info "  → $phase  export=$(round(result.export_time;digits=2))s" *
                   "  parse=$(round(result.parse_time;digits=2))s" *
                   "  sim=$(round(result.sim_time;digits=2))s$cmp_info"

src/simulate.jl

@@ -32,11 +32,21 @@ function run_simulate(ode_prob, model_dir::String, model::String;
         # Redirect all library log output (including Symbolics/MTK warnings)
         # to the log file so they don't clutter stdout.
         sol = Logging.with_logger(logger) do
-            solve(ode_prob, Rodas5P())
+            # Overwrite saveat, always use dense output.
+            solve(ode_prob, Rodas5P(); saveat = Float64[], dense = true)
         end
         sim_time = time() - t0
         if sol.retcode == ReturnCode.Success
-            sim_success = true
+            sys    = sol.prob.f.sys
+            n_vars = length(ModelingToolkit.unknowns(sys))
+            n_obs  = length(ModelingToolkit.observed(sys))
+            if isempty(sol.t)
+                sim_error = "Simulation produced no time points"
+            elseif n_vars == 0 && n_obs == 0
+                sim_error = "Simulation produced no output variables (no states or observed)"
+            else
+                sim_success = true
+            end
         else
             sim_error = "Solver returned: $(sol.retcode)"
         end
@@ -49,21 +59,30 @@ function run_simulate(ode_prob, model_dir::String, model::String;
     isempty(sim_error) || println(log_file, "\n--- Error ---\n$sim_error")
     close(log_file)
 
-    # Write simulation results CSV (time + all state variables)
+    # Write simulation results CSV (time + state variables + observed variables)
     if sim_success && sol !== nothing
         short_name = split(model, ".")[end]
         sim_csv    = joinpath(model_dir, "$(short_name)_sim.csv")
         try
-            sys       = sol.prob.f.sys
-            vars      = ModelingToolkit.unknowns(sys)
-            col_names = [_clean_var_name(string(v)) for v in vars]
+            sys      = sol.prob.f.sys
+            vars     = ModelingToolkit.unknowns(sys)
+            obs_eqs  = ModelingToolkit.observed(sys)
+            obs_syms = [eq.lhs for eq in obs_eqs]
+            col_names = vcat(
+                [_clean_var_name(string(v)) for v in vars],
+                [_clean_var_name(string(s)) for s in obs_syms],
+            )
             open(sim_csv, "w") do f
                 println(f, join(["time"; col_names], ","))
                 for (ti, t) in enumerate(sol.t)
                     row = [@sprintf("%.10g", t)]
                     for vi in eachindex(vars)
                         push!(row, @sprintf("%.10g", sol[vi, ti]))
                     end
+                    for sym in obs_syms
+                        val = try Float64(sol(t; idxs = sym)) catch; NaN end
+                        push!(row, @sprintf("%.10g", val))
+                    end
                     println(f, join(row, ","))
                 end
             end