Skip to content

Commit c4f8085

Browse files
AnHeuermannclaude
AnHeuermann
and
claude
committed
Add cmp_time field, signal pre-filtering, and simulation CSV optimizations
- Add `cmp_time` to `ModelResult` to track reference comparison wall time; report it in the per-model status line - Read comparison signals in `pipeline.jl` before phase 3 and pass them to both `run_simulate` and `compare_with_reference`, avoiding redundant file reads and enabling observed-variable filtering in the CSV writer - Filter observed variables written to the simulation CSV to only those needed for comparison, keeping large models (e.g. LightningSegmented- TransmissionLine) within the CSV size limit - Write CSV at fixed intervals (from `ode_prob.kwargs[:saveat]` or tspan/500) instead of at every solver step; use dense interpolation for accuracy - Fix `Nothing → Float64` for models where BaseModelica sets `saveat=nothing` by using `something(get(..., nothing), default)` - Accept caller-supplied signal list in `compare_with_reference`; fall back to `comparisonSignals.txt` / all ref-CSV columns when none supplied - Delete versioned results directory before rsync in CI to remove stale files - Use qualified `Dates.now()` / `DifferentialEquations.*` after switching to module-level imports Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
1 parent becfe29 commit c4f8085

7 files changed

Lines changed: 88 additions & 42 deletions

File tree

‎.github/workflows/msl-test.yml‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -145,7 +145,11 @@ jobs:
145145
fi
146146
147147
- name: Copy new results into gh-pages tree
148-
run: rsync -a results/ site/results/
148+
env:
149+
BM_VERSION: ${{ steps.versions.outputs.bm_version }}
150+
run: |
151+
rm -rf "site/results/${BM_VERSION}/${LIB_NAME}/${LIB_VERSION}"
152+
rsync -a results/ site/results/
149153
150154
- name: Generate landing page
151155
run: python3 .github/scripts/gen_landing_page.py site

‎src/BaseModelicaLibraryTesting.jl‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4,9 +4,9 @@ import Pkg
44
import OMJulia
55
import OMJulia: sendExpression
66
import BaseModelica
7-
import DifferentialEquations: solve, Rodas5P, ReturnCode
7+
import DifferentialEquations
88
import ModelingToolkit
9-
import Dates: now
9+
import Dates
1010
import Printf: @sprintf
1111

1212
include("types.jl")

‎src/compare.jl‎

Lines changed: 19 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -287,24 +287,32 @@ function compare_with_reference(
287287
ref_csv_path::String,
288288
model_dir::String,
289289
model::String;
290-
settings::CompareSettings = CompareSettings(),
290+
settings::CompareSettings = CompareSettings(),
291+
signals::Vector{String} = String[],
291292
)::Tuple{Int,Int,Int,String}
292293

293294
times, ref_data = _read_ref_csv(ref_csv_path)
294295
isempty(times) && return 0, 0, 0, ""
295296

296-
# Determine which signals to compare: prefer comparisonSignals.txt
297-
sig_file = joinpath(dirname(ref_csv_path), "comparisonSignals.txt")
298-
using_sig_file = isfile(sig_file)
299-
signals = if using_sig_file
300-
sigs = filter(s -> lowercase(s) != "time" && !isempty(s), strip.(readlines(sig_file)))
301-
sigs_missing = filter(s -> !haskey(ref_data, s), sigs)
302-
isempty(sigs_missing) || error("Signal(s) listed in comparisonSignals.txt not present in reference CSV: $(join(sigs_missing, ", "))")
303-
sigs
297+
# Determine which signals to compare.
298+
# Prefer the caller-supplied list; fall back to comparisonSignals.txt, then
299+
# all columns in the reference CSV.
300+
signals = if !isempty(signals)
301+
sigs_missing = filter(s -> !haskey(ref_data, s), signals)
302+
isempty(sigs_missing) || error("Signal(s) not present in reference CSV: $(join(sigs_missing, ", "))")
303+
signals
304304
else
305-
filter(k -> lowercase(k) != "time", collect(keys(ref_data)))
305+
sig_file = joinpath(dirname(ref_csv_path), "comparisonSignals.txt")
306+
if isfile(sig_file)
307+
sigs = String.(filter(s -> lowercase(s) != "time" && !isempty(s), strip.(readlines(sig_file))))
308+
sigs_missing = filter(s -> !haskey(ref_data, s), sigs)
309+
isempty(sigs_missing) || error("Signal(s) listed in comparisonSignals.txt not present in reference CSV: $(join(sigs_missing, ", "))")
310+
sigs
311+
else
312+
filter(k -> lowercase(k) != "time", collect(keys(ref_data)))
313+
end
306314
end
307-
n_total = length(signals)
315+
n_total = length(signals)
308316

309317
# ── Build variable accessor map ──────────────────────────────────────────────
310318
# var_access: normalized name → Int (state index) or MTK symbolic (observed).

‎src/pipeline.jl‎

Lines changed: 34 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -73,27 +73,45 @@ function test_model(omc::OMJulia.OMCSession,
7373
# Phase 1 ──────────────────────────────────────────────────────────────────
7474
exp_ok, exp_t, exp_err = run_export(omc, model, model_dir, bm_path)
7575
exp_ok || return ModelResult(
76-
model, false, exp_t, exp_err, false, 0.0, "", false, 0.0, "", 0, 0, 0, "")
76+
model, false, exp_t, exp_err, false, 0.0, "", false, 0.0, "", 0, 0, 0, 0.0, "")
7777

7878
# Phase 2 ──────────────────────────────────────────────────────────────────
7979
par_ok, par_t, par_err, ode_prob = run_parse(bm_path, model_dir, model)
8080
par_ok || return ModelResult(
81-
model, true, exp_t, exp_err, false, par_t, par_err, false, 0.0, "", 0, 0, 0, "")
81+
model, true, exp_t, exp_err, false, par_t, par_err, false, 0.0, "", 0, 0, 0, 0.0, "")
82+
83+
# Resolve reference CSV and comparison signals early so phase 3 can filter
84+
# the CSV output to only the signals that will actually be verified.
85+
ref_csv = isempty(ref_root) ? nothing : _ref_csv_path(ref_root, model)
86+
cmp_signals = if ref_csv !== nothing
87+
sig_file = joinpath(dirname(ref_csv), "comparisonSignals.txt")
88+
if isfile(sig_file)
89+
String.(filter(s -> lowercase(s) != "time" && !isempty(s), strip.(readlines(sig_file))))
90+
else
91+
_, ref_data = _read_ref_csv(ref_csv)
92+
filter(k -> lowercase(k) != "time", collect(keys(ref_data)))
93+
end
94+
else
95+
String[]
96+
end
8297

8398
# Phase 3 ──────────────────────────────────────────────────────────────────
84-
sim_ok, sim_t, sim_err, sol = run_simulate(ode_prob, model_dir, model; csv_max_size_mb)
99+
sim_ok, sim_t, sim_err, sol = run_simulate(ode_prob, model_dir, model;
100+
csv_max_size_mb, cmp_signals)
85101

86102
# Phase 4 (optional) ───────────────────────────────────────────────────────
87103
cmp_total, cmp_pass, cmp_skip, cmp_csv = 0, 0, 0, ""
88-
if sim_ok && !isempty(ref_root)
89-
ref_csv = _ref_csv_path(ref_root, model)
90-
if ref_csv !== nothing
91-
try
92-
cmp_total, cmp_pass, cmp_skip, cmp_csv =
93-
compare_with_reference(sol, ref_csv, model_dir, model; settings)
94-
catch e
95-
@warn "Reference comparison failed for $model: $(sprint(showerror, e))"
96-
end
104+
cmp_t = 0.0
105+
if sim_ok && ref_csv !== nothing
106+
try
107+
t0_cmp = time()
108+
cmp_total, cmp_pass, cmp_skip, cmp_csv =
109+
compare_with_reference(sol, ref_csv, model_dir, model;
110+
settings, signals = cmp_signals)
111+
cmp_t = time() - t0_cmp
112+
catch e
113+
cmp_t = time() - t0_cmp
114+
@warn "Reference comparison failed for $model: $(sprint(showerror, e))"
97115
end
98116
end
99117

@@ -102,7 +120,7 @@ function test_model(omc::OMJulia.OMCSession,
102120
true, exp_t, exp_err,
103121
true, par_t, par_err,
104122
sim_ok, sim_t, sim_err,
105-
cmp_total, cmp_pass, cmp_skip, cmp_csv)
123+
cmp_total, cmp_pass, cmp_skip, cmp_t, cmp_csv)
106124
end
107125

108126
# ── Main ───────────────────────────────────────────────────────────────────────
@@ -207,7 +225,9 @@ function main(;
207225
end
208226
cmp_info = if result.cmp_total > 0
209227
skip_note = result.cmp_skip > 0 ? " skip=$(result.cmp_skip)" : ""
210-
" cmp=$(result.cmp_pass)/$(result.cmp_total)$skip_note"
228+
" cmp=$(result.cmp_pass)/$(result.cmp_total)$skip_note ($(round(result.cmp_time;digits=2))s)"
229+
elseif result.cmp_time > 0
230+
" cmp=n/a ($(round(result.cmp_time;digits=2))s)"
211231
else
212232
""
213233
end

‎src/report.jl‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
# ── HTML report generation ─────────────────────────────────────────────────────
22

3-
import Dates: now
3+
import Dates
44
import Printf: @sprintf
55

66
function _status_cell(ok::Bool, t::Float64, logFile::Union{String,Nothing})
@@ -145,7 +145,7 @@ function generate_report(results::Vector{ModelResult}, results_root::String,
145145
</head>
146146
<body>
147147
<h1>$(info.library) $(info.lib_version) — Base Modelica / MTK Pipeline Test Results</h1>
148-
<p>Generated: $(now())<br>
148+
<p>Generated: $(Dates.now())<br>
149149
OpenModelica: $(info.omc_version)<br>
150150
OMC options: <code>$(info.omc_options)</code><br>
151151
BaseModelica.jl: $(basemodelica_jl_version)<br>

‎src/simulate.jl‎

Lines changed: 25 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
# ── Phase 3: ODE simulation with DifferentialEquations / MTK ──────────────────
22

3-
import DifferentialEquations: solve, Rodas5P, ReturnCode
3+
import DifferentialEquations
44
import Logging
55
import ModelingToolkit
66
import Printf: @sprintf
@@ -9,15 +9,17 @@ import Printf: @sprintf
99
run_simulate(ode_prob, model_dir, model; csv_max_size_mb) → (success, time, error, sol)
1010
1111
Solve `ode_prob` with Rodas5P (stiff solver). On success, also writes the
12-
full solution as a CSV file `<Short>_sim.csv` in `model_dir`.
12+
solution as a CSV file `<Short>_sim.csv` in `model_dir`.
1313
Writes a `<model>_sim.log` file in `model_dir`.
1414
Returns `nothing` as the fourth element on failure.
1515
1616
CSV files larger than `csv_max_size_mb` MiB are deleted and replaced with a
1717
`<Short>_sim.csv.toobig` marker so that the report can note the omission.
1818
"""
19-
function run_simulate(ode_prob, model_dir::String, model::String;
20-
csv_max_size_mb::Int = CSV_MAX_SIZE_MB)::Tuple{Bool,Float64,String,Any}
19+
function run_simulate(ode_prob, model_dir::String,
20+
model::String;
21+
cmp_signals ::Vector{String} = String[],
22+
csv_max_size_mb::Int = CSV_MAX_SIZE_MB)::Tuple{Bool,Float64,String,Any}
2123
sim_success = false
2224
sim_time = 0.0
2325
sim_error = ""
@@ -27,16 +29,20 @@ function run_simulate(ode_prob, model_dir::String, model::String;
2729
println(log_file, "Model: $model")
2830
logger = Logging.SimpleLogger(log_file, Logging.Debug)
2931
t0 = time()
32+
33+
# Read interval before overwriting it
34+
interval = something(get(ode_prob.kwargs, :saveat, nothing),
35+
(ode_prob.tspan[end] - ode_prob.tspan[1]) / 500)
36+
3037
try
3138
# Rodas5P handles stiff DAE-like systems well.
3239
# Redirect all library log output (including Symbolics/MTK warnings)
3340
# to the log file so they don't clutter stdout.
3441
sol = Logging.with_logger(logger) do
35-
# Overwrite saveat, always use dense output.
36-
solve(ode_prob, Rodas5P(); saveat = Float64[], dense = true)
42+
DifferentialEquations.solve(ode_prob, DifferentialEquations.Rodas5P(); saveat = Float64[], dense = true)
3743
end
3844
sim_time = time() - t0
39-
if sol.retcode == ReturnCode.Success
45+
if sol.retcode == DifferentialEquations.ReturnCode.Success
4046
sys = sol.prob.f.sys
4147
n_vars = length(ModelingToolkit.unknowns(sys))
4248
n_obs = length(ModelingToolkit.observed(sys))
@@ -67,20 +73,28 @@ function run_simulate(ode_prob, model_dir::String, model::String;
6773
sys = sol.prob.f.sys
6874
vars = ModelingToolkit.unknowns(sys)
6975
obs_eqs = ModelingToolkit.observed(sys)
70-
obs_syms = [eq.lhs for eq in obs_eqs]
76+
# Only save observed variables that appear in cmp_signals.
77+
# This avoids writing thousands of algebraic variables to disk when
78+
# only a handful are actually verified during comparison.
79+
norm_cmp = Set(_normalize_var(s) for s in cmp_signals)
80+
obs_eqs_filtered = isempty(norm_cmp) ? obs_eqs :
81+
filter(eq -> _normalize_var(string(eq.lhs)) in norm_cmp, obs_eqs)
82+
obs_syms = [eq.lhs for eq in obs_eqs_filtered]
7183
col_names = vcat(
7284
[_clean_var_name(string(v)) for v in vars],
7385
[_clean_var_name(string(s)) for s in obs_syms],
7486
)
7587
open(sim_csv, "w") do f
7688
println(f, join(["time"; col_names], ","))
77-
for (ti, t) in enumerate(sol.t)
89+
t_csv = range(ode_prob.tspan[1], ode_prob.tspan[end]; step = interval)
90+
for t in t_csv
7891
row = [@sprintf("%.10g", t)]
92+
u = sol(Float64(t))
7993
for vi in eachindex(vars)
80-
push!(row, @sprintf("%.10g", sol[vi, ti]))
94+
push!(row, @sprintf("%.10g", u[vi]))
8195
end
8296
for sym in obs_syms
83-
val = try Float64(sol(t; idxs = sym)) catch; NaN end
97+
val = try Float64(sol(Float64(t); idxs = sym)) catch; NaN end
8498
push!(row, @sprintf("%.10g", val))
8599
end
86100
println(f, join(row, ","))
@@ -90,7 +104,6 @@ function run_simulate(ode_prob, model_dir::String, model::String;
90104
if csv_bytes > csv_max_size_mb * 1024^2
91105
csv_mb = round(csv_bytes / 1024^2; digits=1)
92106
@warn "Simulation CSV for $model is $(csv_mb) MB (> $(csv_max_size_mb) MB limit); skipping."
93-
rm(sim_csv)
94107
write(sim_csv * ".toobig", string(csv_bytes))
95108
end
96109
catch e

‎src/types.jl‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -82,5 +82,6 @@ struct ModelResult
8282
cmp_total :: Int # signals actually compared (found in simulation)
8383
cmp_pass :: Int
8484
cmp_skip :: Int # reference signals not found in simulation
85+
cmp_time :: Float64 # wall time for comparison phase (0.0 if skipped)
8586
cmp_csv :: String # absolute path to diff CSV; "" if all pass or no comparison
8687
end

0 commit comments

Comments
 (0)