diff --git a/.gitignore b/.gitignore index d7ddbf77f..5ad67c9cb 100644 --- a/.gitignore +++ b/.gitignore @@ -13,10 +13,5 @@ .CondaPkg/ # Generated files and example results -*.log -*.html -!asserts/*.html -*.csv -*.bmo +results/ MAP-LIB_ReferenceResults/ -main/ diff --git a/README.md b/README.md index d4b36baa2..c9a513a80 100644 --- a/README.md +++ b/README.md @@ -49,7 +49,7 @@ main( version = "4.1.0", filter = "Modelica.Electrical.Analog.Examples.ChuaCircuit", omc_exe = "omc", - results_root = "main/Modelica/4.1.0/", + results_root = "results/main/Modelica/4.1.0/", ref_root = "MAP-LIB_ReferenceResults" ) ``` @@ -57,7 +57,7 @@ main( Preview the generated HTML report at `main/Modelica/4.1.0/report.html`. ```bash -python -m http.server -d main/Modelica/4.1.0/ +python -m http.server -d results/main/Modelica/4.1.0/ ``` ## License diff --git a/src/BaseModelicaLibraryTesting.jl b/src/BaseModelicaLibraryTesting.jl index 383507fad..840fb5c9a 100644 --- a/src/BaseModelicaLibraryTesting.jl +++ b/src/BaseModelicaLibraryTesting.jl @@ -14,12 +14,13 @@ include("export.jl") include("parse_bm.jl") include("simulate.jl") include("report.jl") +include("summary.jl") include("pipeline.jl") # ── Public API ───────────────────────────────────────────────────────────────── # Shared types and constants -export ModelResult, CompareSettings +export ModelResult, CompareSettings, RunInfo export LIBRARY, LIBRARY_VERSION, CMP_REL_TOL, CMP_ABS_TOL # Comparison configuration @@ -36,6 +37,9 @@ export compare_with_reference, write_diff_html # HTML report export generate_report +# Summary JSON +export RunSummary, write_summary, load_summary + # Top-level orchestration export test_model, main diff --git a/src/pipeline.jl b/src/pipeline.jl index 3e6bb2adf..5f82a1984 100644 --- a/src/pipeline.jl +++ b/src/pipeline.jl @@ -66,6 +66,8 @@ function main(; results_root :: String = "", ref_root :: String = get(ENV, "MAPLIB_REF", ""), ) + t0 = time() + if isempty(results_root) results_root = joinpath(library, version) end @@ -76,6 +78,7 @@ function main(; @info "Starting OMC session ($(omc_exe))..." omc = OMJulia.OMCSession(omc_exe) + omc_version = "unknown" results = ModelResult[] try omc_version = sendExpression(omc, "getVersion()") @@ -136,6 +139,23 @@ function main(; OMJulia.quit(omc) end - generate_report(results, results_root, library, version) + cpu_info = Sys.cpu_info() + info = RunInfo( + library, + version, + something(filter, ""), + omc_exe, + results_root, + ref_root, + omc_version, + string(pkgversion(BaseModelica)), + isempty(cpu_info) ? "unknown" : strip(cpu_info[1].model), + length(cpu_info), + Sys.total_memory() / 1024^3, + time() - t0, + ) + + generate_report(results, results_root, info) + write_summary(results, results_root, info) return results end diff --git a/src/report.jl b/src/report.jl index f85f7ae4a..88a12c071 100644 --- a/src/report.jl +++ b/src/report.jl @@ -2,7 +2,6 @@ import Dates: now import Printf: @sprintf -import BaseModelica function _status_cell(ok::Bool, t::Float64, logFile::Union{String,Nothing}) link = isnothing(logFile) ? "" : """ (log)""" @@ -34,13 +33,22 @@ function rel_log_file_or_nothing(results_root::String, model::String, isfile(path) ? joinpath("files", model, "$(model)_$(phase).log") : nothing end +function _format_duration(t::Float64)::String + t < 60 && return @sprintf("%.1f s", t) + m = div(floor(Int, t), 60) + s = floor(Int, t) % 60 + m < 60 && return @sprintf("%d min %d s", m, s) + h = div(m, 60) + return @sprintf("%d h %d min %d s", h, m % 60, s) +end + """ - generate_report(results, results_root, library, version) → report_path + generate_report(results, results_root, info) → report_path Write an `index.html` overview report to `results_root` and return its path. """ function generate_report(results::Vector{ModelResult}, results_root::String, - library::String, version::String) + info::RunInfo) n = length(results) n_exp = count(r -> r.export_success, results) n_par = count(r -> r.parse_success, results) @@ -64,14 +72,16 @@ function generate_report(results::Vector{ModelResult}, results_root::String, $(_cmp_cell(r, results_root)) """ for r in results], "\n") - omc_ver = try sendExpression(OMJulia.OMCSession("omc"), "getVersion()") catch; "unknown" end - bm_ver = string(pkgversion(BaseModelica)) + filter_row = isempty(info.filter) ? "" : "
Filter: $(info.filter)" + ref_row = isempty(info.ref_root) ? "" : "
Reference results: $(info.ref_root)" + ram_str = @sprintf("%.1f", info.ram_gb) + time_str = _format_duration(info.total_time_s) html = """ - $library $version — Base Modelica / MTK Results + $(info.library) $(info.lib_version) — Base Modelica / MTK Results -

$library $version — Base Modelica / MTK Pipeline Test Results

+

$(info.library) $(info.lib_version) — Base Modelica / MTK Pipeline Test Results

Generated: $(now())
-OpenModelica: $omc_ver
-BaseModelica.jl: $bm_ver

+OpenModelica: $(info.omc_version)
+BaseModelica.jl: $(info.bm_version)$(filter_row)$(ref_row)

+

CPU: $(info.cpu_model) ($(info.cpu_threads) threads)
+RAM: $(ram_str) GiB
+Total run time: $(time_str)

diff --git a/src/summary.jl b/src/summary.jl new file mode 100644 index 000000000..d922deca1 --- /dev/null +++ b/src/summary.jl @@ -0,0 +1,141 @@ +# ── Summary JSON serialization ───────────────────────────────────────────────── + +function _esc_json(s::String)::String + replace(s, "\\" => "\\\\", "\"" => "\\\"", "\n" => "\\n") +end + +""" + write_summary(results, results_root, info) + +Write a `summary.json` to `results_root` encoding run settings, tool versions, +machine info, and per-model pipeline pass/fail data. +Called automatically by `main()` at the end of each run. +""" +function write_summary( + results :: Vector{ModelResult}, + results_root :: String, + info :: RunInfo, +) + path = joinpath(results_root, "summary.json") + open(path, "w") do io + print(io, "{\n") + print(io, " \"library\": \"$(_esc_json(info.library))\",\n") + print(io, " \"lib_version\": \"$(_esc_json(info.lib_version))\",\n") + print(io, " \"filter\": \"$(_esc_json(info.filter))\",\n") + print(io, " \"omc_exe\": \"$(_esc_json(info.omc_exe))\",\n") + print(io, " \"results_root\": \"$(_esc_json(info.results_root))\",\n") + print(io, " \"ref_root\": \"$(_esc_json(info.ref_root))\",\n") + print(io, " \"omc_version\": \"$(_esc_json(info.omc_version))\",\n") + print(io, " \"bm_version\": \"$(_esc_json(info.bm_version))\",\n") + print(io, " \"cpu_model\": \"$(_esc_json(info.cpu_model))\",\n") + print(io, " \"cpu_threads\": $(info.cpu_threads),\n") + print(io, " \"ram_gb\": $(@sprintf "%.2f" info.ram_gb),\n") + print(io, " \"total_time_s\": $(@sprintf "%.2f" info.total_time_s),\n") + print(io, " \"models\": [\n") + for (i, r) in enumerate(results) + sep = i < length(results) ? "," : "" + print(io, + " {\"name\":\"$(_esc_json(r.name))\"," * + "\"export\":$(r.export_success)," * + "\"parse\":$(r.parse_success)," * + "\"sim\":$(r.sim_success)," * + "\"cmp_total\":$(r.cmp_total)," * + "\"cmp_pass\":$(r.cmp_pass)}$sep\n") + end + print(io, " ]\n}\n") + end + @info "summary.json written to $results_root" +end + +# ── Summary type and JSON loading ────────────────────────────────────────────── + +""" + RunSummary + +Parsed contents of a single `summary.json` file. + +# Fields +- `library` — Modelica library name (e.g. `"Modelica"`) +- `lib_version` — library version (e.g. `"4.1.0"`) +- `filter` — model name filter regex, or `""` when none was given +- `omc_exe` — path / command used to launch OMC +- `results_root` — absolute path where results were written +- `ref_root` — absolute path to reference results, or `""` when unused +- `omc_version` — OMC version string +- `bm_version` — BaseModelica.jl version string (e.g. `"1.6.0"`) +- `cpu_model` — CPU model name +- `cpu_threads` — number of logical CPU threads +- `ram_gb` — total system RAM in GiB +- `total_time_s` — wall-clock duration of the full test run in seconds +- `models` — vector of per-model dicts; each has keys + `"name"`, `"export"`, `"parse"`, `"sim"`, `"cmp_total"`, `"cmp_pass"` +""" +struct RunSummary + library :: String + lib_version :: String + filter :: String + omc_exe :: String + results_root :: String + ref_root :: String + omc_version :: String + bm_version :: String + cpu_model :: String + cpu_threads :: Int + ram_gb :: Float64 + total_time_s :: Float64 + models :: Vector{Dict{String,Any}} +end + +""" + load_summary(results_root) → RunSummary or nothing + +Read and parse the `summary.json` written by `write_summary` from `results_root`. +Returns `nothing` if the file does not exist or cannot be parsed. +""" +function load_summary(results_root::String)::Union{RunSummary,Nothing} + path = joinpath(results_root, "summary.json") + isfile(path) || return nothing + txt = read(path, String) + + _str(key) = begin + m = match(Regex("\"$(key)\"\\s*:\\s*\"([^\"]*)\""), txt) + m === nothing ? "" : string(m.captures[1]) + end + _int(key) = begin + m = match(Regex("\"$(key)\"\\s*:\\s*(\\d+)"), txt) + m === nothing ? 0 : parse(Int, m.captures[1]) + end + _float(key) = begin + m = match(Regex("\"$(key)\"\\s*:\\s*([\\d.]+)"), txt) + m === nothing ? 0.0 : parse(Float64, m.captures[1]) + end + + models = Dict{String,Any}[] + for m in eachmatch( + r"\{\"name\":\"([^\"]*)\",\"export\":(true|false),\"parse\":(true|false),\"sim\":(true|false),\"cmp_total\":(\d+),\"cmp_pass\":(\d+)\}", + txt) + push!(models, Dict{String,Any}( + "name" => string(m.captures[1]), + "export" => m.captures[2] == "true", + "parse" => m.captures[3] == "true", + "sim" => m.captures[4] == "true", + "cmp_total" => parse(Int, m.captures[5]), + "cmp_pass" => parse(Int, m.captures[6]), + )) + end + return RunSummary( + _str("library"), + _str("lib_version"), + _str("filter"), + _str("omc_exe"), + _str("results_root"), + _str("ref_root"), + _str("omc_version"), + _str("bm_version"), + _str("cpu_model"), + _int("cpu_threads"), + _float("ram_gb"), + _float("total_time_s"), + models, + ) +end diff --git a/src/types.jl b/src/types.jl index 9830ac08f..188dcea61 100644 --- a/src/types.jl +++ b/src/types.jl @@ -33,6 +33,43 @@ Base.@kwdef mutable struct CompareSettings error_fn :: Symbol = :mixed end +# ── Run metadata ─────────────────────────────────────────────────────────────── + +""" + RunInfo + +Metadata about a single test run, collected by `main()` and written into both +`index.html` and `summary.json`. + +# Fields +- `library` — Modelica library name (e.g. `"Modelica"`) +- `lib_version` — library version (e.g. `"4.1.0"`) +- `filter` — model name filter regex, or `""` when none was given +- `omc_exe` — path / command used to launch OMC +- `results_root` — absolute path where results are written +- `ref_root` — absolute path to reference results, or `""` when unused +- `omc_version` — version string returned by `getVersion()`, e.g. `"v1.23.0"` +- `bm_version` — BaseModelica.jl version string, e.g. `"1.6.0"` +- `cpu_model` — CPU model name from `Sys.cpu_info()` +- `cpu_threads` — number of logical CPU threads +- `ram_gb` — total system RAM in GiB +- `total_time_s` — wall-clock duration of the full test run in seconds +""" +struct RunInfo + library :: String + lib_version :: String + filter :: String # "" when no filter was given + omc_exe :: String + results_root :: String + ref_root :: String # "" when no reference root was given + omc_version :: String + bm_version :: String + cpu_model :: String + cpu_threads :: Int + ram_gb :: Float64 + total_time_s :: Float64 +end + # ── Result type ──────────────────────────────────────────────────────────────── struct ModelResult
StagePassedTotalRate