Skip sim CSVs exceeding a configurable size limit

Simulation result CSVs can exceed GitHub's 100 MB file size limit,
preventing pushes to gh-pages.  Fix by checking the CSV size after
writing and, if it exceeds the limit (default 20 MB), deleting the
file and leaving a <name>_sim.csv.toobig marker in its place.

The HTML report detects the marker and shows "(CSV N/A)" with a
tooltip instead of a download link.  The limit is exposed as a
`csv_max_size_mb` keyword argument on `main()`, `test_model()`,
`run_simulate()`, and `generate_report()`, defaulting to the
module-level constant `CSV_MAX_SIZE_MB = 20`.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

Author: AnHeuermann

src/pipeline.jl

@@ -54,12 +54,12 @@ end
 # ── Per-model orchestrator ─────────────────────────────────────────────────────
 
 """
-    test_model(omc, model, results_root, ref_root) → ModelResult
+    test_model(omc, model, results_root, ref_root; csv_max_size_mb) → ModelResult
 
 Run the four-phase pipeline for a single model and return its result.
 """
 function test_model(omc::OMJulia.OMCSession, model::String, results_root::String,
-                    ref_root::String)::ModelResult
+                    ref_root::String; csv_max_size_mb::Int = CSV_MAX_SIZE_MB)::ModelResult
     model_dir = joinpath(results_root, "files", model)
     mkpath(model_dir)
 
@@ -78,7 +78,7 @@ function test_model(omc::OMJulia.OMCSession, model::String, results_root::String
         model, true, exp_t, exp_err, false, par_t, par_err, false, 0.0, "", 0, 0, "")
 
     # Phase 3 ──────────────────────────────────────────────────────────────────
-    sim_ok, sim_t, sim_err, sol = run_simulate(ode_prob, model_dir, model)
+    sim_ok, sim_t, sim_err, sol = run_simulate(ode_prob, model_dir, model; csv_max_size_mb)
 
     # Phase 4 (optional) ───────────────────────────────────────────────────────
     cmp_total, cmp_pass, cmp_csv = 0, 0, ""
@@ -112,13 +112,14 @@ Discovers models via OMC, runs `test_model` for each, then writes the HTML
 report.  Returns a `Vector{ModelResult}`.
 """
 function main(;
-    library      :: String                = LIBRARY,
-    version      :: String                = LIBRARY_VERSION,
-    filter       :: Union{String,Nothing} = nothing,
-    omc_exe      :: String                = get(ENV, "OMC_EXE", "omc"),
-    results_root :: String                = "",
-    ref_root     :: String                = get(ENV, "MAPLIB_REF", ""),
-    bm_options   :: String                = get(ENV, "BM_OPTIONS", "scalarize,moveBindings,inlineFunctions"),
+    library          :: String                = LIBRARY,
+    version          :: String                = LIBRARY_VERSION,
+    filter           :: Union{String,Nothing} = nothing,
+    omc_exe          :: String                = get(ENV, "OMC_EXE", "omc"),
+    results_root     :: String                = "",
+    ref_root         :: String                = get(ENV, "MAPLIB_REF", ""),
+    bm_options       :: String                = get(ENV, "BM_OPTIONS", "scalarize,moveBindings,inlineFunctions"),
+    csv_max_size_mb  :: Int                   = CSV_MAX_SIZE_MB,
 )
     t0 = time()
 
@@ -186,7 +187,7 @@ function main(;
 
         for (i, model) in enumerate(models)
             @info "[$i/$(length(models))] $model"
-            result = test_model(omc, model, results_root, ref_root)
+            result = test_model(omc, model, results_root, ref_root; csv_max_size_mb)
             push!(results, result)
 
             phase = result.sim_success    ? "SIM OK"     :
@@ -221,7 +222,7 @@ function main(;
         time() - t0,
     )
 
-    generate_report(results, results_root, info)
+    generate_report(results, results_root, info; csv_max_size_mb)
     write_summary(results, results_root, info)
     return results
 end

src/report.jl

@@ -13,23 +13,48 @@ function _status_cell(ok::Bool, t::Float64, logFile::Union{String,Nothing})
     end
 end
 
-function _cmp_cell(r::ModelResult, results_root::String)
+"""
+    _cmp_cell(r, results_root, csv_max_size_mb) → HTML string
+
+Build the "Ref Cmp" table cell for one model row.
+
+- `cmp_total == 0`: no reference data — show `—` (or just the CSV link if present).
+- All signals pass (`p == n`): green cell with score and a link to the sim CSV.
+- Some signals fail: red cell with a link to the interactive diff HTML, the diff
+  CSV, and the sim CSV.  The sim CSV link is always included when the file exists.
+
+`sim_csv` is constructed as `"files/<model>/<short>_sim.csv"`, which is already a
+path relative to `results_root` (where `index.html` lives), so it is used directly
+as the `href` without further `relpath` manipulation.  If the CSV was skipped
+because it exceeded `csv_max_size_mb` MB, a `<short>_sim.csv.toobig` marker file
+is present and a non-linking note is shown instead.
+"""
+function _cmp_cell(r::ModelResult, results_root::String, csv_max_size_mb::Int)
+    short = split(r.name, ".")[end]
+    sim_csv = joinpath("files", r.name, "$(short)_sim.csv")   # relative to results_root
+    abs_sim_csv = joinpath(results_root, sim_csv)
+    csv_link = if isfile(abs_sim_csv)
+        """ <a href="$sim_csv">(CSV)</a>"""
+    elseif isfile(abs_sim_csv * ".toobig")
+        """ <span title="Result file exceeds $(csv_max_size_mb) MB and was not uploaded">(CSV N/A)</span>"""
+    else
+        ""
+    end
+
     if r.cmp_total == 0
-        return """<td class="na">—</td>"""
+        return isempty(csv_link) ? """<td class="na">—</td>""" :
+                                   """<td class="na">$(csv_link)</td>"""
     end
+
     n, p = r.cmp_total, r.cmp_pass
     if p == n
-        # No diff CSV when all signals pass — link the sim CSV instead
-        short = split(r.name, ".")[end]
-        sim_csv = joinpath("files", r.name, "$(short)_sim.csv")
-        csv_link = isfile(joinpath(results_root, sim_csv)) ? """ <a href="$sim_csv">(CSV)</a>""" : ""
         return """<td class="ok">&#10003; $p/$n$(csv_link)</td>"""
     else
-        # Link to the interactive diff HTML (next to the CSV, same name, .html extension)
+        # Link to the interactive diff HTML (next to the diff CSV, same name, .html extension)
         diff_html = replace(r.cmp_csv, r"\.csv$" => ".html")
-        rel = relpath(isfile(diff_html) ? diff_html : r.cmp_csv, results_root)
-        csv_link = isempty(r.cmp_csv) ? "" : """ <a href="$(relpath(r.cmp_csv, results_root))">(CSV)</a>"""
-        return """<td class="fail"><a href="$rel">$p/$n</a>$(csv_link)</td>"""
+        diff_rel  = relpath(isfile(diff_html) ? diff_html : r.cmp_csv, results_root)
+        diff_csv_link = isempty(r.cmp_csv) ? "" : """ <a href="$(relpath(r.cmp_csv, results_root))">(diff CSV)</a>"""
+        return """<td class="fail"><a href="$diff_rel">$p/$n</a>$(diff_csv_link)$(csv_link)</td>"""
     end
 end
 
@@ -49,12 +74,12 @@ function _format_duration(t::Float64)::String
 end
 
 """
-    generate_report(results, results_root, info) → report_path
+    generate_report(results, results_root, info; csv_max_size_mb) → report_path
 
 Write an `index.html` overview report to `results_root` and return its path.
 """
 function generate_report(results::Vector{ModelResult}, results_root::String,
-                         info::RunInfo)
+                         info::RunInfo; csv_max_size_mb::Int = CSV_MAX_SIZE_MB)
     n     = length(results)
     n_exp = count(r -> r.export_success, results)
     n_par = count(r -> r.parse_success,  results)
@@ -75,7 +100,7 @@ function generate_report(results::Vector{ModelResult}, results_root::String,
       $(_status_cell(r.export_success, r.export_time, rel_log_file_or_nothing(results_root, r.name, "export")))
       $(_status_cell(r.parse_success,  r.parse_time,  rel_log_file_or_nothing(results_root, r.name, "parsing")))
       $(_status_cell(r.sim_success,    r.sim_time,    rel_log_file_or_nothing(results_root, r.name, "sim")))
-      $(_cmp_cell(r, results_root))
+      $(_cmp_cell(r, results_root, csv_max_size_mb))
     </tr>""" for r in results], "\n")
 
     bm_sha_link = isempty(info.bm_sha) ? "" :

src/simulate.jl

@@ -5,15 +5,18 @@ import ModelingToolkit
 import Printf: @sprintf
 
 """
-    run_simulate(ode_prob, model_dir, model) → (success, time, error, sol)
+    run_simulate(ode_prob, model_dir, model; csv_max_size_mb) → (success, time, error, sol)
 
 Solve `ode_prob` with Rodas5P (stiff solver).  On success, also writes the
 full solution as a CSV file `<Short>_sim.csv` in `model_dir`.
 Writes a `<model>_sim.log` file in `model_dir`.
 Returns `nothing` as the fourth element on failure.
+
+CSV files larger than `csv_max_size_mb` MiB are deleted and replaced with a
+`<Short>_sim.csv.toobig` marker so that the report can note the omission.
 """
-function run_simulate(ode_prob, model_dir::String,
-                      model::String)::Tuple{Bool,Float64,String,Any}
+function run_simulate(ode_prob, model_dir::String, model::String;
+                      csv_max_size_mb::Int = CSV_MAX_SIZE_MB)::Tuple{Bool,Float64,String,Any}
     sim_success = false
     sim_time    = 0.0
     sim_error   = ""
@@ -59,6 +62,13 @@ function run_simulate(ode_prob, model_dir::String,
                     println(f, join(row, ","))
                 end
             end
+            csv_bytes = filesize(sim_csv)
+            if csv_bytes > csv_max_size_mb * 1024^2
+                csv_mb = round(csv_bytes / 1024^2; digits=1)
+                @warn "Simulation CSV for $model is $(csv_mb) MB (> $(csv_max_size_mb) MB limit); skipping."
+                rm(sim_csv)
+                write(sim_csv * ".toobig", string(csv_bytes))
+            end
         catch e
             @warn "Failed to write simulation CSV for $model: $(sprint(showerror, e))"
         end

src/types.jl

@@ -8,6 +8,10 @@ const LIBRARY_VERSION = "4.1.0"
 const CMP_REL_TOL = 0.02
 const CMP_ABS_TOL = 1e-6
 
+# CSV files larger than this limit are not committed to gh-pages (GitHub
+# enforces a 100 MB hard cap; we use a conservative 20 MB soft limit).
+const CSV_MAX_SIZE_MB = 20
+
 # ── Comparison settings ────────────────────────────────────────────────────────
 
 """