Operator/Benchmark Sweep

Run the benchmark suite with multiple operator presets and compare objective/runtime metrics.

[1]:

import sys
from pathlib import Path

PROJECT_ROOT = Path.cwd().resolve()
while PROJECT_ROOT != PROJECT_ROOT.parent and not (PROJECT_ROOT / "pyproject.toml").exists():
    PROJECT_ROOT = PROJECT_ROOT.parent
if not (PROJECT_ROOT / "pyproject.toml").exists():
    raise RuntimeError(
        "Notebook must be executed within a FHOPS checkout (pyproject.toml not found)."
    )
if str(PROJECT_ROOT) not in sys.path:
    sys.path.insert(0, str(PROJECT_ROOT))


from fhops.cli.benchmarks import run_benchmark_suite

BENCH_DIR = PROJECT_ROOT / "docs/examples/analytics/data/operator_sweep"
summary = run_benchmark_suite(
    [PROJECT_ROOT / "examples/synthetic/medium/scenario.yaml"],
    BENCH_DIR,
    time_limit=5,
    sa_iters=150,
    include_mip=False,
    preset_comparisons=["explore", "stabilise"],
)
summary

                          FHOPS Benchmark Summary                           
┏━━━━━━━━━━┳━━━━━━━━┳━━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━┳━━━━━━━━━━━━━┓
┃ scenario ┃ solver ┃ preset_label ┃ objective   ┃ runtime_s ┃ assignments ┃
┡━━━━━━━━━━╇━━━━━━━━╇━━━━━━━━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━━━╇━━━━━━━━━━━━━┩
│ user-1   │ sa     │ default      │ -285097.033 │ 4.20      │ 272         │
│ user-1   │ sa     │ explore      │ -275097.033 │ 4.10      │ 272         │
│ user-1   │ sa     │ stabilise    │ -283097.033 │ 3.60      │ 272         │
└──────────┴────────┴──────────────┴─────────────┴───────────┴─────────────┘

[1]:
scenario scenario_path solver objective runtime_s assignments kpi_total_production kpi_completed_blocks kpi_staged_production kpi_remaining_work_total ... machine_costs_summary scenario_key objective_vs_mip_gap objective_vs_mip_ratio solver_category best_heuristic_solver best_heuristic_objective best_heuristic_runtime_s objective_gap_vs_best_heuristic runtime_ratio_vs_best_heuristic
0 user-1 /home/runner/work/fhops/fhops/examples/synthet... sa -285097.033 4.196794 272 0.0 0.0 97.033 97.033 ... forwarder: own=32.64763186221744, op=79.202981... user-1 <NA> <NA> heuristic sa -275097.033 4.104256 10000.0 1.022547
1 user-1 /home/runner/work/fhops/fhops/examples/synthet... sa -275097.033 4.104256 272 0.0 0.0 97.033 97.033 ... forwarder: own=32.64763186221744, op=79.202981... user-1 <NA> <NA> heuristic sa -275097.033 4.104256 0.0 1.000000
2 user-1 /home/runner/work/fhops/fhops/examples/synthet... sa -283097.033 3.603955 272 0.0 0.0 97.033 97.033 ... forwarder: own=32.64763186221744, op=79.202981... user-1 <NA> <NA> heuristic sa -275097.033 4.104256 8000.0 0.878102

3 rows × 43 columns

Objective Comparison

[2]:

objective_pivot = summary.pivot_table(index="preset_label", values="objective")
objective_pivot

[2]:
objective
preset_label
default -285097.033
explore -275097.033
stabilise -283097.033

Runtime Comparison

[3]:

runtime_pivot = summary.pivot_table(index="preset_label", values="runtime_s")
runtime_pivot

[3]:
runtime_s
preset_label
default 4.196794
explore 4.104256
stabilise 3.603955