Operator/Benchmark Sweep

Run the benchmark suite with multiple operator presets and compare objective/runtime metrics.

[1]:
import sys
from pathlib import Path

PROJECT_ROOT = Path.cwd().resolve()
while PROJECT_ROOT != PROJECT_ROOT.parent and not (PROJECT_ROOT / "pyproject.toml").exists():
    PROJECT_ROOT = PROJECT_ROOT.parent
if not (PROJECT_ROOT / "pyproject.toml").exists():
    raise RuntimeError(
        "Notebook must be executed within a FHOPS checkout (pyproject.toml not found)."
    )
if str(PROJECT_ROOT) not in sys.path:
    sys.path.insert(0, str(PROJECT_ROOT))


from fhops.cli.benchmarks import run_benchmark_suite

BENCH_DIR = PROJECT_ROOT / "docs/examples/analytics/data/operator_sweep"
summary = run_benchmark_suite(
    [PROJECT_ROOT / "examples/synthetic/medium/scenario.yaml"],
    BENCH_DIR,
    time_limit=5,
    sa_iters=150,
    include_mip=False,
    preset_comparisons=["explore", "stabilise"],
)
summary
                         FHOPS Benchmark Summary                          
┏━━━━━━━━━━┳━━━━━━━━┳━━━━━━━━━━━━━━┳━━━━━━━━━━━┳━━━━━━━━━━━┳━━━━━━━━━━━━━┓
┃ scenario ┃ solver ┃ preset_label ┃ objective ┃ runtime_s ┃ assignments ┃
┡━━━━━━━━━━╇━━━━━━━━╇━━━━━━━━━━━━━━╇━━━━━━━━━━━╇━━━━━━━━━━━╇━━━━━━━━━━━━━┩
│ user-1   │ sa     │ default      │ 61.887    │ 0.67      │ 43          │
│ user-1   │ sa     │ explore      │ 61.887    │ 0.69      │ 43          │
│ user-1   │ sa     │ stabilise    │ 61.887    │ 0.79      │ 43          │
└──────────┴────────┴──────────────┴───────────┴───────────┴─────────────┘
[1]:
scenario scenario_path solver objective runtime_s assignments kpi_total_production kpi_completed_blocks kpi_staged_production kpi_remaining_work_total ... solver_category best_heuristic_solver best_heuristic_objective best_heuristic_runtime_s objective_gap_vs_best_heuristic runtime_ratio_vs_best_heuristic objective_weights_source objective_weight_overrides objective_scenario_weights objective_scenario_weights_vs_mip_gap
0 user-1 /home/runner/work/fhops/fhops/examples/synthet... sa 61.887 0.665823 43 79.46 6.0 17.573 17.573 ... heuristic sa 61.887 0.665823 0.0 1.000000 scenario 61.887 <NA>
1 user-1 /home/runner/work/fhops/fhops/examples/synthet... sa 61.887 0.686834 43 79.46 6.0 17.573 17.573 ... heuristic sa 61.887 0.665823 0.0 1.031555 scenario 61.887 <NA>
2 user-1 /home/runner/work/fhops/fhops/examples/synthet... sa 61.887 0.786884 43 79.46 6.0 17.573 17.573 ... heuristic sa 61.887 0.665823 0.0 1.181822 scenario 61.887 <NA>

3 rows × 47 columns

Objective Comparison

[2]:
objective_pivot = summary.pivot_table(index="preset_label", values="objective")
objective_pivot
[2]:
objective
preset_label
default 61.887
explore 61.887
stabilise 61.887

Runtime Comparison

[3]:
runtime_pivot = summary.pivot_table(index="preset_label", values="runtime_s")
runtime_pivot
[3]:
runtime_s
preset_label
default 0.665823
explore 0.686834
stabilise 0.786884