Operator/Benchmark Sweep
Run the benchmark suite with multiple operator presets and compare objective/runtime metrics.
[1]:
import sys
from pathlib import Path
PROJECT_ROOT = Path.cwd().resolve()
while PROJECT_ROOT != PROJECT_ROOT.parent and not (PROJECT_ROOT / "pyproject.toml").exists():
PROJECT_ROOT = PROJECT_ROOT.parent
if not (PROJECT_ROOT / "pyproject.toml").exists():
raise RuntimeError(
"Notebook must be executed within a FHOPS checkout (pyproject.toml not found)."
)
if str(PROJECT_ROOT) not in sys.path:
sys.path.insert(0, str(PROJECT_ROOT))
from fhops.cli.benchmarks import run_benchmark_suite
BENCH_DIR = PROJECT_ROOT / "docs/examples/analytics/data/operator_sweep"
summary = run_benchmark_suite(
[PROJECT_ROOT / "examples/synthetic/medium/scenario.yaml"],
BENCH_DIR,
time_limit=5,
sa_iters=150,
include_mip=False,
preset_comparisons=["explore", "stabilise"],
)
summary
FHOPS Benchmark Summary ┏━━━━━━━━━━┳━━━━━━━━┳━━━━━━━━━━━━━━┳━━━━━━━━━━━┳━━━━━━━━━━━┳━━━━━━━━━━━━━┓ ┃ scenario ┃ solver ┃ preset_label ┃ objective ┃ runtime_s ┃ assignments ┃ ┡━━━━━━━━━━╇━━━━━━━━╇━━━━━━━━━━━━━━╇━━━━━━━━━━━╇━━━━━━━━━━━╇━━━━━━━━━━━━━┩ │ user-1 │ sa │ default │ 61.887 │ 0.67 │ 43 │ │ user-1 │ sa │ explore │ 61.887 │ 0.69 │ 43 │ │ user-1 │ sa │ stabilise │ 61.887 │ 0.79 │ 43 │ └──────────┴────────┴──────────────┴───────────┴───────────┴─────────────┘
[1]:
| scenario | scenario_path | solver | objective | runtime_s | assignments | kpi_total_production | kpi_completed_blocks | kpi_staged_production | kpi_remaining_work_total | ... | solver_category | best_heuristic_solver | best_heuristic_objective | best_heuristic_runtime_s | objective_gap_vs_best_heuristic | runtime_ratio_vs_best_heuristic | objective_weights_source | objective_weight_overrides | objective_scenario_weights | objective_scenario_weights_vs_mip_gap | |
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| 0 | user-1 | /home/runner/work/fhops/fhops/examples/synthet... | sa | 61.887 | 0.665823 | 43 | 79.46 | 6.0 | 17.573 | 17.573 | ... | heuristic | sa | 61.887 | 0.665823 | 0.0 | 1.000000 | scenario | 61.887 | <NA> | |
| 1 | user-1 | /home/runner/work/fhops/fhops/examples/synthet... | sa | 61.887 | 0.686834 | 43 | 79.46 | 6.0 | 17.573 | 17.573 | ... | heuristic | sa | 61.887 | 0.665823 | 0.0 | 1.031555 | scenario | 61.887 | <NA> | |
| 2 | user-1 | /home/runner/work/fhops/fhops/examples/synthet... | sa | 61.887 | 0.786884 | 43 | 79.46 | 6.0 | 17.573 | 17.573 | ... | heuristic | sa | 61.887 | 0.665823 | 0.0 | 1.181822 | scenario | 61.887 | <NA> |
3 rows × 47 columns
Objective Comparison
[2]:
objective_pivot = summary.pivot_table(index="preset_label", values="objective")
objective_pivot
[2]:
| objective | |
|---|---|
| preset_label | |
| default | 61.887 |
| explore | 61.887 |
| stabilise | 61.887 |
Runtime Comparison
[3]:
runtime_pivot = summary.pivot_table(index="preset_label", values="runtime_s")
runtime_pivot
[3]:
| runtime_s | |
|---|---|
| preset_label | |
| default | 0.665823 |
| explore | 0.686834 |
| stabilise | 0.786884 |