{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "4698bafe",
   "metadata": {},
   "source": [
    "# Telemetry Report Comparison\n",
    "\n",
    "This notebook demonstrates how to load multiple telemetry tuning reports,\n",
    "merge them, and visualize objective improvements using the helper\n",
    "`analyze_tuner_reports.py`.\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ee331d4e",
   "metadata": {},
   "source": [
    "## Setup\n",
    "\n",
    "We'll reuse the CLI helpers that generated the latest `demo_tuner_report.csv`\n",
    "under `docs/examples/analytics/data/tuner_reports/`.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "44107d2b",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2025-11-16T19:37:56.441691Z",
     "iopub.status.busy": "2025-11-16T19:37:56.440776Z",
     "iopub.status.idle": "2025-11-16T19:37:57.028905Z",
     "shell.execute_reply": "2025-11-16T19:37:57.027872Z"
    }
   },
   "outputs": [],
   "source": [
    "from pathlib import Path\n",
    "\n",
    "import pandas as pd\n",
    "\n",
    "try:\n",
    "    import altair as alt\n",
    "except ImportError:  # pragma: no cover - optional for CI\n",
    "    alt = None\n",
    "if alt is not None:\n",
    "    alt.renderers.enable(\"mimetype\")\n",
    "\n",
    "\n",
    "CANDIDATE_DIRS = [\n",
    "    Path(\"data/tuner_reports\"),\n",
    "    Path(\"../data/tuner_reports\"),\n",
    "    Path(\"examples/analytics/data/tuner_reports\"),\n",
    "    Path(\"docs/examples/analytics/data/tuner_reports\"),\n",
    "    Path.cwd() / \"data/tuner_reports\",\n",
    "    Path.cwd() / \"examples/analytics/data/tuner_reports\",\n",
    "]\n",
    "\n",
    "DATA_DIR = None\n",
    "for candidate in CANDIDATE_DIRS:\n",
    "    candidate = candidate.resolve() if not candidate.is_absolute() else candidate\n",
    "    if (candidate / \"demo_tuner_report.csv\").exists():\n",
    "        DATA_DIR = candidate\n",
    "        break\n",
    "\n",
    "if DATA_DIR is None:\n",
    "    print(\"Telemetry reports not found. Generating synthetic sample data instead.\")\n",
    "    BASELINE = None\n",
    "    EXPERIMENT = None\n",
    "else:\n",
    "    BASELINE = DATA_DIR / \"demo_tuner_report.csv\"\n",
    "    EXPERIMENT = DATA_DIR / \"demo_tuner_report.csv\"  # replace with new report as needed"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "643aa5c5",
   "metadata": {},
   "source": [
    "## Load Reports\n",
    "We can either call the helper script via `subprocess` or load the CSVs directly\n",
    "for ad-hoc comparisons.\n",
    "\n",
    "\n",
    "> CI executes with a lightweight telemetry sweep, so the required CSV files are generated automatically.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "30dfc140",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2025-11-16T19:37:57.032857Z",
     "iopub.status.busy": "2025-11-16T19:37:57.032629Z",
     "iopub.status.idle": "2025-11-16T19:37:57.066807Z",
     "shell.execute_reply": "2025-11-16T19:37:57.066155Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>algorithm</th>\n",
       "      <th>scenario</th>\n",
       "      <th>runs</th>\n",
       "      <th>best_objective</th>\n",
       "      <th>mean_objective</th>\n",
       "      <th>best_run_id</th>\n",
       "      <th>best_started_at</th>\n",
       "      <th>best_config</th>\n",
       "      <th>summary_best</th>\n",
       "      <th>summary_configurations</th>\n",
       "      <th>summary_updated_at</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>bayes</td>\n",
       "      <td>FHOPS MiniToy</td>\n",
       "      <td>1</td>\n",
       "      <td>3.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>73a84eb549714c6da52b92ffda8752b6</td>\n",
       "      <td>2025-11-11T21:15:55+00:00</td>\n",
       "      <td>iters=50; operators=(block_insertion:1.9858747...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>baseline</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>grid</td>\n",
       "      <td>FHOPS MiniToy</td>\n",
       "      <td>1</td>\n",
       "      <td>9.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>f590aa00bb3543debcaebd6bdf288e6b</td>\n",
       "      <td>2025-11-11T21:15:54+00:00</td>\n",
       "      <td>iters=50; operators=(block_insertion:0.0, cros...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>baseline</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>random</td>\n",
       "      <td>FHOPS MiniToy</td>\n",
       "      <td>1</td>\n",
       "      <td>-6.0</td>\n",
       "      <td>-6.0</td>\n",
       "      <td>ad611c7346f84950be678527ae2de434</td>\n",
       "      <td>2025-11-11T21:15:53+00:00</td>\n",
       "      <td>batch_size=2; iters=50; operators=(block_inser...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>baseline</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>bayes</td>\n",
       "      <td>FHOPS MiniToy</td>\n",
       "      <td>1</td>\n",
       "      <td>3.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>73a84eb549714c6da52b92ffda8752b6</td>\n",
       "      <td>2025-11-11T21:15:55+00:00</td>\n",
       "      <td>iters=50; operators=(block_insertion:1.9858747...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>experiment</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>grid</td>\n",
       "      <td>FHOPS MiniToy</td>\n",
       "      <td>1</td>\n",
       "      <td>9.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>f590aa00bb3543debcaebd6bdf288e6b</td>\n",
       "      <td>2025-11-11T21:15:54+00:00</td>\n",
       "      <td>iters=50; operators=(block_insertion:0.0, cros...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>experiment</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>random</td>\n",
       "      <td>FHOPS MiniToy</td>\n",
       "      <td>1</td>\n",
       "      <td>-6.0</td>\n",
       "      <td>-6.0</td>\n",
       "      <td>ad611c7346f84950be678527ae2de434</td>\n",
       "      <td>2025-11-11T21:15:53+00:00</td>\n",
       "      <td>batch_size=2; iters=50; operators=(block_inser...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>experiment</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  algorithm       scenario  runs  best_objective  mean_objective  \\\n",
       "0     bayes  FHOPS MiniToy     1             3.0             3.0   \n",
       "1      grid  FHOPS MiniToy     1             9.0             9.0   \n",
       "2    random  FHOPS MiniToy     1            -6.0            -6.0   \n",
       "3     bayes  FHOPS MiniToy     1             3.0             3.0   \n",
       "4      grid  FHOPS MiniToy     1             9.0             9.0   \n",
       "5    random  FHOPS MiniToy     1            -6.0            -6.0   \n",
       "\n",
       "                        best_run_id            best_started_at  \\\n",
       "0  73a84eb549714c6da52b92ffda8752b6  2025-11-11T21:15:55+00:00   \n",
       "1  f590aa00bb3543debcaebd6bdf288e6b  2025-11-11T21:15:54+00:00   \n",
       "2  ad611c7346f84950be678527ae2de434  2025-11-11T21:15:53+00:00   \n",
       "3  73a84eb549714c6da52b92ffda8752b6  2025-11-11T21:15:55+00:00   \n",
       "4  f590aa00bb3543debcaebd6bdf288e6b  2025-11-11T21:15:54+00:00   \n",
       "5  ad611c7346f84950be678527ae2de434  2025-11-11T21:15:53+00:00   \n",
       "\n",
       "                                         best_config  summary_best  \\\n",
       "0  iters=50; operators=(block_insertion:1.9858747...           NaN   \n",
       "1  iters=50; operators=(block_insertion:0.0, cros...           NaN   \n",
       "2  batch_size=2; iters=50; operators=(block_inser...           NaN   \n",
       "3  iters=50; operators=(block_insertion:1.9858747...           NaN   \n",
       "4  iters=50; operators=(block_insertion:0.0, cros...           NaN   \n",
       "5  batch_size=2; iters=50; operators=(block_inser...           NaN   \n",
       "\n",
       "   summary_configurations  summary_updated_at       label  \n",
       "0                     NaN                 NaN    baseline  \n",
       "1                     NaN                 NaN    baseline  \n",
       "2                     NaN                 NaN    baseline  \n",
       "3                     NaN                 NaN  experiment  \n",
       "4                     NaN                 NaN  experiment  \n",
       "5                     NaN                 NaN  experiment  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "if BASELINE is None or EXPERIMENT is None or not BASELINE.exists() or not EXPERIMENT.exists():\n",
    "    print(\"Telemetry reports not found. Generating synthetic sample data instead.\")\n",
    "    sample = pd.DataFrame(\n",
    "        [\n",
    "            {\n",
    "                \"algorithm\": \"random\",\n",
    "                \"scenario\": \"SampleScenario\",\n",
    "                \"best_objective\": 6.2,\n",
    "                \"mean_objective\": 5.9,\n",
    "                \"runs\": 2,\n",
    "                \"label\": \"baseline\",\n",
    "                \"machine_costs_summary\": \"feller_buncher: own=90, op=110, rep=30, usage=10,000h\",\n",
    "                \"repair_usage_alert\": \"\",\n",
    "            },\n",
    "            {\n",
    "                \"algorithm\": \"random\",\n",
    "                \"scenario\": \"SampleScenario\",\n",
    "                \"best_objective\": 7.1,\n",
    "                \"mean_objective\": 6.6,\n",
    "                \"runs\": 2,\n",
    "                \"label\": \"experiment\",\n",
    "                \"machine_costs_summary\": \"feller_buncher: own=90, op=110, rep=30, usage=10,000h\",\n",
    "                \"repair_usage_alert\": \"\",\n",
    "            },\n",
    "            {\n",
    "                \"algorithm\": \"grid\",\n",
    "                \"scenario\": \"SampleScenario\",\n",
    "                \"best_objective\": 6.8,\n",
    "                \"mean_objective\": 6.3,\n",
    "                \"runs\": 2,\n",
    "                \"label\": \"baseline\",\n",
    "                \"machine_costs_summary\": \"grapple_skidder: own=70, op=95, rep=32, usage=5,000h\",\n",
    "                \"repair_usage_alert\": \"M-GRID\",\n",
    "            },\n",
    "            {\n",
    "                \"algorithm\": \"grid\",\n",
    "                \"scenario\": \"SampleScenario\",\n",
    "                \"best_objective\": 7.4,\n",
    "                \"mean_objective\": 6.9,\n",
    "                \"runs\": 2,\n",
    "                \"label\": \"experiment\",\n",
    "                \"machine_costs_summary\": \"grapple_skidder: own=70, op=95, rep=32, usage=15,000h\",\n",
    "                \"repair_usage_alert\": \"M-GRID\",\n",
    "            },\n",
    "            {\n",
    "                \"algorithm\": \"bayes\",\n",
    "                \"scenario\": \"SampleScenario\",\n",
    "                \"best_objective\": 7.0,\n",
    "                \"mean_objective\": 6.7,\n",
    "                \"runs\": 1,\n",
    "                \"label\": \"baseline\",\n",
    "                \"machine_costs_summary\": \"swing_yarder: own=150, op=230, rep=68, usage=10,000h\",\n",
    "                \"repair_usage_alert\": \"\",\n",
    "            },\n",
    "            {\n",
    "                \"algorithm\": \"bayes\",\n",
    "                \"scenario\": \"SampleScenario\",\n",
    "                \"best_objective\": 7.6,\n",
    "                \"mean_objective\": 7.2,\n",
    "                \"runs\": 1,\n",
    "                \"label\": \"experiment\",\n",
    "                \"machine_costs_summary\": \"swing_yarder: own=150, op=230, rep=68, usage=20,000h\",\n",
    "                \"repair_usage_alert\": \"M-BAYES\",\n",
    "            },\n",
    "        ]\n",
    "    )\n",
    "    baseline = sample[sample[\"label\"] == \"baseline\"].drop(columns=\"label\").reset_index(drop=True)\n",
    "    experiment = (\n",
    "        sample[sample[\"label\"] == \"experiment\"].drop(columns=\"label\").reset_index(drop=True)\n",
    "    )\n",
    "else:\n",
    "    baseline = pd.read_csv(BASELINE)\n",
    "    experiment = pd.read_csv(EXPERIMENT)\n",
    "merged = pd.concat(\n",
    "    [baseline.assign(label=\"baseline\"), experiment.assign(label=\"experiment\")], ignore_index=True\n",
    ")\n",
    "display(merged)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "bda5e69e",
   "metadata": {},
   "source": [
    "## Visualize Best Objectives\n",
    "Plot the best objective per algorithm to see improvements/deltas.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "938031d6",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2025-11-16T19:37:57.069420Z",
     "iopub.status.busy": "2025-11-16T19:37:57.069223Z",
     "iopub.status.idle": "2025-11-16T19:37:57.108636Z",
     "shell.execute_reply": "2025-11-16T19:37:57.108056Z"
    }
   },
   "outputs": [
    {
     "data": {
      "application/vnd.vegalite.v6.json": {
       "$schema": "https://vega.github.io/schema/vega-lite/v6.1.0.json",
       "config": {
        "view": {
         "continuousHeight": 300,
         "continuousWidth": 300
        }
       },
       "data": {
        "name": "data-47d297eef74447314bdaaf72f1613327"
       },
       "datasets": {
        "data-47d297eef74447314bdaaf72f1613327": [
         {
          "algorithm": "bayes",
          "best_config": "iters=50; operators=(block_insertion:1.9858747332662832, cross_exchange:1.989984203867689, mobilisation_shake:1.142951075941873)",
          "best_objective": 3,
          "best_run_id": "73a84eb549714c6da52b92ffda8752b6",
          "best_started_at": "2025-11-11T21:15:55+00:00",
          "label": "baseline",
          "mean_objective": 3,
          "runs": 1,
          "scenario": "FHOPS MiniToy",
          "summary_best": null,
          "summary_configurations": null,
          "summary_updated_at": null
         },
         {
          "algorithm": "grid",
          "best_config": "iters=50; operators=(block_insertion:0.0, cross_exchange:0.0, mobilisation_shake:0.0)",
          "best_objective": 9,
          "best_run_id": "f590aa00bb3543debcaebd6bdf288e6b",
          "best_started_at": "2025-11-11T21:15:54+00:00",
          "label": "baseline",
          "mean_objective": 9,
          "runs": 1,
          "scenario": "FHOPS MiniToy",
          "summary_best": null,
          "summary_configurations": null,
          "summary_updated_at": null
         },
         {
          "algorithm": "random",
          "best_config": "batch_size=2; iters=50; operators=(block_insertion:0.0, cross_exchange:0.767, mobilisation_shake:0.0)",
          "best_objective": -6,
          "best_run_id": "ad611c7346f84950be678527ae2de434",
          "best_started_at": "2025-11-11T21:15:53+00:00",
          "label": "baseline",
          "mean_objective": -6,
          "runs": 1,
          "scenario": "FHOPS MiniToy",
          "summary_best": null,
          "summary_configurations": null,
          "summary_updated_at": null
         },
         {
          "algorithm": "bayes",
          "best_config": "iters=50; operators=(block_insertion:1.9858747332662832, cross_exchange:1.989984203867689, mobilisation_shake:1.142951075941873)",
          "best_objective": 3,
          "best_run_id": "73a84eb549714c6da52b92ffda8752b6",
          "best_started_at": "2025-11-11T21:15:55+00:00",
          "label": "experiment",
          "mean_objective": 3,
          "runs": 1,
          "scenario": "FHOPS MiniToy",
          "summary_best": null,
          "summary_configurations": null,
          "summary_updated_at": null
         },
         {
          "algorithm": "grid",
          "best_config": "iters=50; operators=(block_insertion:0.0, cross_exchange:0.0, mobilisation_shake:0.0)",
          "best_objective": 9,
          "best_run_id": "f590aa00bb3543debcaebd6bdf288e6b",
          "best_started_at": "2025-11-11T21:15:54+00:00",
          "label": "experiment",
          "mean_objective": 9,
          "runs": 1,
          "scenario": "FHOPS MiniToy",
          "summary_best": null,
          "summary_configurations": null,
          "summary_updated_at": null
         },
         {
          "algorithm": "random",
          "best_config": "batch_size=2; iters=50; operators=(block_insertion:0.0, cross_exchange:0.767, mobilisation_shake:0.0)",
          "best_objective": -6,
          "best_run_id": "ad611c7346f84950be678527ae2de434",
          "best_started_at": "2025-11-11T21:15:53+00:00",
          "label": "experiment",
          "mean_objective": -6,
          "runs": 1,
          "scenario": "FHOPS MiniToy",
          "summary_best": null,
          "summary_configurations": null,
          "summary_updated_at": null
         }
        ]
       },
       "encoding": {
        "color": {
         "field": "algorithm",
         "type": "nominal"
        },
        "column": {
         "field": "scenario",
         "type": "nominal"
        },
        "x": {
         "field": "label",
         "type": "nominal"
        },
        "y": {
         "field": "best_objective",
         "type": "quantitative"
        }
       },
       "mark": {
        "point": true,
        "type": "line"
       }
      },
      "text/plain": [
       "<VegaLite 6 object>\n",
       "\n",
       "If you see this message, it means the renderer has not been properly enabled\n",
       "for the frontend that you are using. For more information, see\n",
       "https://altair-viz.github.io/user_guide/display_frontends.html#troubleshooting\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "if alt is None or merged.empty:\n",
    "    display(\n",
    "        \"Altair not installed or no data available; install `altair` or supply reports to render the chart.\"\n",
    "    )\n",
    "else:\n",
    "    chart = (\n",
    "        alt.Chart(merged)\n",
    "        .mark_line(point=True)\n",
    "        .encode(x=\"label:N\", y=\"best_objective:Q\", color=\"algorithm:N\", column=\"scenario:N\")\n",
    "    )\n",
    "    chart\n",
    "    from IPython.display import display\n",
    "\n",
    "    display(chart)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ff50867e",
   "metadata": {},
   "source": [
    "## Delta Table\n",
    "Join baseline and experiment to compute deltas using pandas.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "9fa0c3c1",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2025-11-16T19:37:57.111642Z",
     "iopub.status.busy": "2025-11-16T19:37:57.111498Z",
     "iopub.status.idle": "2025-11-16T19:37:57.127058Z",
     "shell.execute_reply": "2025-11-16T19:37:57.126495Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>algorithm</th>\n",
       "      <th>scenario</th>\n",
       "      <th>runs_baseline</th>\n",
       "      <th>best_objective_baseline</th>\n",
       "      <th>mean_objective_baseline</th>\n",
       "      <th>best_run_id_baseline</th>\n",
       "      <th>best_started_at_baseline</th>\n",
       "      <th>best_config_baseline</th>\n",
       "      <th>summary_best_baseline</th>\n",
       "      <th>summary_configurations_baseline</th>\n",
       "      <th>...</th>\n",
       "      <th>runs_experiment</th>\n",
       "      <th>best_objective_experiment</th>\n",
       "      <th>mean_objective_experiment</th>\n",
       "      <th>best_run_id_experiment</th>\n",
       "      <th>best_started_at_experiment</th>\n",
       "      <th>best_config_experiment</th>\n",
       "      <th>summary_best_experiment</th>\n",
       "      <th>summary_configurations_experiment</th>\n",
       "      <th>summary_updated_at_experiment</th>\n",
       "      <th>best_delta</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>bayes</td>\n",
       "      <td>FHOPS MiniToy</td>\n",
       "      <td>1</td>\n",
       "      <td>3.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>73a84eb549714c6da52b92ffda8752b6</td>\n",
       "      <td>2025-11-11T21:15:55+00:00</td>\n",
       "      <td>iters=50; operators=(block_insertion:1.9858747...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>3.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>73a84eb549714c6da52b92ffda8752b6</td>\n",
       "      <td>2025-11-11T21:15:55+00:00</td>\n",
       "      <td>iters=50; operators=(block_insertion:1.9858747...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>grid</td>\n",
       "      <td>FHOPS MiniToy</td>\n",
       "      <td>1</td>\n",
       "      <td>9.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>f590aa00bb3543debcaebd6bdf288e6b</td>\n",
       "      <td>2025-11-11T21:15:54+00:00</td>\n",
       "      <td>iters=50; operators=(block_insertion:0.0, cros...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>9.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>f590aa00bb3543debcaebd6bdf288e6b</td>\n",
       "      <td>2025-11-11T21:15:54+00:00</td>\n",
       "      <td>iters=50; operators=(block_insertion:0.0, cros...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>random</td>\n",
       "      <td>FHOPS MiniToy</td>\n",
       "      <td>1</td>\n",
       "      <td>-6.0</td>\n",
       "      <td>-6.0</td>\n",
       "      <td>ad611c7346f84950be678527ae2de434</td>\n",
       "      <td>2025-11-11T21:15:53+00:00</td>\n",
       "      <td>batch_size=2; iters=50; operators=(block_inser...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>-6.0</td>\n",
       "      <td>-6.0</td>\n",
       "      <td>ad611c7346f84950be678527ae2de434</td>\n",
       "      <td>2025-11-11T21:15:53+00:00</td>\n",
       "      <td>batch_size=2; iters=50; operators=(block_inser...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3 rows × 21 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "  algorithm       scenario  runs_baseline  best_objective_baseline  \\\n",
       "0     bayes  FHOPS MiniToy              1                      3.0   \n",
       "1      grid  FHOPS MiniToy              1                      9.0   \n",
       "2    random  FHOPS MiniToy              1                     -6.0   \n",
       "\n",
       "   mean_objective_baseline              best_run_id_baseline  \\\n",
       "0                      3.0  73a84eb549714c6da52b92ffda8752b6   \n",
       "1                      9.0  f590aa00bb3543debcaebd6bdf288e6b   \n",
       "2                     -6.0  ad611c7346f84950be678527ae2de434   \n",
       "\n",
       "    best_started_at_baseline  \\\n",
       "0  2025-11-11T21:15:55+00:00   \n",
       "1  2025-11-11T21:15:54+00:00   \n",
       "2  2025-11-11T21:15:53+00:00   \n",
       "\n",
       "                                best_config_baseline  summary_best_baseline  \\\n",
       "0  iters=50; operators=(block_insertion:1.9858747...                    NaN   \n",
       "1  iters=50; operators=(block_insertion:0.0, cros...                    NaN   \n",
       "2  batch_size=2; iters=50; operators=(block_inser...                    NaN   \n",
       "\n",
       "   summary_configurations_baseline  ...  runs_experiment  \\\n",
       "0                              NaN  ...                1   \n",
       "1                              NaN  ...                1   \n",
       "2                              NaN  ...                1   \n",
       "\n",
       "   best_objective_experiment  mean_objective_experiment  \\\n",
       "0                        3.0                        3.0   \n",
       "1                        9.0                        9.0   \n",
       "2                       -6.0                       -6.0   \n",
       "\n",
       "             best_run_id_experiment best_started_at_experiment  \\\n",
       "0  73a84eb549714c6da52b92ffda8752b6  2025-11-11T21:15:55+00:00   \n",
       "1  f590aa00bb3543debcaebd6bdf288e6b  2025-11-11T21:15:54+00:00   \n",
       "2  ad611c7346f84950be678527ae2de434  2025-11-11T21:15:53+00:00   \n",
       "\n",
       "                              best_config_experiment summary_best_experiment  \\\n",
       "0  iters=50; operators=(block_insertion:1.9858747...                     NaN   \n",
       "1  iters=50; operators=(block_insertion:0.0, cros...                     NaN   \n",
       "2  batch_size=2; iters=50; operators=(block_inser...                     NaN   \n",
       "\n",
       "   summary_configurations_experiment  summary_updated_at_experiment  \\\n",
       "0                                NaN                            NaN   \n",
       "1                                NaN                            NaN   \n",
       "2                                NaN                            NaN   \n",
       "\n",
       "   best_delta  \n",
       "0         0.0  \n",
       "1         0.0  \n",
       "2         0.0  \n",
       "\n",
       "[3 rows x 21 columns]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "comparison = baseline.merge(\n",
    "    experiment, on=[\"algorithm\", \"scenario\"], suffixes=(\"_baseline\", \"_experiment\")\n",
    ")\n",
    "comparison[\"best_delta\"] = (\n",
    "    comparison[\"best_objective_experiment\"] - comparison[\"best_objective_baseline\"]\n",
    ")\n",
    "comparison"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "84a020d5",
   "metadata": {},
   "source": [
    "## Machine-Cost Summary Pivot\n",
    "Telemetry reports now carry a `machine_costs_summary` column summarizing the owning/operating/repair allowance (and usage bucket) applied during the best run.\n",
    "The snippet below groups the merged report by scenario, label, and machine-cost summary so you can spot assumption drift over time.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "a86ae641",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2025-11-16T19:37:57.129978Z",
     "iopub.status.busy": "2025-11-16T19:37:57.129839Z",
     "iopub.status.idle": "2025-11-16T19:37:57.134703Z",
     "shell.execute_reply": "2025-11-16T19:37:57.134111Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'`machine_costs_summary` column not present in the loaded reports.'"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "if \"machine_costs_summary\" not in merged.columns or merged[\"machine_costs_summary\"].isna().all():\n",
    "    display(\"`machine_costs_summary` column not present in the loaded reports.\")\n",
    "else:\n",
    "    machine_costs_pivot = (\n",
    "        merged.dropna(subset=[\"machine_costs_summary\"])\n",
    "        .groupby([\"scenario\", \"label\", \"machine_costs_summary\"], dropna=False)[\"best_objective\"]\n",
    "        .agg([\"max\", \"mean\", \"count\"])\n",
    "        .reset_index()\n",
    "        .rename(columns={\"max\": \"best_objective\", \"mean\": \"mean_objective\", \"count\": \"runs\"})\n",
    "    )\n",
    "    display(machine_costs_pivot)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7fb27b941602401d91542211134fc71a",
   "metadata": {},
   "source": [
    "## Machine-Cost Trend Chart\n",
    "Use the grouped chart below to see how machine-cost buckets shift between reports (baseline vs experiment or historical labels).\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "acae54e37e7d407bbb7b55eff062a284",
   "metadata": {},
   "outputs": [],
   "source": [
    "if alt is None or merged.empty:\n",
    "    display(\n",
    "        \"Altair not installed or no data available; install `altair` or supply reports to render the chart.\"\n",
    "    )\n",
    "elif \"machine_costs_summary\" not in merged.columns or merged[\"machine_costs_summary\"].isna().all():\n",
    "    display(\"`machine_costs_summary` column not present in the loaded reports.\")\n",
    "else:\n",
    "    trend_df = (\n",
    "        merged.dropna(subset=[\"machine_costs_summary\"])\n",
    "        .groupby([\"label\", \"machine_costs_summary\"], dropna=False)[\"algorithm\"]\n",
    "        .count()\n",
    "        .reset_index(name=\"runs\")\n",
    "    )\n",
    "    chart = (\n",
    "        alt.Chart(trend_df)\n",
    "        .mark_bar()\n",
    "        .encode(\n",
    "            x=\"label:N\",\n",
    "            y=\"runs:Q\",\n",
    "            color=\"machine_costs_summary:N\",\n",
    "            column=\"machine_costs_summary:N\",\n",
    "        )\n",
    "    )\n",
    "    chart"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1231ab1f",
   "metadata": {},
   "source": [
    "Use this notebook as a starting point for richer analytics (e.g., trendlines\n",
    "across multiple nightly reports)."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4024e2a8",
   "metadata": {},
   "source": [
    "## Compare Against Historical Summary\n",
    "If you have downloaded history artefacts, you can re-use the helper script to\n",
    "compare the latest report against prior snapshots. Update `HISTORY_DIR` to point\n",
    "to your archive.\n",
    "\n",
    "\n",
    "This notebook focuses on objectives. See `tuner_history_analysis.ipynb` for multi-metric (KPI) trends."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "e9831f83",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2025-11-16T19:37:57.137791Z",
     "iopub.status.busy": "2025-11-16T19:37:57.137643Z",
     "iopub.status.idle": "2025-11-16T19:37:57.141936Z",
     "shell.execute_reply": "2025-11-16T19:37:57.141325Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "History directory not found. Run analyze_tuner_reports.py manually.\n"
     ]
    }
   ],
   "source": [
    "from pathlib import Path\n",
    "\n",
    "HISTORY_DIR = Path(\"docs/examples/analytics/data/tuner_reports\")\n",
    "if HISTORY_DIR.exists():\n",
    "    !python scripts/analyze_tuner_reports.py --report latest={BASELINE} --history-dir {HISTORY_DIR} --out-history-markdown tmp/notebook_history.md\n",
    "    display(Path(\"tmp/notebook_history.md\").read_text())\n",
    "else:\n",
    "    print(\"History directory not found. Run analyze_tuner_reports.py manually.\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6d252714",
   "metadata": {},
   "source": [
    "## Repair-Usage Alerts\n",
    "When the KPI layer emits `repair_usage_alert`, the table below highlights which scenarios/labels deviated from the baseline 10 000 h FPInnovations bucket.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c1d18abe",
   "metadata": {},
   "outputs": [],
   "source": [
    "if \"repair_usage_alert\" not in merged.columns or merged[\"repair_usage_alert\"].isna().all():\n",
    "    display(\"`repair_usage_alert` column not present in the loaded reports.\")\n",
    "else:\n",
    "    alert_summary = (\n",
    "        merged.dropna(subset=[\"repair_usage_alert\"])\n",
    "        .groupby([\"scenario\", \"label\", \"repair_usage_alert\"], dropna=False)[\"best_objective\"]\n",
    "        .agg([\"max\", \"mean\", \"count\"])\n",
    "        .reset_index()\n",
    "        .rename(columns={\"max\": \"best_objective\", \"mean\": \"mean_objective\", \"count\": \"runs\"})\n",
    "    )\n",
    "    display(alert_summary)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": ".venv",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}