"""Runtime state, preparation, and scenario execution for the FABLE web portal.
The service is deliberately a thin presentation layer over the notebook APIs. Model execution and
workbook discovery happen in background jobs; every response is built from a real run or a real
artifact, never from browser-only state.
"""
from __future__ import annotations
import json
import re
import threading
import time
import uuid
from collections.abc import Callable, Mapping, Sequence
from dataclasses import dataclass, field
from datetime import UTC, datetime, timedelta
from pathlib import Path
from typing import Any, Literal
import pandas as pd
from fable_pyculator import __version__
from fable_pyculator.notebook import build_notebook_spec, load_generated_model, run_notebook_loop
from fable_pyculator.web.artifacts import (
DEFAULT_VERSIONS,
artifact_paths,
artifact_summary,
bundle_summaries,
file_info,
materialize_generated_model,
model_checksum_status,
)
from fable_pyculator.web.assistant import DEFAULT_AGENT_MODEL
from fable_pyculator.web.chats import ChatStore
from fable_pyculator.web.cheatsheet import CheatSheet, ensure_cheat_sheet
from fable_pyculator.web.spec_io import jsonable, spec_from_payload, spec_summary, spec_to_payload
__all__ = [
"Job",
"JobManager",
"PortalConfig",
"PortalError",
"PortalState",
"PreparedModel",
]
JobStatus = Literal["queued", "running", "ok", "error"]
Report = Callable[[int, str], None]
SPEC_CACHE_VERSION = "2026-11-06-ghg-output-components"
NATIVE_LAST_YEAR = 2050
EXTENSION_YEARS = (2055, 2060, 2065, 2070)
EXTENSION_NOTE = (
"2055-2070 are trend extensions extrapolated from the 2035-2050 linear trend, not native FABLE Calculator results."
)
_PRESET_STATE_KEYS = (
"version",
"x",
"y",
"z",
"colour",
"size",
"floor_dimension",
"floor",
"front",
"warning_free",
"brushes",
"reference",
"point",
)
def _clean_timeline_view(
name: Any,
timeline_filter: Any,
saved_at: Any = None,
author: Any = None,
note: Any = None,
review_status: Any = None,
) -> dict[str, Any]:
"""Validate and normalize one shared timeline filter view."""
if not isinstance(name, str) or not name.strip():
raise ValueError("timeline view name must be a non-empty string")
if not isinstance(timeline_filter, Mapping):
raise ValueError("timeline view filter must be an object")
clean_name = name.strip()[:40]
if not clean_name:
raise ValueError("timeline view name must be a non-empty string")
window = str(timeline_filter.get("window", "all"))
action = str(timeline_filter.get("action", ""))
if window not in {"all", "24h", "7d", "30d"}:
raise ValueError("timeline view window must be all, 24h, 7d, or 30d")
if action not in {"", "created", "updated", "reviewed"}:
raise ValueError("timeline view action is invalid")
timestamp = saved_at if isinstance(saved_at, str) and len(saved_at) <= 40 else None
if timestamp is None:
timestamp = datetime.now(UTC).isoformat(timespec="seconds")
status = str(review_status or "draft").strip().lower()
if status not in {"draft", "reviewed"}:
raise ValueError("timeline view review_status must be draft or reviewed")
return {
"name": clean_name,
"saved_at": timestamp,
"author": author.strip()[:40] if isinstance(author, str) else "",
"note": note.strip()[:200] if isinstance(note, str) else "",
"review_status": status,
"filter": {
"author": str(timeline_filter.get("author", ""))[:40],
"action": action,
"preset": str(timeline_filter.get("preset", ""))[:40],
"window": window,
},
}
def _clean_preset_history(value: Any) -> list[dict[str, Any]]:
"""Validate the server-owned review history list."""
if not isinstance(value, (list, tuple)):
return []
history: list[dict[str, Any]] = []
for event in value[-10:]:
if not isinstance(event, Mapping):
continue
action = str(event.get("action", "")).strip().lower()
status = str(event.get("review_status", "draft")).strip().lower()
if action not in {"created", "updated", "reviewed"} or status not in {"draft", "reviewed"}:
continue
history.append(
{
"at": str(event.get("at", ""))[:40],
"action": action,
"author": str(event.get("author", ""))[:40],
"note": str(event.get("note", ""))[:80],
"review_status": status,
}
)
return history[-10:]
def _clean_optimization_preset(
name: Any,
state: Any,
saved_at: Any = None,
author: Any = None,
note: Any = None,
review_status: Any = None,
) -> dict[str, Any]:
"""Validate and normalize one shared Optimization preset."""
if not isinstance(name, str) or not name.strip():
raise ValueError("preset name must be a non-empty string")
if not isinstance(state, Mapping):
raise ValueError("preset state must be an object")
clean_name = name.strip()[:40]
if not clean_name:
raise ValueError("preset name must be a non-empty string")
clean_state: dict[str, Any] = {}
for key in _PRESET_STATE_KEYS:
if key not in state:
continue
value = state[key]
if key in {"version", "x", "y", "z", "colour", "size", "floor_dimension"}:
clean_state[key] = str(value)[:60]
elif key == "floor":
number = float(value)
if not 0 <= number <= 100:
raise ValueError("preset floor must be between 0 and 100")
clean_state[key] = number
elif key in {"front", "warning_free"}:
clean_state[key] = bool(value)
elif key == "brushes":
if not isinstance(value, (list, tuple)) or len(value) > 20:
raise ValueError("preset brushes must be a list of at most 20 entries")
brushes: list[dict[str, Any]] = []
for brush in value:
if not isinstance(brush, Mapping):
raise ValueError("each preset brush must be an object")
axis_key = brush.get("axisKey")
if not isinstance(axis_key, str) or not axis_key:
raise ValueError("each preset brush needs an axisKey")
start = float(brush.get("start"))
end = float(brush.get("end"))
if not 0 <= start <= 1 or not 0 <= end <= 1:
raise ValueError("preset brush ranges must be between 0 and 1")
brushes.append({"axisKey": axis_key[:60], "start": start, "end": end})
clean_state[key] = brushes
elif key in {"reference", "point"}:
clean_state[key] = None if value in (None, "") else str(value)[:20]
if not clean_state.get("version"):
raise ValueError("preset state needs a version")
timestamp = saved_at if isinstance(saved_at, str) and len(saved_at) <= 40 else None
if timestamp is None:
timestamp = datetime.now(UTC).isoformat(timespec="seconds")
clean_author = author.strip()[:40] if isinstance(author, str) else ""
clean_note = note.strip()[:200] if isinstance(note, str) else ""
status = str(review_status or "draft").strip().lower()
if status not in {"draft", "reviewed"}:
raise ValueError("preset review_status must be draft or reviewed")
return {
"name": clean_name,
"saved_at": timestamp,
"author": clean_author,
"note": clean_note,
"review_status": status,
"state": clean_state,
}
def _clean_restored_preset(value: Any) -> dict[str, Any]:
"""Validate one preset from a shared-library backup, including its history."""
if not isinstance(value, Mapping):
raise ValueError("each preset must be an object")
preset = _clean_optimization_preset(
value.get("name"),
value.get("state"),
value.get("saved_at"),
author=value.get("author"),
note=value.get("note"),
review_status=value.get("review_status"),
)
preset["history"] = _clean_preset_history(value.get("history"))
return preset
def _preset_event_key(event: Mapping[str, Any]) -> tuple[str, str, str, str, str]:
"""Return the exact comparison key for one bounded history event."""
return (
str(event.get("at", "")),
str(event.get("action", "")),
str(event.get("author", "")),
str(event.get("note", "")),
str(event.get("review_status", "")),
)
def _merge_preset_history(
existing: Any,
incoming: Any,
) -> list[dict[str, Any]]:
"""Merge two bounded review histories without duplicating events."""
merged: list[dict[str, Any]] = []
seen: set[tuple[str, str, str, str, str]] = set()
for event in [*(_clean_preset_history(existing)), *(_clean_preset_history(incoming))]:
key = _preset_event_key(event)
if key in seen:
continue
seen.add(key)
merged.append(event)
merged.sort(key=lambda event: str(event.get("at", "")))
return merged[-10:]
[docs]
class PortalError(RuntimeError):
"""A user-facing portal error with an HTTP status hint."""
def __init__(self, message: str, *, status_code: int = 400) -> None:
super().__init__(message)
self.status_code = status_code
[docs]
@dataclass(slots=True)
class Job:
"""One background preparation or scenario-run job."""
job_id: str
kind: str
status: JobStatus = "queued"
progress: int = 0
message: str = "queued"
result: Any = None
error: str | None = None
created_at: str = field(default_factory=lambda: datetime.now(UTC).isoformat(timespec="seconds"))
started_at: str | None = None
finished_at: str | None = None
[docs]
def to_dict(self) -> dict[str, Any]:
"""Return a JSON-ready snapshot."""
return {
"job_id": self.job_id,
"kind": self.kind,
"status": self.status,
"progress": self.progress,
"message": self.message,
"result": self.result,
"error": self.error,
"created_at": self.created_at,
"started_at": self.started_at,
"finished_at": self.finished_at,
}
[docs]
class JobManager:
"""Run callables on daemon threads and expose pollable status snapshots."""
def __init__(self, *, keep: int = 50) -> None:
self._jobs: dict[str, Job] = {}
self._lock = threading.Lock()
self._keep = keep
[docs]
def start(self, kind: str, target: Callable[[Report], Any]) -> Job:
"""Start a job and return its initial snapshot."""
job = Job(job_id=uuid.uuid4().hex[:12], kind=kind)
with self._lock:
self._jobs[job.job_id] = job
self._trim()
thread = threading.Thread(target=self._run, args=(job, target), name=f"portal-{job.job_id}", daemon=True)
thread.start()
return job
[docs]
def get(self, job_id: str) -> Job:
"""Return one job or raise a 404 portal error."""
with self._lock:
job = self._jobs.get(job_id)
if job is None:
raise PortalError(f"unknown job id {job_id!r}", status_code=404)
return job
def _run(self, job: Job, target: Callable[[Report], Any]) -> None:
with self._lock:
job.status = "running"
job.started_at = datetime.now(UTC).isoformat(timespec="seconds")
def report(progress: int, message: str) -> None:
with self._lock:
job.progress = max(0, min(100, int(progress)))
job.message = str(message)
try:
result = target(report)
except Exception as error: # noqa: BLE001 - represented in the job payload
with self._lock:
job.status = "error"
job.progress = 100
job.message = "failed"
job.error = f"{type(error).__name__}: {error}"
job.finished_at = datetime.now(UTC).isoformat(timespec="seconds")
return
with self._lock:
job.status = "ok"
job.progress = 100
job.message = "complete"
job.result = result
job.finished_at = datetime.now(UTC).isoformat(timespec="seconds")
def _trim(self) -> None:
if len(self._jobs) <= self._keep:
return
ordered = sorted(self._jobs.values(), key=lambda item: item.created_at)
for job in ordered[: len(self._jobs) - self._keep]:
self._jobs.pop(job.job_id, None)
[docs]
@dataclass(slots=True)
class PreparedModel:
"""A workbook-derived spec and its matching generated-model module."""
version: str
spec: Any
model: Any
summary: dict[str, Any]
prepared_at: str
report: dict[str, Any]
[docs]
@dataclass(slots=True)
class PortalConfig:
"""Portal configuration.
Attributes
----------
repo_root:
Repository root containing ``examples/`` and ``tmp/``.
runtime_dir:
Ignored directory for prepared specs and saved runs.
versions:
Workbook versions offered by the portal.
allow_agent:
Enable the optional local assistant.
agent_command:
Command prefix for the assistant CLI.
agent_model:
Optional model identifier passed to the assistant CLI.
agent_timeout:
Assistant timeout in seconds.
title:
Browser title and header label.
"""
repo_root: Path = field(default_factory=Path.cwd)
runtime_dir: Path | None = None
versions: tuple[str, ...] = DEFAULT_VERSIONS
allow_agent: bool = False
agent_command: tuple[str, ...] = ("opencode", "run", "--format", "json", "--auto")
agent_model: str | None = DEFAULT_AGENT_MODEL
agent_timeout: float = 600.0
auth_user: str | None = None
auth_password: str | None = None
history_retention_days: int = 0
history_retention_dry_run: bool = False
history_retention_interval_minutes: int = 0
library_snapshot_interval_minutes: int = 0
library_snapshot_keep: int = 10
title: str = "FABLE Pyculator"
def __post_init__(self) -> None:
self.repo_root = Path(self.repo_root).expanduser().resolve()
self.runtime_dir = (
Path(self.runtime_dir).expanduser().resolve()
if self.runtime_dir is not None
else self.repo_root / "runtime" / "portal"
)
self.versions = tuple(dict.fromkeys(str(version) for version in self.versions))
if bool(self.auth_user) != bool(self.auth_password):
raise ValueError("auth_user and auth_password must be configured together")
if self.history_retention_days < 0 or self.history_retention_days > 3650:
raise ValueError("history_retention_days must be between 0 and 3650")
if self.history_retention_interval_minutes < 0 or self.history_retention_interval_minutes > 10080:
raise ValueError("history_retention_interval_minutes must be between 0 and 10080")
if self.library_snapshot_interval_minutes < 0 or self.library_snapshot_interval_minutes > 10080:
raise ValueError("library_snapshot_interval_minutes must be between 0 and 10080")
if self.library_snapshot_keep < 1 or self.library_snapshot_keep > 100:
raise ValueError("library_snapshot_keep must be between 1 and 100")
[docs]
class PortalState:
"""Shared state for artifact summaries, prepared models, saved runs, and jobs."""
def __init__(self, config: PortalConfig) -> None:
self.config = config
self.config.runtime_dir.mkdir(parents=True, exist_ok=True) # type: ignore[union-attr]
self.jobs = JobManager()
self.chats = ChatStore(self.runtime_dir / "chats")
self._prepared: dict[str, PreparedModel] = {}
self._compute_lock = threading.RLock()
self._preset_lock = threading.Lock()
self._timeline_lock = threading.Lock()
self._retention_lock = threading.Lock()
self._retention_stop = threading.Event()
self._retention_thread: threading.Thread | None = None
self._snapshot_lock = threading.Lock()
self._snapshot_stop = threading.Event()
self._snapshot_thread: threading.Thread | None = None
self._cheat_sheet: CheatSheet | None = None
self._apply_startup_retention()
self._start_retention_scheduler()
self._start_snapshot_scheduler()
self.ensure_cheat_sheet()
@property
def runtime_dir(self) -> Path:
"""Return the resolved runtime directory."""
assert self.config.runtime_dir is not None
return self.config.runtime_dir
[docs]
def options(self) -> dict[str, Any]:
"""Return portal-wide options for the browser shell."""
return {
"title": self.config.title,
"repo_root": str(self.config.repo_root),
"runtime_dir": str(self.runtime_dir),
"versions": list(self.config.versions),
"assistant_enabled": self.config.allow_agent,
"assistant_model": self.config.agent_model,
"docs_url": "https://ubc-fresh.github.io/fable-pyculator/",
}
[docs]
def artifacts(self) -> list[dict[str, Any]]:
"""Return one artifact summary per configured version."""
return [
{
**artifact_summary(self.config.repo_root, version),
"prepared": version in self._prepared,
"summary": self._prepared[version].summary if version in self._prepared else None,
}
for version in self.config.versions
]
[docs]
def artifact(self, version: str) -> dict[str, Any]:
"""Return one artifact summary with its validation state."""
self._require_version(version)
summary = artifact_summary(self.config.repo_root, version)
return {
**summary,
"prepared": version in self._prepared,
"summary": self._prepared[version].summary if version in self._prepared else None,
"validation": self.validation(version),
}
[docs]
def validation(self, version: str) -> dict[str, Any]:
"""Return compact validation evidence without loading the large stage artifacts."""
self._require_version(version)
paths = artifact_paths(self.config.repo_root, version)
files = {
"validation_summary": file_info(paths.validation_summary),
"validation_scenario": file_info(paths.validation_scenario),
"evaluation_report": file_info(paths.evaluation_report),
"contract": file_info(paths.contract),
}
if paths.validation_summary.exists():
try:
payload = json.loads(paths.validation_summary.read_text(encoding="utf-8"))
except (OSError, json.JSONDecodeError) as error:
return {
"version": version,
"available": False,
"files": files,
"message": f"validation summary could not be read: {error}",
}
return {
"version": version,
"available": True,
"files": files,
"summary": payload,
"message": "compact validation evidence loaded",
}
if paths.validation_scenario.exists():
try:
scenario = json.loads(paths.validation_scenario.read_text(encoding="utf-8"))
except (OSError, json.JSONDecodeError):
scenario = {}
return {
"version": version,
"available": False,
"files": files,
"scenario": {
"scenario_id": scenario.get("scenario_id"),
"description": scenario.get("description"),
"source_workbook": scenario.get("source_workbook"),
"generated_model": scenario.get("generated_model"),
"output_count": len(scenario.get("outputs", [])),
}
if scenario
else None,
"message": (
"A validation scenario exists, but the compact validation-evidence summary is missing. "
"Run the validation-evidence packaging workflow to populate it."
),
}
return {
"version": version,
"available": False,
"files": files,
"message": "No validation evidence is available for this version.",
}
[docs]
def status(self, version: str) -> dict[str, Any]:
"""Return in-memory preparation status for one version."""
self._require_version(version)
prepared = self._prepared.get(version)
return {
"version": version,
"prepared": prepared is not None,
"summary": prepared.summary if prepared is not None else None,
"artifact": artifact_summary(self.config.repo_root, version),
}
[docs]
def prepare_job(self, version: str) -> Job:
"""Start a background preparation job for one version."""
self._require_version(version)
return self.jobs.start("prepare", lambda report: self.prepare(version, report))
[docs]
def prepare(self, version: str, report: Report) -> dict[str, Any]:
"""Materialize, discover, load, and cache one version.
The operation is guarded so two analyst tabs do not decompress or build the same model at the
same time. A valid cache avoids rebuilding the expensive workbook spec after a restart.
"""
with self._compute_lock:
prepared = self._prepared.get(version)
if prepared is not None:
report(100, "model already prepared")
return prepared.report
paths = artifact_paths(self.config.repo_root, version)
if not paths.workbook.exists():
raise PortalError(
f"source workbook is missing: {paths.workbook}. "
"Restore it under the ignored tmp/private-workbooks/ path.",
status_code=409,
)
if not paths.generated_model.exists():
if not paths.compressed_model.exists():
raise PortalError(
f"generated model is missing: {paths.generated_model}",
status_code=409,
)
report(5, "decompressing approved generated-model artifact")
materialize_generated_model(self.config.repo_root, version)
checksum = model_checksum_status(self.config.repo_root, version)
if checksum["state"] == "mismatch":
if not paths.compressed_model.exists():
raise PortalError(
"working generated model does not match the approved checksum and no "
f"compressed archive is available: {paths.generated_model}",
status_code=409,
)
report(6, "working model checksum mismatch; re-materializing approved archive")
materialize_generated_model(self.config.repo_root, version, force=True)
checksum = model_checksum_status(self.config.repo_root, version)
if checksum["state"] not in {"verified", "unknown"}:
raise PortalError(
"approved generated model failed checksum verification after materialization",
status_code=409,
)
fingerprint = _fingerprint(paths.workbook, paths.generated_model)
cache = self._read_spec_cache(version, fingerprint)
if cache is None:
report(12, "discovering workbook controls and output tables")
started = time.perf_counter()
spec = build_notebook_spec(paths.workbook, workbook_id=version)
summary = spec_summary(spec)
duration = round(time.perf_counter() - started, 2)
report(72, f"workbook spec ready in {duration:g} s")
self._write_spec_cache(version, fingerprint, spec, summary)
else:
spec, summary = cache
report(55, "loaded cached workbook spec")
report(78, "loading generated model")
started = time.perf_counter()
model = load_generated_model(
paths.generated_model,
module_name=f"fable_pyculator_web_generated_fable_{version}",
)
load_seconds = round(time.perf_counter() - started, 2)
report(96, f"generated model loaded in {load_seconds:g} s")
prepared_at = datetime.now(UTC).isoformat(timespec="seconds")
payload = {
"version": version,
"prepared": True,
"prepared_at": prepared_at,
"model_load_seconds": load_seconds,
"artifact": artifact_summary(self.config.repo_root, version),
"summary": summary,
}
self._prepared[version] = PreparedModel(
version=version,
spec=spec,
model=model,
summary=summary,
prepared_at=prepared_at,
report=payload,
)
self._refresh_cheat_sheet()
report(100, "model prepared")
return payload
[docs]
def spec(self, version: str) -> dict[str, Any]:
"""Return the compact prepared spec, or a 409 when preparation is not done."""
self._require_version(version)
prepared = self._prepared.get(version)
if prepared is None:
raise PortalError(
f"model {version} is not prepared; start POST /api/models/{version}/prepare",
status_code=409,
)
return {
"version": version,
"prepared_at": prepared.prepared_at,
"summary": prepared.summary,
}
[docs]
def run_job(
self,
*,
version: str,
name: str,
selections: Mapping[str, object],
output_table_names: Sequence[str] | None = None,
headline_series_names: Sequence[str] | None = None,
column_flavour_tags: str | Sequence[str] | None = None,
include_context_columns: bool = True,
horizon: int = NATIVE_LAST_YEAR,
) -> Job:
"""Start a background scenario run."""
self._require_version(version)
return self.jobs.start(
"run",
lambda report: self.run_scenario(
version=version,
name=name,
selections=selections,
output_table_names=output_table_names,
headline_series_names=headline_series_names,
column_flavour_tags=column_flavour_tags,
include_context_columns=include_context_columns,
horizon=horizon,
report=report,
),
)
[docs]
def run_scenario(
self,
*,
version: str,
name: str,
selections: Mapping[str, object],
output_table_names: Sequence[str] | None = None,
headline_series_names: Sequence[str] | None = None,
column_flavour_tags: str | Sequence[str] | None = None,
include_context_columns: bool = True,
horizon: int = NATIVE_LAST_YEAR,
report: Report | None = None,
) -> dict[str, Any]:
"""Run one scenario and persist its summary and table payloads."""
if horizon not in {NATIVE_LAST_YEAR, 2070}:
raise PortalError("horizon must be 2050 or 2070")
report = report or (lambda _progress, _message: None)
with self._compute_lock:
prepared = self._prepared.get(version)
if prepared is None:
report(5, "model is not prepared; preparing it first")
self.prepare(version, lambda progress, message: report(5 + int(progress * 0.45), message))
prepared = self._prepared[version]
scenario_name = (name or "scenario").strip() or "scenario"
report(55, "running generated model")
started = time.perf_counter()
result = run_notebook_loop(
prepared.model,
prepared.spec,
dict(selections),
scenario_name=scenario_name,
output_table_names=output_table_names,
output_table_column_flavour_tags=column_flavour_tags,
include_context_columns=include_context_columns,
headline_series_names=headline_series_names,
include_figures=False,
)
duration = round(time.perf_counter() - started, 2)
report(85, f"generated model run complete in {duration:g} s")
run_id = _run_id(scenario_name)
run_dir = self.runtime_dir / "runs" / run_id
table_dir = run_dir / "tables"
table_dir.mkdir(parents=True, exist_ok=True)
table_summaries: list[dict[str, Any]] = []
table_files: dict[str, str] = {}
table_frames: dict[str, pd.DataFrame] = {}
for table_name, frame in result.output_tables.items():
frame = _repair_known_output_table(version, table_name, frame)
if horizon == 2070:
frame = _extend_frame_to_2070(frame)
table_frames[table_name] = frame
payload = _frame_payload(frame, name=table_name)
slug = _slug(table_name)
relative = Path("tables") / f"{slug}.json"
(run_dir / relative).write_text(
json.dumps(payload, ensure_ascii=False) + "\n",
encoding="utf-8",
)
table_files[table_name] = str(relative)
table_summaries.append(
{
"name": table_name,
"label": payload.get("label"),
"sheet": payload.get("sheet"),
"range_ref": payload.get("range_ref"),
"row_count": len(payload["rows"]),
"column_count": len(payload["columns"]),
"file": str(relative),
"display_repair": payload.get("display_repair"),
"extension_note": payload.get("extension_note"),
}
)
headline_frames = {
series_name: (_extend_frame_to_2070(frame) if horizon == 2070 else frame)
for series_name, frame in result.headline_frames.items()
}
headline_payloads = {
series_name: _frame_payload(frame, name=series_name) for series_name, frame in headline_frames.items()
}
headline_summary = _headline_summary(prepared.spec, headline_frames)
quality_warnings = _quality_warnings(
version=version,
spec=prepared.spec,
horizon=horizon,
headline_frames=headline_frames,
table_frames=table_frames,
table_summaries=table_summaries,
)
paths = artifact_paths(self.config.repo_root, version)
provenance = {
"portal_version": __version__,
"repo_root": str(self.config.repo_root),
"prepared_at": prepared.prepared_at,
"workbook_id": prepared.spec.workbook_id,
"workbook": file_info(paths.workbook),
"generated_model": file_info(paths.generated_model),
"spec_provenance": prepared.spec.provenance,
"run_started_at": datetime.now(UTC).isoformat(timespec="seconds"),
}
record = {
"run_id": run_id,
"version": version,
"name": scenario_name,
"created_at": datetime.now(UTC).isoformat(timespec="seconds"),
"status": "ok",
"duration_seconds": duration,
"horizon": horizon,
"extension": (
{
"enabled": horizon == 2070,
"method": "linear_trend_2035_2050",
"native_last_year": NATIVE_LAST_YEAR,
"note": EXTENSION_NOTE,
}
if horizon == 2070
else None
),
"quality_warnings": quality_warnings,
"selections": jsonable(dict(selections)),
"input_count": len(result.run.inputs),
"inputs": jsonable(result.run.inputs),
"headline_summary": headline_summary,
"headline_series": headline_payloads,
"table_summaries": sorted(table_summaries, key=lambda item: item["name"]),
"table_files": table_files,
"skipped_output_tables": jsonable(result.skipped_output_tables),
"provenance": provenance,
"spec": {
"workbook_id": prepared.spec.workbook_id,
"provenance": prepared.spec.provenance,
},
}
(run_dir / "run.json").write_text(
json.dumps(record, indent=2, ensure_ascii=False) + "\n",
encoding="utf-8",
)
self._refresh_cheat_sheet()
report(100, "scenario saved")
return record
[docs]
def bundles(self) -> list[dict[str, Any]]:
"""Return tracked scenario-bundle examples."""
return bundle_summaries(self.config.repo_root)
[docs]
def optimization_frontiers(self) -> list[dict[str, Any]]:
"""Return summaries of tracked exact frontier artifacts."""
directory = self.config.repo_root / "examples" / "optimization"
if not directory.is_dir():
return []
records: list[dict[str, Any]] = []
for path in sorted(directory.glob("*/frontier.json")):
try:
payload = json.loads(path.read_text(encoding="utf-8"))
except (OSError, json.JSONDecodeError):
continue
if not isinstance(payload, dict):
continue
points = payload.get("points") or []
origins: dict[str, int] = {}
for point in points if isinstance(points, list) else []:
if isinstance(point, dict):
origin = str(point.get("origin") or "Other")
origins[origin] = origins.get(origin, 0) + 1
records.append(
{
"version": str(payload.get("version") or path.parent.name.removeprefix("fable_")),
"title": str(payload.get("title") or "Exact decision frontier"),
"point_count": len(points) if isinstance(points, list) else 0,
"pareto_count": len(payload.get("pareto_ids") or []),
"generated_at": payload.get("generated_at"),
"boundary": payload.get("boundary"),
"origins": origins,
}
)
return sorted(records, key=lambda item: item["version"])
[docs]
def optimization_frontier(self, version: str) -> dict[str, Any]:
"""Return one tracked exact frontier artifact for the browser."""
directory = self.config.repo_root / "examples" / "optimization"
for path in sorted(directory.glob("*/frontier.json")) if directory.is_dir() else []:
try:
payload = json.loads(path.read_text(encoding="utf-8"))
except (OSError, json.JSONDecodeError):
continue
if not isinstance(payload, dict):
continue
if str(payload.get("version")) == str(version):
return payload
raise PortalError(f"unknown optimization frontier {version!r}", status_code=404)
[docs]
def optimization_presets(self) -> list[dict[str, Any]]:
"""Return the shared Optimization preset library."""
with self._preset_lock:
return self._load_shared_presets()
[docs]
def save_optimization_preset(
self,
name: str,
state: Mapping[str, Any],
saved_at: str | None = None,
*,
author: str | None = None,
note: str | None = None,
review_status: str | None = None,
) -> dict[str, Any]:
"""Create or replace one shared Optimization preset."""
preset = _clean_optimization_preset(
name,
state,
saved_at,
author=author,
note=note,
review_status=review_status,
)
with self._preset_lock:
existing = next(
(item for item in self._load_shared_presets() if item["name"] == preset["name"]),
None,
)
history = list(existing.get("history", [])) if existing else []
if not existing:
action = "created"
elif existing.get("review_status") != "reviewed" and preset["review_status"] == "reviewed":
action = "reviewed"
else:
action = "updated"
history.append(
{
"at": preset["saved_at"],
"action": action,
"author": preset["author"],
"note": preset["note"][:80],
"review_status": preset["review_status"],
}
)
preset["history"] = history[-10:]
presets = [item for item in self._load_shared_presets() if item["name"] != preset["name"]]
presets.insert(0, preset)
self._write_shared_presets(presets[:20])
return preset
[docs]
def delete_optimization_preset(self, name: str) -> dict[str, Any]:
"""Delete one shared Optimization preset by name."""
with self._preset_lock:
presets = self._load_shared_presets()
remaining = [item for item in presets if item["name"] != name]
if len(remaining) == len(presets):
raise PortalError(f"unknown optimization preset {name!r}", status_code=404)
self._write_shared_presets(remaining)
return {"deleted": name, "count": len(remaining)}
[docs]
def timeline_views(self) -> list[dict[str, Any]]:
"""Return the shared timeline filter library."""
with self._timeline_lock:
return self._load_shared_timeline_views()
[docs]
def optimization_library(self) -> dict[str, Any]:
"""Return a portable backup of shared presets and timeline views."""
return {
"schema": "fable-pyculator.optimization-library",
"schema_version": 1,
"exported_at": datetime.now(UTC).isoformat(timespec="seconds"),
"presets": self.optimization_presets(),
"timeline_views": self.timeline_views(),
}
[docs]
def restore_optimization_library(
self,
presets: Any,
timeline_views: Any,
*,
mode: str = "merge",
dry_run: bool = True,
) -> dict[str, Any]:
"""Preview or apply a shared-library backup in merge or replace mode."""
if mode not in {"merge", "replace"}:
raise ValueError("restore mode must be merge or replace")
if not isinstance(presets, (list, tuple)) or not isinstance(timeline_views, (list, tuple)):
raise ValueError("restore payload needs presets and timeline_views lists")
if len(presets) > 20 or len(timeline_views) > 20:
raise ValueError("a restore can contain at most 20 presets and 20 timeline views")
incoming_presets: list[dict[str, Any]] = []
seen_presets: set[str] = set()
for index, item in enumerate(presets, start=1):
try:
preset = _clean_restored_preset(item)
except ValueError as error:
raise ValueError(f"preset {index}: {error}") from error
if preset["name"] in seen_presets:
raise ValueError(f"duplicate preset name {preset['name']!r}")
seen_presets.add(preset["name"])
incoming_presets.append(preset)
incoming_views: list[dict[str, Any]] = []
seen_views: set[str] = set()
for index, item in enumerate(timeline_views, start=1):
if not isinstance(item, Mapping):
raise ValueError(f"timeline view {index} must be an object")
try:
view = _clean_timeline_view(
item.get("name"),
item.get("filter"),
item.get("saved_at"),
author=item.get("author"),
note=item.get("note"),
review_status=item.get("review_status"),
)
except ValueError as error:
raise ValueError(f"timeline view {index}: {error}") from error
if view["name"] in seen_views:
raise ValueError(f"duplicate timeline view name {view['name']!r}")
seen_views.add(view["name"])
incoming_views.append(view)
with self._preset_lock, self._timeline_lock:
existing_presets = self._load_shared_presets()
existing_views = self._load_shared_timeline_views()
existing_preset_names = {preset["name"] for preset in existing_presets}
existing_view_names = {view["name"] for view in existing_views}
incoming_preset_names = {preset["name"] for preset in incoming_presets}
incoming_view_names = {view["name"] for view in incoming_views}
preset_added = [name for name in incoming_preset_names if name not in existing_preset_names]
preset_replaced = [name for name in incoming_preset_names if name in existing_preset_names]
view_added = [name for name in incoming_view_names if name not in existing_view_names]
view_replaced = [name for name in incoming_view_names if name in existing_view_names]
if mode == "merge":
if len(existing_preset_names | incoming_preset_names) > 20:
raise ValueError("merge would exceed 20 shared presets; use replace or reduce the bundle")
if len(existing_view_names | incoming_view_names) > 20:
raise ValueError("merge would exceed 20 shared timeline views; use replace or reduce the bundle")
result_presets: list[dict[str, Any]] = []
existing_by_name = {preset["name"]: preset for preset in existing_presets}
for preset in incoming_presets:
old = existing_by_name.get(preset["name"])
if old is not None:
preset = dict(preset)
preset["history"] = _merge_preset_history(old.get("history"), preset.get("history"))
result_presets.append(preset)
result_presets.extend(
preset for preset in existing_presets if preset["name"] not in incoming_preset_names
)
result_views: list[dict[str, Any]] = list(incoming_views)
result_views.extend(view for view in existing_views if view["name"] not in incoming_view_names)
preset_removed: list[str] = []
view_removed: list[str] = []
else:
result_presets = list(incoming_presets)
result_views = list(incoming_views)
preset_removed = [name for name in existing_preset_names if name not in incoming_preset_names]
view_removed = [name for name in existing_view_names if name not in incoming_view_names]
changed = bool(
preset_added or preset_replaced or preset_removed or view_added or view_replaced or view_removed
)
if not dry_run and changed:
self._write_shared_presets(result_presets[:20])
self._write_shared_timeline_views(result_views[:20])
return {
"schema": "fable-pyculator.optimization-library",
"schema_version": 1,
"mode": mode,
"dry_run": bool(dry_run),
"changed": changed,
"at": datetime.now(UTC).isoformat(timespec="seconds"),
"presets": {
"incoming": len(incoming_presets),
"added": len(preset_added),
"replaced": len(preset_replaced),
"removed": len(preset_removed),
"result": len(result_presets),
"added_names": sorted(preset_added),
"replaced_names": sorted(preset_replaced),
"removed_names": sorted(preset_removed),
},
"timeline_views": {
"incoming": len(incoming_views),
"added": len(view_added),
"replaced": len(view_replaced),
"removed": len(view_removed),
"result": len(result_views),
"added_names": sorted(view_added),
"replaced_names": sorted(view_replaced),
"removed_names": sorted(view_removed),
},
}
def _snapshot_dir(self) -> Path:
path = self.runtime_dir / "library-snapshots"
path.mkdir(parents=True, exist_ok=True)
return path
@staticmethod
def _snapshot_name(value: datetime) -> str:
return f"library-{value.strftime('%Y%m%dT%H%M%S%fZ')}.json"
@staticmethod
def _snapshot_path_name(name: Any) -> str:
if not isinstance(name, str) or not re.fullmatch(r"library-\d{8}T\d{12}Z\.json", name):
raise ValueError("invalid library snapshot name")
return name
@staticmethod
def _snapshot_summary(path: Path, payload: Mapping[str, Any]) -> dict[str, Any]:
presets = payload.get("presets")
views = payload.get("timeline_views")
return {
"name": path.name,
"at": str(payload.get("exported_at", "")),
"source": str(payload.get("snapshot_source", "manual")),
"presets": len(presets) if isinstance(presets, list) else 0,
"timeline_views": len(views) if isinstance(views, list) else 0,
"bytes": path.stat().st_size,
}
[docs]
def create_library_snapshot(self, source: str = "manual") -> dict[str, Any]:
"""Write one immutable library snapshot and retain the newest configured count."""
if source not in {"manual", "scheduled"}:
raise ValueError("snapshot source must be manual or scheduled")
with self._snapshot_lock:
at = datetime.now(UTC)
path = self._snapshot_dir() / self._snapshot_name(at)
while path.exists():
at = at + timedelta(microseconds=1)
path = self._snapshot_dir() / self._snapshot_name(at)
bundle = self.optimization_library()
bundle["snapshot_source"] = source
temporary = path.with_name(f".{path.name}.{uuid.uuid4().hex}.tmp")
temporary.write_text(json.dumps(bundle, indent=2, sort_keys=True) + "\n", encoding="utf-8")
temporary.replace(path)
pruned = 0
snapshots = sorted(self._snapshot_dir().glob("library-*.json"), key=lambda item: item.name, reverse=True)
for old in snapshots[self.config.library_snapshot_keep :]:
try:
old.unlink()
pruned += 1
except OSError:
continue
summary = self._snapshot_summary(path, bundle)
summary["pruned"] = pruned
return summary
[docs]
def library_snapshots(self, limit: int = 20) -> list[dict[str, Any]]:
"""Return recent snapshot metadata, newest first."""
bounded = max(1, min(100, int(limit)))
directory = self._snapshot_dir()
summaries: list[dict[str, Any]] = []
for path in sorted(directory.glob("library-*.json"), key=lambda item: item.name, reverse=True):
if not re.fullmatch(r"library-\d{8}T\d{12}Z\.json", path.name):
continue
try:
payload = json.loads(path.read_text(encoding="utf-8"))
except (OSError, json.JSONDecodeError):
continue
if isinstance(payload, Mapping):
summaries.append(self._snapshot_summary(path, payload))
if len(summaries) >= bounded:
break
return summaries
[docs]
def library_snapshot_policy(self) -> dict[str, Any]:
"""Return snapshot scheduling and retention policy metadata."""
return {
"interval_minutes": self.config.library_snapshot_interval_minutes,
"keep": self.config.library_snapshot_keep,
"scheduled": self.config.library_snapshot_interval_minutes > 0,
"count": len(self.library_snapshots(limit=100)),
"directory": str(self._snapshot_dir()),
}
[docs]
def library_snapshot(self, name: str) -> dict[str, Any]:
"""Return one stored snapshot bundle."""
clean_name = self._snapshot_path_name(name)
path = self._snapshot_dir() / clean_name
if not path.is_file():
raise PortalError(f"unknown library snapshot {clean_name!r}", status_code=404)
try:
payload = json.loads(path.read_text(encoding="utf-8"))
except (OSError, json.JSONDecodeError) as error:
raise PortalError(f"library snapshot {clean_name!r} could not be read: {error}", status_code=500) from error
if not isinstance(payload, Mapping):
raise PortalError(f"library snapshot {clean_name!r} is invalid", status_code=500)
return dict(payload)
[docs]
def delete_library_snapshot(self, name: str) -> dict[str, Any]:
"""Delete one stored snapshot."""
clean_name = self._snapshot_path_name(name)
path = self._snapshot_dir() / clean_name
with self._snapshot_lock:
if not path.is_file():
raise PortalError(f"unknown library snapshot {clean_name!r}", status_code=404)
path.unlink()
return {"deleted": clean_name, "count": len(self.library_snapshots(limit=100))}
[docs]
def restore_library_snapshot(
self,
name: str,
*,
mode: str = "merge",
dry_run: bool = True,
) -> dict[str, Any]:
"""Preview or apply a restore directly from a stored snapshot."""
payload = self.library_snapshot(name)
result = self.restore_optimization_library(
payload.get("presets", []),
payload.get("timeline_views", []),
mode=mode,
dry_run=dry_run,
)
result["snapshot"] = name
return result
[docs]
def diff_library_snapshot(self, name: str, mode: str = "merge") -> dict[str, Any]:
"""Return an exact read-only diff between one snapshot and the current library."""
if mode not in {"merge", "replace"}:
raise ValueError("diff mode must be merge or replace")
snapshot = self.library_snapshot(name)
current = self.optimization_library()
snapshot_presets = {
str(item["name"]): item
for item in snapshot.get("presets", [])
if isinstance(item, Mapping) and isinstance(item.get("name"), str)
}
current_presets = {
str(item["name"]): item
for item in current.get("presets", [])
if isinstance(item, Mapping) and isinstance(item.get("name"), str)
}
added = sorted(set(snapshot_presets) - set(current_presets))
replaced = sorted(set(snapshot_presets) & set(current_presets))
current_only = sorted(set(current_presets) - set(snapshot_presets))
changed: list[dict[str, Any]] = []
for preset_name in replaced:
old = current_presets[preset_name]
new = snapshot_presets[preset_name]
metadata = [
field for field in ("author", "note", "review_status", "saved_at") if old.get(field) != new.get(field)
]
old_state = old.get("state") if isinstance(old.get("state"), Mapping) else {}
new_state = new.get("state") if isinstance(new.get("state"), Mapping) else {}
state = sorted(key for key in set(old_state) | set(new_state) if old_state.get(key) != new_state.get(key))
old_history = old.get("history") if isinstance(old.get("history"), list) else []
new_history = new.get("history") if isinstance(new.get("history"), list) else []
old_keys = {_preset_event_key(event) for event in old_history if isinstance(event, Mapping)}
new_keys = {_preset_event_key(event) for event in new_history if isinstance(event, Mapping)}
history_added = [
event
for event in new_history
if isinstance(event, Mapping) and _preset_event_key(event) not in old_keys
]
history_removed = [
event
for event in old_history
if isinstance(event, Mapping) and _preset_event_key(event) not in new_keys
]
if metadata or state or history_added or history_removed:
changed.append(
{
"name": preset_name,
"metadata": metadata,
"state": state,
"history_added": history_added,
"history_removed": history_removed,
}
)
snapshot_views = {
str(item["name"]): item
for item in snapshot.get("timeline_views", [])
if isinstance(item, Mapping) and isinstance(item.get("name"), str)
}
current_views = {
str(item["name"]): item
for item in current.get("timeline_views", [])
if isinstance(item, Mapping) and isinstance(item.get("name"), str)
}
view_added = sorted(set(snapshot_views) - set(current_views))
view_replaced = sorted(set(snapshot_views) & set(current_views))
view_current_only = sorted(set(current_views) - set(snapshot_views))
view_changed: list[dict[str, Any]] = []
for view_name in view_replaced:
old = current_views[view_name]
new = snapshot_views[view_name]
metadata = [
field for field in ("author", "note", "review_status", "saved_at") if old.get(field) != new.get(field)
]
old_filter = old.get("filter") if isinstance(old.get("filter"), Mapping) else {}
new_filter = new.get("filter") if isinstance(new.get("filter"), Mapping) else {}
filter_changed = sorted(
key for key in set(old_filter) | set(new_filter) if old_filter.get(key) != new_filter.get(key)
)
if metadata or filter_changed:
view_changed.append({"name": view_name, "metadata": metadata, "filter": filter_changed})
return {
"snapshot": name,
"mode": mode,
"at": datetime.now(UTC).isoformat(timespec="seconds"),
"presets": {
"added": added,
"replaced": replaced,
"removed": current_only if mode == "replace" else [],
"kept": current_only if mode == "merge" else [],
"changed": changed,
"unchanged": len(replaced) - len(changed),
},
"timeline_views": {
"added": view_added,
"replaced": view_replaced,
"removed": view_current_only if mode == "replace" else [],
"kept": view_current_only if mode == "merge" else [],
"changed": view_changed,
"unchanged": len(view_replaced) - len(view_changed),
},
}
[docs]
def save_timeline_view(
self,
name: str,
timeline_filter: Mapping[str, Any],
saved_at: str | None = None,
*,
author: str | None = None,
note: str | None = None,
review_status: str | None = None,
) -> dict[str, Any]:
"""Create or replace one shared timeline filter view."""
view = _clean_timeline_view(
name,
timeline_filter,
saved_at,
author=author,
note=note,
review_status=review_status,
)
with self._timeline_lock:
views = [item for item in self._load_shared_timeline_views() if item["name"] != view["name"]]
views.insert(0, view)
self._write_shared_timeline_views(views[:20])
return view
[docs]
def delete_timeline_view(self, name: str) -> dict[str, Any]:
"""Delete one shared timeline filter view."""
with self._timeline_lock:
views = self._load_shared_timeline_views()
remaining = [item for item in views if item["name"] != name]
if len(remaining) == len(views):
raise PortalError(f"unknown timeline view {name!r}", status_code=404)
self._write_shared_timeline_views(remaining)
return {"deleted": name, "count": len(remaining)}
def _load_shared_timeline_views(self) -> list[dict[str, Any]]:
path = self.runtime_dir / "timeline-views.json"
if not path.is_file():
return []
try:
payload = json.loads(path.read_text(encoding="utf-8"))
except (OSError, json.JSONDecodeError):
return []
if not isinstance(payload, list):
return []
views: list[dict[str, Any]] = []
for item in payload:
if not isinstance(item, Mapping):
continue
try:
views.append(
_clean_timeline_view(
item.get("name"),
item.get("filter"),
item.get("saved_at"),
author=item.get("author"),
note=item.get("note"),
review_status=item.get("review_status"),
)
)
except ValueError:
continue
return views[:20]
def _write_shared_timeline_views(self, views: list[dict[str, Any]]) -> None:
path = self.runtime_dir / "timeline-views.json"
path.parent.mkdir(parents=True, exist_ok=True)
temporary = path.with_name(f".{path.name}.{uuid.uuid4().hex}.tmp")
temporary.write_text(json.dumps(views, indent=2, sort_keys=True) + "\n", encoding="utf-8")
temporary.replace(path)
[docs]
def prune_optimization_history(
self,
days: int = 90,
dry_run: bool = False,
*,
source: str = "manual",
) -> dict[str, Any]:
"""Remove history events older than the retention window, keeping each latest event."""
if not isinstance(days, int) or days < 1 or days > 3650:
raise ValueError("retention days must be between 1 and 3650")
cutoff = datetime.now(UTC) - timedelta(days=days)
with self._preset_lock:
presets = self._load_shared_presets()
removed = 0
kept = 0
touched = 0
details: list[dict[str, Any]] = []
details_truncated = False
updated: list[dict[str, Any]] = []
for preset in presets:
history = list(preset.get("history", []))
if not history:
updated.append(preset)
continue
retained: list[dict[str, Any]] = []
removed_events: list[dict[str, Any]] = []
for index, event in enumerate(history):
keep = index == len(history) - 1
if not keep:
try:
event_time = datetime.fromisoformat(str(event.get("at", "")))
if event_time.tzinfo is None:
event_time = event_time.replace(tzinfo=UTC)
keep = event_time >= cutoff
except ValueError:
keep = True
if keep:
retained.append(event)
else:
removed += 1
removed_events.append(
{"at": str(event.get("at", ""))[:40], "action": str(event.get("action", ""))[:20]}
)
kept += len(retained)
if len(retained) != len(history):
touched += 1
if len(details) < 20:
details.append(
{
"preset": preset["name"],
"before": [
{"at": str(event.get("at", ""))[:40], "action": str(event.get("action", ""))[:20]}
for event in history[:10]
],
"after": [
{"at": str(event.get("at", ""))[:40], "action": str(event.get("action", ""))[:20]}
for event in retained[:10]
],
"removed": removed_events[:10],
"kept": len(retained),
}
)
else:
details_truncated = True
preset = dict(preset)
preset["history"] = retained
updated.append(preset)
if not dry_run and touched:
self._write_shared_presets(updated)
result = {
"days": days,
"dry_run": bool(dry_run),
"removed": removed,
"kept": kept,
"presets_touched": touched,
"presets_total": len(presets),
"source": source,
"at": datetime.now(UTC).isoformat(timespec="seconds"),
"details": details,
"details_truncated": details_truncated,
}
self._record_retention(result)
return result
[docs]
def retention_status(self) -> dict[str, Any]:
"""Return the startup retention policy and the last recorded result."""
path = self.runtime_dir / "retention-log.jsonl"
last: dict[str, Any] | None = None
if path.is_file():
try:
lines = path.read_text(encoding="utf-8").splitlines()
if lines:
last = json.loads(lines[-1])
except (OSError, json.JSONDecodeError):
last = None
interval = self.config.history_retention_interval_minutes
next_run = None
if interval > 0 and self.config.history_retention_days > 0:
base = datetime.now(UTC)
if last and isinstance(last.get("at"), str):
try:
base = datetime.fromisoformat(last["at"])
if base.tzinfo is None:
base = base.replace(tzinfo=UTC)
except ValueError:
base = datetime.now(UTC)
next_run = (base + timedelta(minutes=interval)).isoformat(timespec="seconds")
return {
"history_retention_days": self.config.history_retention_days,
"history_retention_dry_run": self.config.history_retention_dry_run,
"history_retention_interval_minutes": interval,
"next_run_at": next_run,
"log": str(path),
"last": last,
}
[docs]
def retention_runs(self, limit: int = 20) -> list[dict[str, Any]]:
"""Return recent retention maintenance records, newest first."""
bounded = max(1, min(100, int(limit)))
path = self.runtime_dir / "retention-log.jsonl"
if not path.is_file():
return []
try:
lines = path.read_text(encoding="utf-8").splitlines()
except OSError:
return []
records: list[dict[str, Any]] = []
for line in lines:
if not line.strip():
continue
try:
payload = json.loads(line)
except json.JSONDecodeError:
continue
if isinstance(payload, Mapping):
records.append(dict(payload))
return list(reversed(records[-bounded:]))
def _apply_startup_retention(self) -> None:
days = self.config.history_retention_days
if days <= 0:
return
try:
self.prune_optimization_history(
days,
dry_run=self.config.history_retention_dry_run,
source="startup",
)
except ValueError:
return
def _start_retention_scheduler(self) -> None:
days = self.config.history_retention_days
interval = self.config.history_retention_interval_minutes
if days <= 0 or interval <= 0:
return
def loop() -> None:
while not self._retention_stop.wait(interval * 60):
try:
self.prune_optimization_history(
days,
dry_run=self.config.history_retention_dry_run,
source="scheduled",
)
except Exception: # noqa: BLE001 - maintenance must not kill the portal
continue
thread = threading.Thread(target=loop, name="fable-retention", daemon=True)
self._retention_thread = thread
thread.start()
def _start_snapshot_scheduler(self) -> None:
interval = self.config.library_snapshot_interval_minutes
if interval <= 0:
return
def loop() -> None:
while not self._snapshot_stop.wait(interval * 60):
try:
self.create_library_snapshot(source="scheduled")
except Exception: # noqa: BLE001 - maintenance must not kill the portal
continue
thread = threading.Thread(target=loop, name="fable-snapshots", daemon=True)
self._snapshot_thread = thread
thread.start()
[docs]
def close(self) -> None:
"""Stop optional retention and snapshot maintenance threads."""
self._retention_stop.set()
thread = self._retention_thread
if thread is not None and thread.is_alive():
thread.join(timeout=1.0)
self._snapshot_stop.set()
snapshot_thread = self._snapshot_thread
if snapshot_thread is not None and snapshot_thread.is_alive():
snapshot_thread.join(timeout=1.0)
def _record_retention(self, result: Mapping[str, Any]) -> None:
path = self.runtime_dir / "retention-log.jsonl"
with self._retention_lock:
path.parent.mkdir(parents=True, exist_ok=True)
with path.open("a", encoding="utf-8") as target:
target.write(json.dumps(dict(result), sort_keys=True) + "\n")
def _load_shared_presets(self) -> list[dict[str, Any]]:
path = self.runtime_dir / "optimization-presets.json"
if not path.is_file():
return []
try:
payload = json.loads(path.read_text(encoding="utf-8"))
except (OSError, json.JSONDecodeError):
return []
if not isinstance(payload, list):
return []
presets: list[dict[str, Any]] = []
for item in payload:
if not isinstance(item, Mapping):
continue
try:
preset = _clean_optimization_preset(
item.get("name"),
item.get("state"),
item.get("saved_at"),
author=item.get("author"),
note=item.get("note"),
review_status=item.get("review_status"),
)
preset["history"] = _clean_preset_history(item.get("history"))
presets.append(preset)
except ValueError:
continue
return presets[:20]
def _write_shared_presets(self, presets: list[dict[str, Any]]) -> None:
path = self.runtime_dir / "optimization-presets.json"
path.parent.mkdir(parents=True, exist_ok=True)
temporary = path.with_name(f".{path.name}.{uuid.uuid4().hex}.tmp")
temporary.write_text(json.dumps(presets, indent=2, sort_keys=True) + "\n", encoding="utf-8")
temporary.replace(path)
[docs]
def runs(self) -> list[dict[str, Any]]:
"""Return saved runs, newest first, without loading table files."""
directory = self.runtime_dir / "runs"
records: list[dict[str, Any]] = []
if not directory.is_dir():
return records
for path in directory.glob("*/run.json"):
try:
payload = json.loads(path.read_text(encoding="utf-8"))
except (OSError, json.JSONDecodeError):
continue
record = dict(payload)
record.pop("inputs", None)
record.pop("headline_series", None)
records.append(record)
return sorted(records, key=lambda item: str(item.get("created_at", "")), reverse=True)
[docs]
def run(self, run_id: str) -> dict[str, Any]:
"""Return one saved run."""
path = self._run_path(run_id)
if not path.exists():
raise PortalError(f"unknown run id {run_id!r}", status_code=404)
try:
return json.loads(path.read_text(encoding="utf-8"))
except (OSError, json.JSONDecodeError) as error:
raise PortalError(f"run {run_id!r} could not be read: {error}", status_code=500) from error
[docs]
def table(self, run_id: str, table_name: str) -> dict[str, Any]:
"""Return one saved table payload."""
record = self.run(run_id)
relative = record.get("table_files", {}).get(table_name)
if not relative:
raise PortalError(f"unknown table {table_name!r} for run {run_id!r}", status_code=404)
path = (self._run_path(run_id).parent / str(relative)).resolve()
if not path.is_relative_to(self._run_path(run_id).parent.resolve()):
raise PortalError("table path escapes the run directory", status_code=400)
try:
return json.loads(path.read_text(encoding="utf-8"))
except (OSError, json.JSONDecodeError) as error:
raise PortalError(f"table {table_name!r} could not be read: {error}", status_code=500) from error
[docs]
def compare(self, run_ids: Sequence[str]) -> dict[str, Any]:
"""Compare two or more saved runs by headline series and selected inputs."""
unique = list(dict.fromkeys(str(run_id) for run_id in run_ids))
if len(unique) < 2:
raise PortalError("select at least two runs to compare")
records = [self.run(run_id) for run_id in unique]
series: dict[str, dict[str, Any]] = {}
years: set[Any] = set()
for record in records:
for name, payload in record.get("headline_series", {}).items():
entry = series.setdefault(
name,
{
"name": name,
"label": payload.get("label") or name,
"unit": payload.get("unit"),
"group": payload.get("group"),
"series": {},
},
)
mapping = {
row.get("year"): row.get("value") for row in payload.get("rows", []) if isinstance(row, dict)
}
entry["series"][record["run_id"]] = mapping
years.update(mapping)
for entry in series.values():
entry["years"] = sorted(years, key=_year_key)
entry["values"] = {
record["run_id"]: [entry["series"][record["run_id"]].get(year) for year in entry["years"]]
for record in records
}
entry.pop("series", None)
return {
"runs": [
{
"run_id": record["run_id"],
"name": record.get("name"),
"version": record.get("version"),
"created_at": record.get("created_at"),
"selections": record.get("selections", {}),
"headline_summary": record.get("headline_summary", []),
}
for record in records
],
"series": sorted(series.values(), key=lambda item: item["name"]),
}
[docs]
def assistant_context(self, run_id: str | None) -> dict[str, Any]:
"""Build a compact context object for the optional local assistant."""
cheat_sheet = self._cheat_sheet or self.ensure_cheat_sheet()
context: dict[str, Any] = {
"versions": [artifact["version"] for artifact in self.artifacts()],
"cheat_sheet_path": str(cheat_sheet.path),
"cheat_sheet_generated_at": cheat_sheet.generated_at,
}
if run_id:
try:
record = self.run(run_id)
except PortalError:
return context
context["run"] = {
key: record.get(key)
for key in ("run_id", "name", "version", "created_at", "selections", "headline_summary")
}
return context
[docs]
def ensure_cheat_sheet(self) -> CheatSheet:
"""Regenerate the compiled assistant grounding sheet from current portal state."""
cheat_sheet = ensure_cheat_sheet(
repo_root=self.config.repo_root,
runtime_dir=self.runtime_dir,
artifacts=self.artifacts(),
prepared_summaries={version: prepared.summary for version, prepared in self._prepared.items()},
runs=self.runs(),
)
self._cheat_sheet = cheat_sheet
return cheat_sheet
def _refresh_cheat_sheet(self) -> None:
"""Best-effort refresh after preparation or a saved run."""
try:
self.ensure_cheat_sheet()
except Exception: # noqa: BLE001 - auxiliary cache must not fail model jobs
# The grounding sheet is auxiliary; never fail a model job because its cache write failed.
return
[docs]
def create_chat(self, *, run_id: str | None = None) -> dict[str, Any]:
"""Create a persistent assistant chat."""
return self.chats.create(run_id=run_id).to_dict()
[docs]
def list_chats(
self,
*,
include_archived: bool = False,
query: str | None = None,
) -> list[dict[str, Any]]:
"""Return persistent chat summaries, pinned first and newest first."""
return self.chats.list(include_archived=include_archived, query=query)
[docs]
def update_chat(
self,
chat_id: str,
*,
title: str | None = None,
pinned: bool | None = None,
archived: bool | None = None,
) -> dict[str, Any]:
"""Update chat title, pin, or archive state."""
try:
return self.chats.set_metadata(
chat_id,
title=title,
pinned=pinned,
archived=archived,
).to_dict()
except KeyError as error:
raise PortalError(f"unknown chat id {chat_id!r}", status_code=404) from error
[docs]
def export_chat(self, chat_id: str) -> str:
"""Return a Markdown export of one chat."""
try:
return self.chats.get(chat_id).to_markdown()
except KeyError as error:
raise PortalError(f"unknown chat id {chat_id!r}", status_code=404) from error
[docs]
def get_chat(self, chat_id: str) -> dict[str, Any]:
"""Return one persistent chat."""
try:
return self.chats.get(chat_id).to_dict()
except KeyError as error:
raise PortalError(f"unknown chat id {chat_id!r}", status_code=404) from error
[docs]
def delete_chat(self, chat_id: str) -> dict[str, Any]:
"""Delete one persistent chat."""
try:
deleted = self.chats.delete(chat_id)
except KeyError as error:
raise PortalError(f"invalid chat id {chat_id!r}", status_code=400) from error
if not deleted:
raise PortalError(f"unknown chat id {chat_id!r}", status_code=404)
return {"chat_id": chat_id, "deleted": True}
def _require_version(self, version: str) -> None:
if version not in self.config.versions:
raise PortalError(f"unknown workbook version {version!r}", status_code=404)
def _run_path(self, run_id: str) -> Path:
if not re.fullmatch(r"[A-Za-z0-9._-]+", run_id):
raise PortalError("invalid run id", status_code=400)
return self.runtime_dir / "runs" / run_id / "run.json"
def _cache_path(self, version: str) -> Path:
return self.runtime_dir / "specs" / f"fable-{version}.json"
def _read_spec_cache(
self,
version: str,
fingerprint: dict[str, Any],
) -> tuple[Any, dict[str, Any]] | None:
path = self._cache_path(version)
if not path.exists():
return None
try:
payload = json.loads(path.read_text(encoding="utf-8"))
if payload.get("cache_version") != SPEC_CACHE_VERSION:
return None
if payload.get("fingerprint") != fingerprint:
return None
spec = spec_from_payload(payload["spec"])
summary = payload["summary"]
except (OSError, json.JSONDecodeError, KeyError, TypeError, ValueError):
return None
return spec, summary
def _write_spec_cache(
self,
version: str,
fingerprint: dict[str, Any],
spec: Any,
summary: dict[str, Any],
) -> None:
path = self._cache_path(version)
path.parent.mkdir(parents=True, exist_ok=True)
payload = {
"version": version,
"cache_version": SPEC_CACHE_VERSION,
"fingerprint": fingerprint,
"summary": summary,
"spec": spec_to_payload(spec),
}
temporary = path.with_name(path.name + ".tmp")
temporary.write_text(json.dumps(payload, ensure_ascii=False) + "\n", encoding="utf-8")
temporary.replace(path)
def _fingerprint(workbook: Path, model: Path) -> dict[str, Any]:
def stat(path: Path) -> dict[str, Any]:
info = path.stat()
return {"size_bytes": info.st_size, "modified_ns": info.st_mtime_ns}
return {"workbook": stat(workbook), "model": stat(model)}
def _frame_payload(frame: pd.DataFrame, *, name: str) -> dict[str, Any]:
attrs = frame.attrs
rows = [
{str(column): jsonable(value) for column, value in zip(frame.columns, row, strict=False)}
for row in frame.itertuples(index=False, name=None)
]
return {
"name": name,
"label": attrs.get("label"),
"description": attrs.get("description"),
"sheet": attrs.get("sheet"),
"range_ref": attrs.get("range_ref"),
"columns": [str(column) for column in frame.columns],
"index": [jsonable(value) for value in frame.index],
"rows": rows,
"selected_column_flavour_tags": jsonable(attrs.get("selected_column_flavour_tags")),
"matched_column_flavour_tags": jsonable(attrs.get("matched_column_flavour_tags")),
"column_flavour_tags": jsonable(attrs.get("column_flavour_tags")),
"display_repair": attrs.get("display_repair"),
"extension_note": attrs.get("extension_note"),
}
def _repair_known_output_table(
version: str,
table_name: str,
frame: pd.DataFrame,
) -> pd.DataFrame:
"""Repair known broken display values without changing generated-model semantics.
The 2024 generated model still contains the pre-repair ``LAND!AA`` ``#REF!`` expressions. Until
that artifact is regenerated, the portal replaces only the displayed LAND TOTAL column with the
workbook's ``TotallandFin`` component sum and records the repair in ``DataFrame.attrs``.
"""
if version != "2024" or table_name != "land_resultsland":
return frame
components = ("Cropland", "Pasture", "Urban", "ForPlantations", "OtherForest", "OtherLand")
if "TOTAL" not in frame.columns or not all(component in frame.columns for component in components):
return frame
total = frame["TOTAL"]
if not any(isinstance(value, str) and "#REF!" in value for value in total):
return frame
repaired = frame.copy()
numeric = repaired.loc[:, list(components)].apply(pd.to_numeric, errors="coerce")
repaired["TOTAL"] = numeric.sum(axis=1, min_count=len(components))
repaired.attrs.update(frame.attrs)
repaired.attrs["display_repair"] = (
"The generated model's LAND TOTAL column is #REF!; this table displays the workbook's "
"TotallandFin component sum instead (LoggedForest is represented inside OtherForest and is "
"not added separately)."
)
return repaired
def _quality_warnings(
*,
version: str,
spec: Any,
horizon: int,
headline_frames: Mapping[str, pd.DataFrame],
table_frames: Mapping[str, pd.DataFrame],
table_summaries: Sequence[dict[str, Any]],
) -> list[dict[str, Any]]:
"""Return run-scoped data-quality notes for the Results panel."""
warnings: list[dict[str, Any]] = []
if horizon == 2070:
warnings.append(
{
"code": "trend_extension_2070",
"severity": "warn",
"title": "2070 values are trend extensions",
"message": EXTENSION_NOTE,
"scope": "headline series and extended output tables",
}
)
land = next((series for series in spec.headline_series if series.name == "land_total_area"), None)
if land is not None and (
"component sum" in (land.label or "").casefold() or "totallandfin" in (land.description or "").casefold()
):
warnings.append(
{
"code": "land_component_sum",
"severity": "info",
"title": "Land total uses a component sum",
"message": (
"The workbook's ResultsLand[TOTAL] total is not usable as-is. The land series "
"reproduces calc_land_cor[TotallandFin] from the component columns and records "
"the source caveat in its description."
),
"scope": "land_total_area",
}
)
if version == "2024" and not any(series.name == "water_total_footprint" for series in spec.headline_series):
warnings.append(
{
"code": "water_unavailable_2024",
"severity": "info",
"title": "Water-footprint series is unavailable",
"message": (
"The 2024 Canada workbook has no TotalResultsWF table, so no water-footprint "
"headline series is exposed."
),
"scope": "WATER",
}
)
mismatch = _ghg_cached_mismatch(headline_frames, table_frames, spec=spec)
if mismatch is not None:
years, maximum, max_year = mismatch
warnings.append(
{
"code": "ghg_cached_mismatch",
"severity": "warn",
"title": "GHG components differ from cached workbook totals",
"message": (
f"The derived GHG series differs from cached ResultsGHG[TotalCO2e] at "
f"{len(years)} native year(s); the largest difference is {maximum:.3g} MtCO2e "
f"in {max_year}. Cached values may reflect a different scenario/GWP, or an "
"individual generated-model component may differ."
),
"scope": "ghg_total_co2e vs cached TotalCO2e",
}
)
skipped = [
str(summary.get("label") or summary.get("name"))
for summary in table_summaries
if "not applied" in str(summary.get("extension_note") or "").casefold()
]
if skipped:
warnings.append(
{
"code": "tables_not_extended",
"severity": "info",
"title": "Some tables were not extended to 2070",
"message": (
"These tables have no usable identity/trend grouping and remain at native 2050: "
+ ", ".join(skipped)
),
"scope": "output tables",
}
)
return warnings
def _cached_ghg_from_spec(spec: Any | None) -> dict[int, float]:
if spec is None:
return {}
tables = getattr(spec, "output_tables", ())
table = next((item for item in tables if item.name == "ghg_resultsghg"), None)
if table is None or not table.values:
return {}
labels = [str(label).casefold() for label in table.column_labels]
try:
year_index = labels.index("year")
total_index = labels.index("totalco2e")
except ValueError:
return {}
cached: dict[int, float] = {}
for row in table.values:
if len(row) <= max(year_index, total_index):
continue
year = _coerce_year(row[year_index])
value = row[total_index]
if year is not None and isinstance(value, int | float) and not isinstance(value, bool):
cached[year] = float(value)
return cached
def _ghg_cached_mismatch(
headline_frames: Mapping[str, pd.DataFrame],
table_frames: Mapping[str, pd.DataFrame],
*,
spec: Any | None = None,
) -> tuple[list[int], float, int] | None:
headline = headline_frames.get("ghg_total_co2e")
if headline is None:
return None
cached = _cached_ghg_from_spec(spec)
if not cached:
table = table_frames.get("ghg_resultsghg")
if table is None or table.empty:
return None
year_column = next((column for column in table.columns if str(column).casefold() == "year"), None)
if year_column is None or "TotalCO2e" not in table.columns:
return None
for _, row in table.iterrows():
year = _coerce_year(row.get(year_column))
value = row.get("TotalCO2e")
if year is not None and isinstance(value, int | float) and not isinstance(value, bool):
cached[year] = float(value)
mismatches: list[tuple[int, float]] = []
for _, row in headline.iterrows():
year = _coerce_year(row.get("year"))
value = row.get("value")
if (
year is None
or year > NATIVE_LAST_YEAR
or year not in cached
or not isinstance(value, int | float)
or isinstance(value, bool)
):
continue
difference = abs(float(value) - cached[year])
tolerance = max(1e-6, abs(cached[year]) * 1e-6)
if difference > tolerance:
mismatches.append((year, difference))
if not mismatches:
return None
max_year, maximum = max(mismatches, key=lambda item: item[1])
return [year for year, _ in mismatches], maximum, max_year
def _extend_frame_to_2070(frame: pd.DataFrame) -> pd.DataFrame:
"""Extend a year-indexed or year-column table using the 2035-2050 linear trend."""
if frame.empty:
return frame
copied = frame.copy()
copied.attrs.update(frame.attrs)
year_column = next(
(column for column in copied.columns if str(column).casefold() == "year"),
None,
)
if year_column is not None:
extended = _extend_year_rows(copied, year_column)
if extended is not None:
extended.attrs.update(copied.attrs)
extended.attrs["extension_note"] = EXTENSION_NOTE
return extended
copied.attrs["extension_note"] = (
"This table has multiple rows per year; 2055-2070 trend extension was not applied."
)
return copied
if _extend_year_columns(copied):
copied.attrs["extension_note"] = EXTENSION_NOTE
return copied
return frame
def _extend_year_rows(frame: pd.DataFrame, year_column: Any) -> pd.DataFrame | None:
years = [_coerce_year(value) for value in frame[year_column]]
numeric_years = [year for year in years if year is not None]
if not numeric_years:
return None
native_max = max(numeric_years)
targets = [year for year in EXTENSION_YEARS if year > native_max]
if not targets:
return None
if len(set(numeric_years)) == len(numeric_years):
return _extend_series_rows(frame, year_column, years, native_max, targets)
return _extend_grouped_rows(frame, year_column, native_max, targets)
def _extend_series_rows(
frame: pd.DataFrame,
year_column: Any,
years: list[int | None],
native_max: int,
targets: list[int],
) -> pd.DataFrame:
"""Extend a frame where each row is one year."""
trend_floor = max(2035, native_max - 15)
last_index = frame.index[max(range(len(years)), key=lambda index: years[index] or -1)]
appended: list[dict[Any, Any]] = []
for target in targets:
row: dict[Any, Any] = {}
for column in frame.columns:
if column == year_column:
row[column] = target
continue
if column == "cell_refs":
row[column] = ()
continue
if column == "description":
row[column] = "Trend extension (non-native)"
continue
values = [
(year, value)
for year, value in zip(years, frame[column], strict=False)
if year is not None and trend_floor <= year <= native_max
]
projected = _trend_value(values, target)
row[column] = projected if projected is not None else frame.loc[last_index, column]
appended.append(row)
new = pd.DataFrame(appended, index=targets)
extended = pd.concat([frame, new])
extended.attrs.update(frame.attrs)
return extended
def _extend_grouped_rows(
frame: pd.DataFrame,
year_column: Any,
native_max: int,
targets: list[int],
) -> pd.DataFrame | None:
"""Extend a multi-row-per-year frame by grouping on its text identity columns."""
group_columns = [
column
for column in frame.columns
if column != year_column
and (pd.api.types.is_object_dtype(frame[column]) or pd.api.types.is_string_dtype(frame[column]))
]
if not group_columns:
return None
trend_floor = max(2035, native_max - 15)
appended: list[dict[Any, Any]] = []
extended_groups = 0
for _, group in frame.groupby(group_columns, dropna=False, sort=False):
years = [_coerce_year(value) for value in group[year_column]]
numeric_years = [year for year in years if year is not None]
trend_years = [year for year in numeric_years if trend_floor <= year <= native_max]
if len(trend_years) < 2 or len(set(numeric_years)) != len(numeric_years):
continue
last_index = group.index[max(range(len(years)), key=lambda index: years[index] or -1)]
for target in targets:
row: dict[Any, Any] = {}
for column in frame.columns:
if column == year_column:
row[column] = target
elif column == "cell_refs":
row[column] = ()
elif column == "description":
row[column] = "Trend extension (non-native)"
else:
values = [
(year, value)
for year, value in zip(years, group[column], strict=False)
if year is not None and trend_floor <= year <= native_max
]
projected = _trend_value(values, target)
row[column] = projected if projected is not None else group.loc[last_index, column]
appended.append(row)
extended_groups += 1
if not appended:
return None
new = pd.DataFrame(appended, index=[target for _ in range(extended_groups) for target in targets])
extended = pd.concat([frame, new])
extended.attrs.update(frame.attrs)
return extended
def _extend_year_columns(frame: pd.DataFrame) -> bool:
year_columns: list[tuple[Any, int]] = []
for column in frame.columns:
text = str(column)
if text.isdigit() and 2000 <= int(text) <= 2100:
year_columns.append((column, int(text)))
if len(year_columns) < 2:
return False
native_max = max(year for _, year in year_columns)
targets = [year for year in EXTENSION_YEARS if year > native_max]
if not targets:
return False
trend_floor = max(2035, native_max - 15)
style = type(year_columns[0][0])
for target in targets:
frame[style(target)] = [
_trend_value(
[
(year, row[column])
for column, year in year_columns
if year is not None and trend_floor <= year <= native_max
],
target,
)
for _, row in frame.iterrows()
]
return True
def _trend_value(points: Sequence[tuple[int, Any]], target: int) -> float | None:
usable = [
(float(year), float(value))
for year, value in points
if isinstance(value, int | float) and not isinstance(value, bool)
]
if not usable:
return None
if len(usable) == 1:
return usable[0][1]
count = len(usable)
sum_x = sum(year for year, _ in usable)
sum_y = sum(value for _, value in usable)
sum_xy = sum(year * value for year, value in usable)
sum_xx = sum(year * year for year, _ in usable)
denominator = count * sum_xx - sum_x * sum_x
if denominator == 0:
return usable[-1][1]
slope = (count * sum_xy - sum_x * sum_y) / denominator
intercept = (sum_y - slope * sum_x) / count
return intercept + slope * target
def _coerce_year(value: Any) -> int | None:
if isinstance(value, bool):
return None
if isinstance(value, int | float):
return int(value)
text = str(value).strip()
return int(text) if text.isdigit() else None
def _headline_summary(spec: Any, frames: Mapping[str, pd.DataFrame]) -> list[dict[str, Any]]:
series_by_name = {series.name: series for series in spec.headline_series}
summary: list[dict[str, Any]] = []
for name, frame in frames.items():
if "year" not in frame.columns or "value" not in frame.columns:
continue
points = [
{"year": jsonable(row["year"]), "value": jsonable(row["value"])}
for _, row in frame.iterrows()
if row.get("value") is not None
]
if not points:
continue
first = points[0]
last = points[-1]
first_value = first["value"]
last_value = last["value"]
delta = None
percent_change = None
if isinstance(first_value, int | float) and isinstance(last_value, int | float):
delta = last_value - first_value
if first_value not in (0, None):
percent_change = (delta / abs(first_value)) * 100
series = series_by_name.get(name)
summary.append(
{
"name": name,
"label": series.label if series else name,
"group": series.group if series else None,
"unit": series.unit if series else None,
"first_year": first["year"],
"first_value": first_value,
"last_year": last["year"],
"last_value": last_value,
"delta": jsonable(delta),
"percent_change": jsonable(percent_change),
"point_count": len(points),
}
)
return sorted(summary, key=lambda item: item["name"])
def _run_id(name: str) -> str:
stamp = datetime.now(UTC).strftime("%Y%m%dT%H%M%SZ")
return f"{stamp}-{_slug(name)}-{uuid.uuid4().hex[:4]}"
def _slug(value: str) -> str:
slug = re.sub(r"[^A-Za-z0-9._-]+", "-", str(value)).strip("-").lower()
return slug or "item"
def _year_key(value: Any) -> tuple[int, Any]:
try:
return (0, int(value))
except (TypeError, ValueError):
return (1, str(value))