Source code for fable_pyculator.web.service

"""Runtime state, preparation, and scenario execution for the FABLE web portal.

The service is deliberately a thin presentation layer over the notebook APIs. Model execution and
workbook discovery happen in background jobs; every response is built from a real run or a real
artifact, never from browser-only state.
"""

from __future__ import annotations

import json
import re
import threading
import time
import uuid
from collections.abc import Callable, Mapping, Sequence
from dataclasses import dataclass, field
from datetime import UTC, datetime, timedelta
from pathlib import Path
from typing import Any, Literal

import pandas as pd

from fable_pyculator import __version__
from fable_pyculator.notebook import build_notebook_spec, load_generated_model, run_notebook_loop
from fable_pyculator.web.artifacts import (
    DEFAULT_VERSIONS,
    artifact_paths,
    artifact_summary,
    bundle_summaries,
    file_info,
    materialize_generated_model,
    model_checksum_status,
)
from fable_pyculator.web.assistant import DEFAULT_AGENT_MODEL
from fable_pyculator.web.chats import ChatStore
from fable_pyculator.web.cheatsheet import CheatSheet, ensure_cheat_sheet
from fable_pyculator.web.spec_io import jsonable, spec_from_payload, spec_summary, spec_to_payload

__all__ = [
    "Job",
    "JobManager",
    "PortalConfig",
    "PortalError",
    "PortalState",
    "PreparedModel",
]

JobStatus = Literal["queued", "running", "ok", "error"]
Report = Callable[[int, str], None]
SPEC_CACHE_VERSION = "2026-11-06-ghg-output-components"
NATIVE_LAST_YEAR = 2050
EXTENSION_YEARS = (2055, 2060, 2065, 2070)
EXTENSION_NOTE = (
    "2055-2070 are trend extensions extrapolated from the 2035-2050 linear trend, not native FABLE Calculator results."
)
_PRESET_STATE_KEYS = (
    "version",
    "x",
    "y",
    "z",
    "colour",
    "size",
    "floor_dimension",
    "floor",
    "front",
    "warning_free",
    "brushes",
    "reference",
    "point",
)


def _clean_timeline_view(
    name: Any,
    timeline_filter: Any,
    saved_at: Any = None,
    author: Any = None,
    note: Any = None,
    review_status: Any = None,
) -> dict[str, Any]:
    """Validate and normalize one shared timeline filter view."""

    if not isinstance(name, str) or not name.strip():
        raise ValueError("timeline view name must be a non-empty string")
    if not isinstance(timeline_filter, Mapping):
        raise ValueError("timeline view filter must be an object")
    clean_name = name.strip()[:40]
    if not clean_name:
        raise ValueError("timeline view name must be a non-empty string")
    window = str(timeline_filter.get("window", "all"))
    action = str(timeline_filter.get("action", ""))
    if window not in {"all", "24h", "7d", "30d"}:
        raise ValueError("timeline view window must be all, 24h, 7d, or 30d")
    if action not in {"", "created", "updated", "reviewed"}:
        raise ValueError("timeline view action is invalid")
    timestamp = saved_at if isinstance(saved_at, str) and len(saved_at) <= 40 else None
    if timestamp is None:
        timestamp = datetime.now(UTC).isoformat(timespec="seconds")
    status = str(review_status or "draft").strip().lower()
    if status not in {"draft", "reviewed"}:
        raise ValueError("timeline view review_status must be draft or reviewed")
    return {
        "name": clean_name,
        "saved_at": timestamp,
        "author": author.strip()[:40] if isinstance(author, str) else "",
        "note": note.strip()[:200] if isinstance(note, str) else "",
        "review_status": status,
        "filter": {
            "author": str(timeline_filter.get("author", ""))[:40],
            "action": action,
            "preset": str(timeline_filter.get("preset", ""))[:40],
            "window": window,
        },
    }


def _clean_preset_history(value: Any) -> list[dict[str, Any]]:
    """Validate the server-owned review history list."""

    if not isinstance(value, (list, tuple)):
        return []
    history: list[dict[str, Any]] = []
    for event in value[-10:]:
        if not isinstance(event, Mapping):
            continue
        action = str(event.get("action", "")).strip().lower()
        status = str(event.get("review_status", "draft")).strip().lower()
        if action not in {"created", "updated", "reviewed"} or status not in {"draft", "reviewed"}:
            continue
        history.append(
            {
                "at": str(event.get("at", ""))[:40],
                "action": action,
                "author": str(event.get("author", ""))[:40],
                "note": str(event.get("note", ""))[:80],
                "review_status": status,
            }
        )
    return history[-10:]


def _clean_optimization_preset(
    name: Any,
    state: Any,
    saved_at: Any = None,
    author: Any = None,
    note: Any = None,
    review_status: Any = None,
) -> dict[str, Any]:
    """Validate and normalize one shared Optimization preset."""

    if not isinstance(name, str) or not name.strip():
        raise ValueError("preset name must be a non-empty string")
    if not isinstance(state, Mapping):
        raise ValueError("preset state must be an object")
    clean_name = name.strip()[:40]
    if not clean_name:
        raise ValueError("preset name must be a non-empty string")
    clean_state: dict[str, Any] = {}
    for key in _PRESET_STATE_KEYS:
        if key not in state:
            continue
        value = state[key]
        if key in {"version", "x", "y", "z", "colour", "size", "floor_dimension"}:
            clean_state[key] = str(value)[:60]
        elif key == "floor":
            number = float(value)
            if not 0 <= number <= 100:
                raise ValueError("preset floor must be between 0 and 100")
            clean_state[key] = number
        elif key in {"front", "warning_free"}:
            clean_state[key] = bool(value)
        elif key == "brushes":
            if not isinstance(value, (list, tuple)) or len(value) > 20:
                raise ValueError("preset brushes must be a list of at most 20 entries")
            brushes: list[dict[str, Any]] = []
            for brush in value:
                if not isinstance(brush, Mapping):
                    raise ValueError("each preset brush must be an object")
                axis_key = brush.get("axisKey")
                if not isinstance(axis_key, str) or not axis_key:
                    raise ValueError("each preset brush needs an axisKey")
                start = float(brush.get("start"))
                end = float(brush.get("end"))
                if not 0 <= start <= 1 or not 0 <= end <= 1:
                    raise ValueError("preset brush ranges must be between 0 and 1")
                brushes.append({"axisKey": axis_key[:60], "start": start, "end": end})
            clean_state[key] = brushes
        elif key in {"reference", "point"}:
            clean_state[key] = None if value in (None, "") else str(value)[:20]
    if not clean_state.get("version"):
        raise ValueError("preset state needs a version")
    timestamp = saved_at if isinstance(saved_at, str) and len(saved_at) <= 40 else None
    if timestamp is None:
        timestamp = datetime.now(UTC).isoformat(timespec="seconds")
    clean_author = author.strip()[:40] if isinstance(author, str) else ""
    clean_note = note.strip()[:200] if isinstance(note, str) else ""
    status = str(review_status or "draft").strip().lower()
    if status not in {"draft", "reviewed"}:
        raise ValueError("preset review_status must be draft or reviewed")
    return {
        "name": clean_name,
        "saved_at": timestamp,
        "author": clean_author,
        "note": clean_note,
        "review_status": status,
        "state": clean_state,
    }


def _clean_restored_preset(value: Any) -> dict[str, Any]:
    """Validate one preset from a shared-library backup, including its history."""

    if not isinstance(value, Mapping):
        raise ValueError("each preset must be an object")
    preset = _clean_optimization_preset(
        value.get("name"),
        value.get("state"),
        value.get("saved_at"),
        author=value.get("author"),
        note=value.get("note"),
        review_status=value.get("review_status"),
    )
    preset["history"] = _clean_preset_history(value.get("history"))
    return preset


def _preset_event_key(event: Mapping[str, Any]) -> tuple[str, str, str, str, str]:
    """Return the exact comparison key for one bounded history event."""

    return (
        str(event.get("at", "")),
        str(event.get("action", "")),
        str(event.get("author", "")),
        str(event.get("note", "")),
        str(event.get("review_status", "")),
    )


def _merge_preset_history(
    existing: Any,
    incoming: Any,
) -> list[dict[str, Any]]:
    """Merge two bounded review histories without duplicating events."""

    merged: list[dict[str, Any]] = []
    seen: set[tuple[str, str, str, str, str]] = set()
    for event in [*(_clean_preset_history(existing)), *(_clean_preset_history(incoming))]:
        key = _preset_event_key(event)
        if key in seen:
            continue
        seen.add(key)
        merged.append(event)
    merged.sort(key=lambda event: str(event.get("at", "")))
    return merged[-10:]


[docs] class PortalError(RuntimeError): """A user-facing portal error with an HTTP status hint.""" def __init__(self, message: str, *, status_code: int = 400) -> None: super().__init__(message) self.status_code = status_code
[docs] @dataclass(slots=True) class Job: """One background preparation or scenario-run job.""" job_id: str kind: str status: JobStatus = "queued" progress: int = 0 message: str = "queued" result: Any = None error: str | None = None created_at: str = field(default_factory=lambda: datetime.now(UTC).isoformat(timespec="seconds")) started_at: str | None = None finished_at: str | None = None
[docs] def to_dict(self) -> dict[str, Any]: """Return a JSON-ready snapshot.""" return { "job_id": self.job_id, "kind": self.kind, "status": self.status, "progress": self.progress, "message": self.message, "result": self.result, "error": self.error, "created_at": self.created_at, "started_at": self.started_at, "finished_at": self.finished_at, }
[docs] class JobManager: """Run callables on daemon threads and expose pollable status snapshots.""" def __init__(self, *, keep: int = 50) -> None: self._jobs: dict[str, Job] = {} self._lock = threading.Lock() self._keep = keep
[docs] def start(self, kind: str, target: Callable[[Report], Any]) -> Job: """Start a job and return its initial snapshot.""" job = Job(job_id=uuid.uuid4().hex[:12], kind=kind) with self._lock: self._jobs[job.job_id] = job self._trim() thread = threading.Thread(target=self._run, args=(job, target), name=f"portal-{job.job_id}", daemon=True) thread.start() return job
[docs] def get(self, job_id: str) -> Job: """Return one job or raise a 404 portal error.""" with self._lock: job = self._jobs.get(job_id) if job is None: raise PortalError(f"unknown job id {job_id!r}", status_code=404) return job
def _run(self, job: Job, target: Callable[[Report], Any]) -> None: with self._lock: job.status = "running" job.started_at = datetime.now(UTC).isoformat(timespec="seconds") def report(progress: int, message: str) -> None: with self._lock: job.progress = max(0, min(100, int(progress))) job.message = str(message) try: result = target(report) except Exception as error: # noqa: BLE001 - represented in the job payload with self._lock: job.status = "error" job.progress = 100 job.message = "failed" job.error = f"{type(error).__name__}: {error}" job.finished_at = datetime.now(UTC).isoformat(timespec="seconds") return with self._lock: job.status = "ok" job.progress = 100 job.message = "complete" job.result = result job.finished_at = datetime.now(UTC).isoformat(timespec="seconds") def _trim(self) -> None: if len(self._jobs) <= self._keep: return ordered = sorted(self._jobs.values(), key=lambda item: item.created_at) for job in ordered[: len(self._jobs) - self._keep]: self._jobs.pop(job.job_id, None)
[docs] @dataclass(slots=True) class PreparedModel: """A workbook-derived spec and its matching generated-model module.""" version: str spec: Any model: Any summary: dict[str, Any] prepared_at: str report: dict[str, Any]
[docs] @dataclass(slots=True) class PortalConfig: """Portal configuration. Attributes ---------- repo_root: Repository root containing ``examples/`` and ``tmp/``. runtime_dir: Ignored directory for prepared specs and saved runs. versions: Workbook versions offered by the portal. allow_agent: Enable the optional local assistant. agent_command: Command prefix for the assistant CLI. agent_model: Optional model identifier passed to the assistant CLI. agent_timeout: Assistant timeout in seconds. title: Browser title and header label. """ repo_root: Path = field(default_factory=Path.cwd) runtime_dir: Path | None = None versions: tuple[str, ...] = DEFAULT_VERSIONS allow_agent: bool = False agent_command: tuple[str, ...] = ("opencode", "run", "--format", "json", "--auto") agent_model: str | None = DEFAULT_AGENT_MODEL agent_timeout: float = 600.0 auth_user: str | None = None auth_password: str | None = None history_retention_days: int = 0 history_retention_dry_run: bool = False history_retention_interval_minutes: int = 0 library_snapshot_interval_minutes: int = 0 library_snapshot_keep: int = 10 title: str = "FABLE Pyculator" def __post_init__(self) -> None: self.repo_root = Path(self.repo_root).expanduser().resolve() self.runtime_dir = ( Path(self.runtime_dir).expanduser().resolve() if self.runtime_dir is not None else self.repo_root / "runtime" / "portal" ) self.versions = tuple(dict.fromkeys(str(version) for version in self.versions)) if bool(self.auth_user) != bool(self.auth_password): raise ValueError("auth_user and auth_password must be configured together") if self.history_retention_days < 0 or self.history_retention_days > 3650: raise ValueError("history_retention_days must be between 0 and 3650") if self.history_retention_interval_minutes < 0 or self.history_retention_interval_minutes > 10080: raise ValueError("history_retention_interval_minutes must be between 0 and 10080") if self.library_snapshot_interval_minutes < 0 or self.library_snapshot_interval_minutes > 10080: raise ValueError("library_snapshot_interval_minutes must be between 0 and 10080") if self.library_snapshot_keep < 1 or self.library_snapshot_keep > 100: raise ValueError("library_snapshot_keep must be between 1 and 100")
[docs] class PortalState: """Shared state for artifact summaries, prepared models, saved runs, and jobs.""" def __init__(self, config: PortalConfig) -> None: self.config = config self.config.runtime_dir.mkdir(parents=True, exist_ok=True) # type: ignore[union-attr] self.jobs = JobManager() self.chats = ChatStore(self.runtime_dir / "chats") self._prepared: dict[str, PreparedModel] = {} self._compute_lock = threading.RLock() self._preset_lock = threading.Lock() self._timeline_lock = threading.Lock() self._retention_lock = threading.Lock() self._retention_stop = threading.Event() self._retention_thread: threading.Thread | None = None self._snapshot_lock = threading.Lock() self._snapshot_stop = threading.Event() self._snapshot_thread: threading.Thread | None = None self._cheat_sheet: CheatSheet | None = None self._apply_startup_retention() self._start_retention_scheduler() self._start_snapshot_scheduler() self.ensure_cheat_sheet() @property def runtime_dir(self) -> Path: """Return the resolved runtime directory.""" assert self.config.runtime_dir is not None return self.config.runtime_dir
[docs] def options(self) -> dict[str, Any]: """Return portal-wide options for the browser shell.""" return { "title": self.config.title, "repo_root": str(self.config.repo_root), "runtime_dir": str(self.runtime_dir), "versions": list(self.config.versions), "assistant_enabled": self.config.allow_agent, "assistant_model": self.config.agent_model, "docs_url": "https://ubc-fresh.github.io/fable-pyculator/", }
[docs] def artifacts(self) -> list[dict[str, Any]]: """Return one artifact summary per configured version.""" return [ { **artifact_summary(self.config.repo_root, version), "prepared": version in self._prepared, "summary": self._prepared[version].summary if version in self._prepared else None, } for version in self.config.versions ]
[docs] def artifact(self, version: str) -> dict[str, Any]: """Return one artifact summary with its validation state.""" self._require_version(version) summary = artifact_summary(self.config.repo_root, version) return { **summary, "prepared": version in self._prepared, "summary": self._prepared[version].summary if version in self._prepared else None, "validation": self.validation(version), }
[docs] def validation(self, version: str) -> dict[str, Any]: """Return compact validation evidence without loading the large stage artifacts.""" self._require_version(version) paths = artifact_paths(self.config.repo_root, version) files = { "validation_summary": file_info(paths.validation_summary), "validation_scenario": file_info(paths.validation_scenario), "evaluation_report": file_info(paths.evaluation_report), "contract": file_info(paths.contract), } if paths.validation_summary.exists(): try: payload = json.loads(paths.validation_summary.read_text(encoding="utf-8")) except (OSError, json.JSONDecodeError) as error: return { "version": version, "available": False, "files": files, "message": f"validation summary could not be read: {error}", } return { "version": version, "available": True, "files": files, "summary": payload, "message": "compact validation evidence loaded", } if paths.validation_scenario.exists(): try: scenario = json.loads(paths.validation_scenario.read_text(encoding="utf-8")) except (OSError, json.JSONDecodeError): scenario = {} return { "version": version, "available": False, "files": files, "scenario": { "scenario_id": scenario.get("scenario_id"), "description": scenario.get("description"), "source_workbook": scenario.get("source_workbook"), "generated_model": scenario.get("generated_model"), "output_count": len(scenario.get("outputs", [])), } if scenario else None, "message": ( "A validation scenario exists, but the compact validation-evidence summary is missing. " "Run the validation-evidence packaging workflow to populate it." ), } return { "version": version, "available": False, "files": files, "message": "No validation evidence is available for this version.", }
[docs] def status(self, version: str) -> dict[str, Any]: """Return in-memory preparation status for one version.""" self._require_version(version) prepared = self._prepared.get(version) return { "version": version, "prepared": prepared is not None, "summary": prepared.summary if prepared is not None else None, "artifact": artifact_summary(self.config.repo_root, version), }
[docs] def prepare_job(self, version: str) -> Job: """Start a background preparation job for one version.""" self._require_version(version) return self.jobs.start("prepare", lambda report: self.prepare(version, report))
[docs] def prepare(self, version: str, report: Report) -> dict[str, Any]: """Materialize, discover, load, and cache one version. The operation is guarded so two analyst tabs do not decompress or build the same model at the same time. A valid cache avoids rebuilding the expensive workbook spec after a restart. """ with self._compute_lock: prepared = self._prepared.get(version) if prepared is not None: report(100, "model already prepared") return prepared.report paths = artifact_paths(self.config.repo_root, version) if not paths.workbook.exists(): raise PortalError( f"source workbook is missing: {paths.workbook}. " "Restore it under the ignored tmp/private-workbooks/ path.", status_code=409, ) if not paths.generated_model.exists(): if not paths.compressed_model.exists(): raise PortalError( f"generated model is missing: {paths.generated_model}", status_code=409, ) report(5, "decompressing approved generated-model artifact") materialize_generated_model(self.config.repo_root, version) checksum = model_checksum_status(self.config.repo_root, version) if checksum["state"] == "mismatch": if not paths.compressed_model.exists(): raise PortalError( "working generated model does not match the approved checksum and no " f"compressed archive is available: {paths.generated_model}", status_code=409, ) report(6, "working model checksum mismatch; re-materializing approved archive") materialize_generated_model(self.config.repo_root, version, force=True) checksum = model_checksum_status(self.config.repo_root, version) if checksum["state"] not in {"verified", "unknown"}: raise PortalError( "approved generated model failed checksum verification after materialization", status_code=409, ) fingerprint = _fingerprint(paths.workbook, paths.generated_model) cache = self._read_spec_cache(version, fingerprint) if cache is None: report(12, "discovering workbook controls and output tables") started = time.perf_counter() spec = build_notebook_spec(paths.workbook, workbook_id=version) summary = spec_summary(spec) duration = round(time.perf_counter() - started, 2) report(72, f"workbook spec ready in {duration:g} s") self._write_spec_cache(version, fingerprint, spec, summary) else: spec, summary = cache report(55, "loaded cached workbook spec") report(78, "loading generated model") started = time.perf_counter() model = load_generated_model( paths.generated_model, module_name=f"fable_pyculator_web_generated_fable_{version}", ) load_seconds = round(time.perf_counter() - started, 2) report(96, f"generated model loaded in {load_seconds:g} s") prepared_at = datetime.now(UTC).isoformat(timespec="seconds") payload = { "version": version, "prepared": True, "prepared_at": prepared_at, "model_load_seconds": load_seconds, "artifact": artifact_summary(self.config.repo_root, version), "summary": summary, } self._prepared[version] = PreparedModel( version=version, spec=spec, model=model, summary=summary, prepared_at=prepared_at, report=payload, ) self._refresh_cheat_sheet() report(100, "model prepared") return payload
[docs] def spec(self, version: str) -> dict[str, Any]: """Return the compact prepared spec, or a 409 when preparation is not done.""" self._require_version(version) prepared = self._prepared.get(version) if prepared is None: raise PortalError( f"model {version} is not prepared; start POST /api/models/{version}/prepare", status_code=409, ) return { "version": version, "prepared_at": prepared.prepared_at, "summary": prepared.summary, }
[docs] def run_job( self, *, version: str, name: str, selections: Mapping[str, object], output_table_names: Sequence[str] | None = None, headline_series_names: Sequence[str] | None = None, column_flavour_tags: str | Sequence[str] | None = None, include_context_columns: bool = True, horizon: int = NATIVE_LAST_YEAR, ) -> Job: """Start a background scenario run.""" self._require_version(version) return self.jobs.start( "run", lambda report: self.run_scenario( version=version, name=name, selections=selections, output_table_names=output_table_names, headline_series_names=headline_series_names, column_flavour_tags=column_flavour_tags, include_context_columns=include_context_columns, horizon=horizon, report=report, ), )
[docs] def run_scenario( self, *, version: str, name: str, selections: Mapping[str, object], output_table_names: Sequence[str] | None = None, headline_series_names: Sequence[str] | None = None, column_flavour_tags: str | Sequence[str] | None = None, include_context_columns: bool = True, horizon: int = NATIVE_LAST_YEAR, report: Report | None = None, ) -> dict[str, Any]: """Run one scenario and persist its summary and table payloads.""" if horizon not in {NATIVE_LAST_YEAR, 2070}: raise PortalError("horizon must be 2050 or 2070") report = report or (lambda _progress, _message: None) with self._compute_lock: prepared = self._prepared.get(version) if prepared is None: report(5, "model is not prepared; preparing it first") self.prepare(version, lambda progress, message: report(5 + int(progress * 0.45), message)) prepared = self._prepared[version] scenario_name = (name or "scenario").strip() or "scenario" report(55, "running generated model") started = time.perf_counter() result = run_notebook_loop( prepared.model, prepared.spec, dict(selections), scenario_name=scenario_name, output_table_names=output_table_names, output_table_column_flavour_tags=column_flavour_tags, include_context_columns=include_context_columns, headline_series_names=headline_series_names, include_figures=False, ) duration = round(time.perf_counter() - started, 2) report(85, f"generated model run complete in {duration:g} s") run_id = _run_id(scenario_name) run_dir = self.runtime_dir / "runs" / run_id table_dir = run_dir / "tables" table_dir.mkdir(parents=True, exist_ok=True) table_summaries: list[dict[str, Any]] = [] table_files: dict[str, str] = {} table_frames: dict[str, pd.DataFrame] = {} for table_name, frame in result.output_tables.items(): frame = _repair_known_output_table(version, table_name, frame) if horizon == 2070: frame = _extend_frame_to_2070(frame) table_frames[table_name] = frame payload = _frame_payload(frame, name=table_name) slug = _slug(table_name) relative = Path("tables") / f"{slug}.json" (run_dir / relative).write_text( json.dumps(payload, ensure_ascii=False) + "\n", encoding="utf-8", ) table_files[table_name] = str(relative) table_summaries.append( { "name": table_name, "label": payload.get("label"), "sheet": payload.get("sheet"), "range_ref": payload.get("range_ref"), "row_count": len(payload["rows"]), "column_count": len(payload["columns"]), "file": str(relative), "display_repair": payload.get("display_repair"), "extension_note": payload.get("extension_note"), } ) headline_frames = { series_name: (_extend_frame_to_2070(frame) if horizon == 2070 else frame) for series_name, frame in result.headline_frames.items() } headline_payloads = { series_name: _frame_payload(frame, name=series_name) for series_name, frame in headline_frames.items() } headline_summary = _headline_summary(prepared.spec, headline_frames) quality_warnings = _quality_warnings( version=version, spec=prepared.spec, horizon=horizon, headline_frames=headline_frames, table_frames=table_frames, table_summaries=table_summaries, ) paths = artifact_paths(self.config.repo_root, version) provenance = { "portal_version": __version__, "repo_root": str(self.config.repo_root), "prepared_at": prepared.prepared_at, "workbook_id": prepared.spec.workbook_id, "workbook": file_info(paths.workbook), "generated_model": file_info(paths.generated_model), "spec_provenance": prepared.spec.provenance, "run_started_at": datetime.now(UTC).isoformat(timespec="seconds"), } record = { "run_id": run_id, "version": version, "name": scenario_name, "created_at": datetime.now(UTC).isoformat(timespec="seconds"), "status": "ok", "duration_seconds": duration, "horizon": horizon, "extension": ( { "enabled": horizon == 2070, "method": "linear_trend_2035_2050", "native_last_year": NATIVE_LAST_YEAR, "note": EXTENSION_NOTE, } if horizon == 2070 else None ), "quality_warnings": quality_warnings, "selections": jsonable(dict(selections)), "input_count": len(result.run.inputs), "inputs": jsonable(result.run.inputs), "headline_summary": headline_summary, "headline_series": headline_payloads, "table_summaries": sorted(table_summaries, key=lambda item: item["name"]), "table_files": table_files, "skipped_output_tables": jsonable(result.skipped_output_tables), "provenance": provenance, "spec": { "workbook_id": prepared.spec.workbook_id, "provenance": prepared.spec.provenance, }, } (run_dir / "run.json").write_text( json.dumps(record, indent=2, ensure_ascii=False) + "\n", encoding="utf-8", ) self._refresh_cheat_sheet() report(100, "scenario saved") return record
[docs] def bundles(self) -> list[dict[str, Any]]: """Return tracked scenario-bundle examples.""" return bundle_summaries(self.config.repo_root)
[docs] def optimization_frontiers(self) -> list[dict[str, Any]]: """Return summaries of tracked exact frontier artifacts.""" directory = self.config.repo_root / "examples" / "optimization" if not directory.is_dir(): return [] records: list[dict[str, Any]] = [] for path in sorted(directory.glob("*/frontier.json")): try: payload = json.loads(path.read_text(encoding="utf-8")) except (OSError, json.JSONDecodeError): continue if not isinstance(payload, dict): continue points = payload.get("points") or [] origins: dict[str, int] = {} for point in points if isinstance(points, list) else []: if isinstance(point, dict): origin = str(point.get("origin") or "Other") origins[origin] = origins.get(origin, 0) + 1 records.append( { "version": str(payload.get("version") or path.parent.name.removeprefix("fable_")), "title": str(payload.get("title") or "Exact decision frontier"), "point_count": len(points) if isinstance(points, list) else 0, "pareto_count": len(payload.get("pareto_ids") or []), "generated_at": payload.get("generated_at"), "boundary": payload.get("boundary"), "origins": origins, } ) return sorted(records, key=lambda item: item["version"])
[docs] def optimization_frontier(self, version: str) -> dict[str, Any]: """Return one tracked exact frontier artifact for the browser.""" directory = self.config.repo_root / "examples" / "optimization" for path in sorted(directory.glob("*/frontier.json")) if directory.is_dir() else []: try: payload = json.loads(path.read_text(encoding="utf-8")) except (OSError, json.JSONDecodeError): continue if not isinstance(payload, dict): continue if str(payload.get("version")) == str(version): return payload raise PortalError(f"unknown optimization frontier {version!r}", status_code=404)
[docs] def optimization_presets(self) -> list[dict[str, Any]]: """Return the shared Optimization preset library.""" with self._preset_lock: return self._load_shared_presets()
[docs] def save_optimization_preset( self, name: str, state: Mapping[str, Any], saved_at: str | None = None, *, author: str | None = None, note: str | None = None, review_status: str | None = None, ) -> dict[str, Any]: """Create or replace one shared Optimization preset.""" preset = _clean_optimization_preset( name, state, saved_at, author=author, note=note, review_status=review_status, ) with self._preset_lock: existing = next( (item for item in self._load_shared_presets() if item["name"] == preset["name"]), None, ) history = list(existing.get("history", [])) if existing else [] if not existing: action = "created" elif existing.get("review_status") != "reviewed" and preset["review_status"] == "reviewed": action = "reviewed" else: action = "updated" history.append( { "at": preset["saved_at"], "action": action, "author": preset["author"], "note": preset["note"][:80], "review_status": preset["review_status"], } ) preset["history"] = history[-10:] presets = [item for item in self._load_shared_presets() if item["name"] != preset["name"]] presets.insert(0, preset) self._write_shared_presets(presets[:20]) return preset
[docs] def delete_optimization_preset(self, name: str) -> dict[str, Any]: """Delete one shared Optimization preset by name.""" with self._preset_lock: presets = self._load_shared_presets() remaining = [item for item in presets if item["name"] != name] if len(remaining) == len(presets): raise PortalError(f"unknown optimization preset {name!r}", status_code=404) self._write_shared_presets(remaining) return {"deleted": name, "count": len(remaining)}
[docs] def timeline_views(self) -> list[dict[str, Any]]: """Return the shared timeline filter library.""" with self._timeline_lock: return self._load_shared_timeline_views()
[docs] def optimization_library(self) -> dict[str, Any]: """Return a portable backup of shared presets and timeline views.""" return { "schema": "fable-pyculator.optimization-library", "schema_version": 1, "exported_at": datetime.now(UTC).isoformat(timespec="seconds"), "presets": self.optimization_presets(), "timeline_views": self.timeline_views(), }
[docs] def restore_optimization_library( self, presets: Any, timeline_views: Any, *, mode: str = "merge", dry_run: bool = True, ) -> dict[str, Any]: """Preview or apply a shared-library backup in merge or replace mode.""" if mode not in {"merge", "replace"}: raise ValueError("restore mode must be merge or replace") if not isinstance(presets, (list, tuple)) or not isinstance(timeline_views, (list, tuple)): raise ValueError("restore payload needs presets and timeline_views lists") if len(presets) > 20 or len(timeline_views) > 20: raise ValueError("a restore can contain at most 20 presets and 20 timeline views") incoming_presets: list[dict[str, Any]] = [] seen_presets: set[str] = set() for index, item in enumerate(presets, start=1): try: preset = _clean_restored_preset(item) except ValueError as error: raise ValueError(f"preset {index}: {error}") from error if preset["name"] in seen_presets: raise ValueError(f"duplicate preset name {preset['name']!r}") seen_presets.add(preset["name"]) incoming_presets.append(preset) incoming_views: list[dict[str, Any]] = [] seen_views: set[str] = set() for index, item in enumerate(timeline_views, start=1): if not isinstance(item, Mapping): raise ValueError(f"timeline view {index} must be an object") try: view = _clean_timeline_view( item.get("name"), item.get("filter"), item.get("saved_at"), author=item.get("author"), note=item.get("note"), review_status=item.get("review_status"), ) except ValueError as error: raise ValueError(f"timeline view {index}: {error}") from error if view["name"] in seen_views: raise ValueError(f"duplicate timeline view name {view['name']!r}") seen_views.add(view["name"]) incoming_views.append(view) with self._preset_lock, self._timeline_lock: existing_presets = self._load_shared_presets() existing_views = self._load_shared_timeline_views() existing_preset_names = {preset["name"] for preset in existing_presets} existing_view_names = {view["name"] for view in existing_views} incoming_preset_names = {preset["name"] for preset in incoming_presets} incoming_view_names = {view["name"] for view in incoming_views} preset_added = [name for name in incoming_preset_names if name not in existing_preset_names] preset_replaced = [name for name in incoming_preset_names if name in existing_preset_names] view_added = [name for name in incoming_view_names if name not in existing_view_names] view_replaced = [name for name in incoming_view_names if name in existing_view_names] if mode == "merge": if len(existing_preset_names | incoming_preset_names) > 20: raise ValueError("merge would exceed 20 shared presets; use replace or reduce the bundle") if len(existing_view_names | incoming_view_names) > 20: raise ValueError("merge would exceed 20 shared timeline views; use replace or reduce the bundle") result_presets: list[dict[str, Any]] = [] existing_by_name = {preset["name"]: preset for preset in existing_presets} for preset in incoming_presets: old = existing_by_name.get(preset["name"]) if old is not None: preset = dict(preset) preset["history"] = _merge_preset_history(old.get("history"), preset.get("history")) result_presets.append(preset) result_presets.extend( preset for preset in existing_presets if preset["name"] not in incoming_preset_names ) result_views: list[dict[str, Any]] = list(incoming_views) result_views.extend(view for view in existing_views if view["name"] not in incoming_view_names) preset_removed: list[str] = [] view_removed: list[str] = [] else: result_presets = list(incoming_presets) result_views = list(incoming_views) preset_removed = [name for name in existing_preset_names if name not in incoming_preset_names] view_removed = [name for name in existing_view_names if name not in incoming_view_names] changed = bool( preset_added or preset_replaced or preset_removed or view_added or view_replaced or view_removed ) if not dry_run and changed: self._write_shared_presets(result_presets[:20]) self._write_shared_timeline_views(result_views[:20]) return { "schema": "fable-pyculator.optimization-library", "schema_version": 1, "mode": mode, "dry_run": bool(dry_run), "changed": changed, "at": datetime.now(UTC).isoformat(timespec="seconds"), "presets": { "incoming": len(incoming_presets), "added": len(preset_added), "replaced": len(preset_replaced), "removed": len(preset_removed), "result": len(result_presets), "added_names": sorted(preset_added), "replaced_names": sorted(preset_replaced), "removed_names": sorted(preset_removed), }, "timeline_views": { "incoming": len(incoming_views), "added": len(view_added), "replaced": len(view_replaced), "removed": len(view_removed), "result": len(result_views), "added_names": sorted(view_added), "replaced_names": sorted(view_replaced), "removed_names": sorted(view_removed), }, }
def _snapshot_dir(self) -> Path: path = self.runtime_dir / "library-snapshots" path.mkdir(parents=True, exist_ok=True) return path @staticmethod def _snapshot_name(value: datetime) -> str: return f"library-{value.strftime('%Y%m%dT%H%M%S%fZ')}.json" @staticmethod def _snapshot_path_name(name: Any) -> str: if not isinstance(name, str) or not re.fullmatch(r"library-\d{8}T\d{12}Z\.json", name): raise ValueError("invalid library snapshot name") return name @staticmethod def _snapshot_summary(path: Path, payload: Mapping[str, Any]) -> dict[str, Any]: presets = payload.get("presets") views = payload.get("timeline_views") return { "name": path.name, "at": str(payload.get("exported_at", "")), "source": str(payload.get("snapshot_source", "manual")), "presets": len(presets) if isinstance(presets, list) else 0, "timeline_views": len(views) if isinstance(views, list) else 0, "bytes": path.stat().st_size, }
[docs] def create_library_snapshot(self, source: str = "manual") -> dict[str, Any]: """Write one immutable library snapshot and retain the newest configured count.""" if source not in {"manual", "scheduled"}: raise ValueError("snapshot source must be manual or scheduled") with self._snapshot_lock: at = datetime.now(UTC) path = self._snapshot_dir() / self._snapshot_name(at) while path.exists(): at = at + timedelta(microseconds=1) path = self._snapshot_dir() / self._snapshot_name(at) bundle = self.optimization_library() bundle["snapshot_source"] = source temporary = path.with_name(f".{path.name}.{uuid.uuid4().hex}.tmp") temporary.write_text(json.dumps(bundle, indent=2, sort_keys=True) + "\n", encoding="utf-8") temporary.replace(path) pruned = 0 snapshots = sorted(self._snapshot_dir().glob("library-*.json"), key=lambda item: item.name, reverse=True) for old in snapshots[self.config.library_snapshot_keep :]: try: old.unlink() pruned += 1 except OSError: continue summary = self._snapshot_summary(path, bundle) summary["pruned"] = pruned return summary
[docs] def library_snapshots(self, limit: int = 20) -> list[dict[str, Any]]: """Return recent snapshot metadata, newest first.""" bounded = max(1, min(100, int(limit))) directory = self._snapshot_dir() summaries: list[dict[str, Any]] = [] for path in sorted(directory.glob("library-*.json"), key=lambda item: item.name, reverse=True): if not re.fullmatch(r"library-\d{8}T\d{12}Z\.json", path.name): continue try: payload = json.loads(path.read_text(encoding="utf-8")) except (OSError, json.JSONDecodeError): continue if isinstance(payload, Mapping): summaries.append(self._snapshot_summary(path, payload)) if len(summaries) >= bounded: break return summaries
[docs] def library_snapshot_policy(self) -> dict[str, Any]: """Return snapshot scheduling and retention policy metadata.""" return { "interval_minutes": self.config.library_snapshot_interval_minutes, "keep": self.config.library_snapshot_keep, "scheduled": self.config.library_snapshot_interval_minutes > 0, "count": len(self.library_snapshots(limit=100)), "directory": str(self._snapshot_dir()), }
[docs] def library_snapshot(self, name: str) -> dict[str, Any]: """Return one stored snapshot bundle.""" clean_name = self._snapshot_path_name(name) path = self._snapshot_dir() / clean_name if not path.is_file(): raise PortalError(f"unknown library snapshot {clean_name!r}", status_code=404) try: payload = json.loads(path.read_text(encoding="utf-8")) except (OSError, json.JSONDecodeError) as error: raise PortalError(f"library snapshot {clean_name!r} could not be read: {error}", status_code=500) from error if not isinstance(payload, Mapping): raise PortalError(f"library snapshot {clean_name!r} is invalid", status_code=500) return dict(payload)
[docs] def delete_library_snapshot(self, name: str) -> dict[str, Any]: """Delete one stored snapshot.""" clean_name = self._snapshot_path_name(name) path = self._snapshot_dir() / clean_name with self._snapshot_lock: if not path.is_file(): raise PortalError(f"unknown library snapshot {clean_name!r}", status_code=404) path.unlink() return {"deleted": clean_name, "count": len(self.library_snapshots(limit=100))}
[docs] def restore_library_snapshot( self, name: str, *, mode: str = "merge", dry_run: bool = True, ) -> dict[str, Any]: """Preview or apply a restore directly from a stored snapshot.""" payload = self.library_snapshot(name) result = self.restore_optimization_library( payload.get("presets", []), payload.get("timeline_views", []), mode=mode, dry_run=dry_run, ) result["snapshot"] = name return result
[docs] def diff_library_snapshot(self, name: str, mode: str = "merge") -> dict[str, Any]: """Return an exact read-only diff between one snapshot and the current library.""" if mode not in {"merge", "replace"}: raise ValueError("diff mode must be merge or replace") snapshot = self.library_snapshot(name) current = self.optimization_library() snapshot_presets = { str(item["name"]): item for item in snapshot.get("presets", []) if isinstance(item, Mapping) and isinstance(item.get("name"), str) } current_presets = { str(item["name"]): item for item in current.get("presets", []) if isinstance(item, Mapping) and isinstance(item.get("name"), str) } added = sorted(set(snapshot_presets) - set(current_presets)) replaced = sorted(set(snapshot_presets) & set(current_presets)) current_only = sorted(set(current_presets) - set(snapshot_presets)) changed: list[dict[str, Any]] = [] for preset_name in replaced: old = current_presets[preset_name] new = snapshot_presets[preset_name] metadata = [ field for field in ("author", "note", "review_status", "saved_at") if old.get(field) != new.get(field) ] old_state = old.get("state") if isinstance(old.get("state"), Mapping) else {} new_state = new.get("state") if isinstance(new.get("state"), Mapping) else {} state = sorted(key for key in set(old_state) | set(new_state) if old_state.get(key) != new_state.get(key)) old_history = old.get("history") if isinstance(old.get("history"), list) else [] new_history = new.get("history") if isinstance(new.get("history"), list) else [] old_keys = {_preset_event_key(event) for event in old_history if isinstance(event, Mapping)} new_keys = {_preset_event_key(event) for event in new_history if isinstance(event, Mapping)} history_added = [ event for event in new_history if isinstance(event, Mapping) and _preset_event_key(event) not in old_keys ] history_removed = [ event for event in old_history if isinstance(event, Mapping) and _preset_event_key(event) not in new_keys ] if metadata or state or history_added or history_removed: changed.append( { "name": preset_name, "metadata": metadata, "state": state, "history_added": history_added, "history_removed": history_removed, } ) snapshot_views = { str(item["name"]): item for item in snapshot.get("timeline_views", []) if isinstance(item, Mapping) and isinstance(item.get("name"), str) } current_views = { str(item["name"]): item for item in current.get("timeline_views", []) if isinstance(item, Mapping) and isinstance(item.get("name"), str) } view_added = sorted(set(snapshot_views) - set(current_views)) view_replaced = sorted(set(snapshot_views) & set(current_views)) view_current_only = sorted(set(current_views) - set(snapshot_views)) view_changed: list[dict[str, Any]] = [] for view_name in view_replaced: old = current_views[view_name] new = snapshot_views[view_name] metadata = [ field for field in ("author", "note", "review_status", "saved_at") if old.get(field) != new.get(field) ] old_filter = old.get("filter") if isinstance(old.get("filter"), Mapping) else {} new_filter = new.get("filter") if isinstance(new.get("filter"), Mapping) else {} filter_changed = sorted( key for key in set(old_filter) | set(new_filter) if old_filter.get(key) != new_filter.get(key) ) if metadata or filter_changed: view_changed.append({"name": view_name, "metadata": metadata, "filter": filter_changed}) return { "snapshot": name, "mode": mode, "at": datetime.now(UTC).isoformat(timespec="seconds"), "presets": { "added": added, "replaced": replaced, "removed": current_only if mode == "replace" else [], "kept": current_only if mode == "merge" else [], "changed": changed, "unchanged": len(replaced) - len(changed), }, "timeline_views": { "added": view_added, "replaced": view_replaced, "removed": view_current_only if mode == "replace" else [], "kept": view_current_only if mode == "merge" else [], "changed": view_changed, "unchanged": len(view_replaced) - len(view_changed), }, }
[docs] def save_timeline_view( self, name: str, timeline_filter: Mapping[str, Any], saved_at: str | None = None, *, author: str | None = None, note: str | None = None, review_status: str | None = None, ) -> dict[str, Any]: """Create or replace one shared timeline filter view.""" view = _clean_timeline_view( name, timeline_filter, saved_at, author=author, note=note, review_status=review_status, ) with self._timeline_lock: views = [item for item in self._load_shared_timeline_views() if item["name"] != view["name"]] views.insert(0, view) self._write_shared_timeline_views(views[:20]) return view
[docs] def delete_timeline_view(self, name: str) -> dict[str, Any]: """Delete one shared timeline filter view.""" with self._timeline_lock: views = self._load_shared_timeline_views() remaining = [item for item in views if item["name"] != name] if len(remaining) == len(views): raise PortalError(f"unknown timeline view {name!r}", status_code=404) self._write_shared_timeline_views(remaining) return {"deleted": name, "count": len(remaining)}
def _load_shared_timeline_views(self) -> list[dict[str, Any]]: path = self.runtime_dir / "timeline-views.json" if not path.is_file(): return [] try: payload = json.loads(path.read_text(encoding="utf-8")) except (OSError, json.JSONDecodeError): return [] if not isinstance(payload, list): return [] views: list[dict[str, Any]] = [] for item in payload: if not isinstance(item, Mapping): continue try: views.append( _clean_timeline_view( item.get("name"), item.get("filter"), item.get("saved_at"), author=item.get("author"), note=item.get("note"), review_status=item.get("review_status"), ) ) except ValueError: continue return views[:20] def _write_shared_timeline_views(self, views: list[dict[str, Any]]) -> None: path = self.runtime_dir / "timeline-views.json" path.parent.mkdir(parents=True, exist_ok=True) temporary = path.with_name(f".{path.name}.{uuid.uuid4().hex}.tmp") temporary.write_text(json.dumps(views, indent=2, sort_keys=True) + "\n", encoding="utf-8") temporary.replace(path)
[docs] def prune_optimization_history( self, days: int = 90, dry_run: bool = False, *, source: str = "manual", ) -> dict[str, Any]: """Remove history events older than the retention window, keeping each latest event.""" if not isinstance(days, int) or days < 1 or days > 3650: raise ValueError("retention days must be between 1 and 3650") cutoff = datetime.now(UTC) - timedelta(days=days) with self._preset_lock: presets = self._load_shared_presets() removed = 0 kept = 0 touched = 0 details: list[dict[str, Any]] = [] details_truncated = False updated: list[dict[str, Any]] = [] for preset in presets: history = list(preset.get("history", [])) if not history: updated.append(preset) continue retained: list[dict[str, Any]] = [] removed_events: list[dict[str, Any]] = [] for index, event in enumerate(history): keep = index == len(history) - 1 if not keep: try: event_time = datetime.fromisoformat(str(event.get("at", ""))) if event_time.tzinfo is None: event_time = event_time.replace(tzinfo=UTC) keep = event_time >= cutoff except ValueError: keep = True if keep: retained.append(event) else: removed += 1 removed_events.append( {"at": str(event.get("at", ""))[:40], "action": str(event.get("action", ""))[:20]} ) kept += len(retained) if len(retained) != len(history): touched += 1 if len(details) < 20: details.append( { "preset": preset["name"], "before": [ {"at": str(event.get("at", ""))[:40], "action": str(event.get("action", ""))[:20]} for event in history[:10] ], "after": [ {"at": str(event.get("at", ""))[:40], "action": str(event.get("action", ""))[:20]} for event in retained[:10] ], "removed": removed_events[:10], "kept": len(retained), } ) else: details_truncated = True preset = dict(preset) preset["history"] = retained updated.append(preset) if not dry_run and touched: self._write_shared_presets(updated) result = { "days": days, "dry_run": bool(dry_run), "removed": removed, "kept": kept, "presets_touched": touched, "presets_total": len(presets), "source": source, "at": datetime.now(UTC).isoformat(timespec="seconds"), "details": details, "details_truncated": details_truncated, } self._record_retention(result) return result
[docs] def retention_status(self) -> dict[str, Any]: """Return the startup retention policy and the last recorded result.""" path = self.runtime_dir / "retention-log.jsonl" last: dict[str, Any] | None = None if path.is_file(): try: lines = path.read_text(encoding="utf-8").splitlines() if lines: last = json.loads(lines[-1]) except (OSError, json.JSONDecodeError): last = None interval = self.config.history_retention_interval_minutes next_run = None if interval > 0 and self.config.history_retention_days > 0: base = datetime.now(UTC) if last and isinstance(last.get("at"), str): try: base = datetime.fromisoformat(last["at"]) if base.tzinfo is None: base = base.replace(tzinfo=UTC) except ValueError: base = datetime.now(UTC) next_run = (base + timedelta(minutes=interval)).isoformat(timespec="seconds") return { "history_retention_days": self.config.history_retention_days, "history_retention_dry_run": self.config.history_retention_dry_run, "history_retention_interval_minutes": interval, "next_run_at": next_run, "log": str(path), "last": last, }
[docs] def retention_runs(self, limit: int = 20) -> list[dict[str, Any]]: """Return recent retention maintenance records, newest first.""" bounded = max(1, min(100, int(limit))) path = self.runtime_dir / "retention-log.jsonl" if not path.is_file(): return [] try: lines = path.read_text(encoding="utf-8").splitlines() except OSError: return [] records: list[dict[str, Any]] = [] for line in lines: if not line.strip(): continue try: payload = json.loads(line) except json.JSONDecodeError: continue if isinstance(payload, Mapping): records.append(dict(payload)) return list(reversed(records[-bounded:]))
def _apply_startup_retention(self) -> None: days = self.config.history_retention_days if days <= 0: return try: self.prune_optimization_history( days, dry_run=self.config.history_retention_dry_run, source="startup", ) except ValueError: return def _start_retention_scheduler(self) -> None: days = self.config.history_retention_days interval = self.config.history_retention_interval_minutes if days <= 0 or interval <= 0: return def loop() -> None: while not self._retention_stop.wait(interval * 60): try: self.prune_optimization_history( days, dry_run=self.config.history_retention_dry_run, source="scheduled", ) except Exception: # noqa: BLE001 - maintenance must not kill the portal continue thread = threading.Thread(target=loop, name="fable-retention", daemon=True) self._retention_thread = thread thread.start() def _start_snapshot_scheduler(self) -> None: interval = self.config.library_snapshot_interval_minutes if interval <= 0: return def loop() -> None: while not self._snapshot_stop.wait(interval * 60): try: self.create_library_snapshot(source="scheduled") except Exception: # noqa: BLE001 - maintenance must not kill the portal continue thread = threading.Thread(target=loop, name="fable-snapshots", daemon=True) self._snapshot_thread = thread thread.start()
[docs] def close(self) -> None: """Stop optional retention and snapshot maintenance threads.""" self._retention_stop.set() thread = self._retention_thread if thread is not None and thread.is_alive(): thread.join(timeout=1.0) self._snapshot_stop.set() snapshot_thread = self._snapshot_thread if snapshot_thread is not None and snapshot_thread.is_alive(): snapshot_thread.join(timeout=1.0)
def _record_retention(self, result: Mapping[str, Any]) -> None: path = self.runtime_dir / "retention-log.jsonl" with self._retention_lock: path.parent.mkdir(parents=True, exist_ok=True) with path.open("a", encoding="utf-8") as target: target.write(json.dumps(dict(result), sort_keys=True) + "\n") def _load_shared_presets(self) -> list[dict[str, Any]]: path = self.runtime_dir / "optimization-presets.json" if not path.is_file(): return [] try: payload = json.loads(path.read_text(encoding="utf-8")) except (OSError, json.JSONDecodeError): return [] if not isinstance(payload, list): return [] presets: list[dict[str, Any]] = [] for item in payload: if not isinstance(item, Mapping): continue try: preset = _clean_optimization_preset( item.get("name"), item.get("state"), item.get("saved_at"), author=item.get("author"), note=item.get("note"), review_status=item.get("review_status"), ) preset["history"] = _clean_preset_history(item.get("history")) presets.append(preset) except ValueError: continue return presets[:20] def _write_shared_presets(self, presets: list[dict[str, Any]]) -> None: path = self.runtime_dir / "optimization-presets.json" path.parent.mkdir(parents=True, exist_ok=True) temporary = path.with_name(f".{path.name}.{uuid.uuid4().hex}.tmp") temporary.write_text(json.dumps(presets, indent=2, sort_keys=True) + "\n", encoding="utf-8") temporary.replace(path)
[docs] def runs(self) -> list[dict[str, Any]]: """Return saved runs, newest first, without loading table files.""" directory = self.runtime_dir / "runs" records: list[dict[str, Any]] = [] if not directory.is_dir(): return records for path in directory.glob("*/run.json"): try: payload = json.loads(path.read_text(encoding="utf-8")) except (OSError, json.JSONDecodeError): continue record = dict(payload) record.pop("inputs", None) record.pop("headline_series", None) records.append(record) return sorted(records, key=lambda item: str(item.get("created_at", "")), reverse=True)
[docs] def run(self, run_id: str) -> dict[str, Any]: """Return one saved run.""" path = self._run_path(run_id) if not path.exists(): raise PortalError(f"unknown run id {run_id!r}", status_code=404) try: return json.loads(path.read_text(encoding="utf-8")) except (OSError, json.JSONDecodeError) as error: raise PortalError(f"run {run_id!r} could not be read: {error}", status_code=500) from error
[docs] def table(self, run_id: str, table_name: str) -> dict[str, Any]: """Return one saved table payload.""" record = self.run(run_id) relative = record.get("table_files", {}).get(table_name) if not relative: raise PortalError(f"unknown table {table_name!r} for run {run_id!r}", status_code=404) path = (self._run_path(run_id).parent / str(relative)).resolve() if not path.is_relative_to(self._run_path(run_id).parent.resolve()): raise PortalError("table path escapes the run directory", status_code=400) try: return json.loads(path.read_text(encoding="utf-8")) except (OSError, json.JSONDecodeError) as error: raise PortalError(f"table {table_name!r} could not be read: {error}", status_code=500) from error
[docs] def compare(self, run_ids: Sequence[str]) -> dict[str, Any]: """Compare two or more saved runs by headline series and selected inputs.""" unique = list(dict.fromkeys(str(run_id) for run_id in run_ids)) if len(unique) < 2: raise PortalError("select at least two runs to compare") records = [self.run(run_id) for run_id in unique] series: dict[str, dict[str, Any]] = {} years: set[Any] = set() for record in records: for name, payload in record.get("headline_series", {}).items(): entry = series.setdefault( name, { "name": name, "label": payload.get("label") or name, "unit": payload.get("unit"), "group": payload.get("group"), "series": {}, }, ) mapping = { row.get("year"): row.get("value") for row in payload.get("rows", []) if isinstance(row, dict) } entry["series"][record["run_id"]] = mapping years.update(mapping) for entry in series.values(): entry["years"] = sorted(years, key=_year_key) entry["values"] = { record["run_id"]: [entry["series"][record["run_id"]].get(year) for year in entry["years"]] for record in records } entry.pop("series", None) return { "runs": [ { "run_id": record["run_id"], "name": record.get("name"), "version": record.get("version"), "created_at": record.get("created_at"), "selections": record.get("selections", {}), "headline_summary": record.get("headline_summary", []), } for record in records ], "series": sorted(series.values(), key=lambda item: item["name"]), }
[docs] def assistant_context(self, run_id: str | None) -> dict[str, Any]: """Build a compact context object for the optional local assistant.""" cheat_sheet = self._cheat_sheet or self.ensure_cheat_sheet() context: dict[str, Any] = { "versions": [artifact["version"] for artifact in self.artifacts()], "cheat_sheet_path": str(cheat_sheet.path), "cheat_sheet_generated_at": cheat_sheet.generated_at, } if run_id: try: record = self.run(run_id) except PortalError: return context context["run"] = { key: record.get(key) for key in ("run_id", "name", "version", "created_at", "selections", "headline_summary") } return context
[docs] def ensure_cheat_sheet(self) -> CheatSheet: """Regenerate the compiled assistant grounding sheet from current portal state.""" cheat_sheet = ensure_cheat_sheet( repo_root=self.config.repo_root, runtime_dir=self.runtime_dir, artifacts=self.artifacts(), prepared_summaries={version: prepared.summary for version, prepared in self._prepared.items()}, runs=self.runs(), ) self._cheat_sheet = cheat_sheet return cheat_sheet
def _refresh_cheat_sheet(self) -> None: """Best-effort refresh after preparation or a saved run.""" try: self.ensure_cheat_sheet() except Exception: # noqa: BLE001 - auxiliary cache must not fail model jobs # The grounding sheet is auxiliary; never fail a model job because its cache write failed. return
[docs] def create_chat(self, *, run_id: str | None = None) -> dict[str, Any]: """Create a persistent assistant chat.""" return self.chats.create(run_id=run_id).to_dict()
[docs] def list_chats( self, *, include_archived: bool = False, query: str | None = None, ) -> list[dict[str, Any]]: """Return persistent chat summaries, pinned first and newest first.""" return self.chats.list(include_archived=include_archived, query=query)
[docs] def update_chat( self, chat_id: str, *, title: str | None = None, pinned: bool | None = None, archived: bool | None = None, ) -> dict[str, Any]: """Update chat title, pin, or archive state.""" try: return self.chats.set_metadata( chat_id, title=title, pinned=pinned, archived=archived, ).to_dict() except KeyError as error: raise PortalError(f"unknown chat id {chat_id!r}", status_code=404) from error
[docs] def export_chat(self, chat_id: str) -> str: """Return a Markdown export of one chat.""" try: return self.chats.get(chat_id).to_markdown() except KeyError as error: raise PortalError(f"unknown chat id {chat_id!r}", status_code=404) from error
[docs] def get_chat(self, chat_id: str) -> dict[str, Any]: """Return one persistent chat.""" try: return self.chats.get(chat_id).to_dict() except KeyError as error: raise PortalError(f"unknown chat id {chat_id!r}", status_code=404) from error
[docs] def delete_chat(self, chat_id: str) -> dict[str, Any]: """Delete one persistent chat.""" try: deleted = self.chats.delete(chat_id) except KeyError as error: raise PortalError(f"invalid chat id {chat_id!r}", status_code=400) from error if not deleted: raise PortalError(f"unknown chat id {chat_id!r}", status_code=404) return {"chat_id": chat_id, "deleted": True}
def _require_version(self, version: str) -> None: if version not in self.config.versions: raise PortalError(f"unknown workbook version {version!r}", status_code=404) def _run_path(self, run_id: str) -> Path: if not re.fullmatch(r"[A-Za-z0-9._-]+", run_id): raise PortalError("invalid run id", status_code=400) return self.runtime_dir / "runs" / run_id / "run.json" def _cache_path(self, version: str) -> Path: return self.runtime_dir / "specs" / f"fable-{version}.json" def _read_spec_cache( self, version: str, fingerprint: dict[str, Any], ) -> tuple[Any, dict[str, Any]] | None: path = self._cache_path(version) if not path.exists(): return None try: payload = json.loads(path.read_text(encoding="utf-8")) if payload.get("cache_version") != SPEC_CACHE_VERSION: return None if payload.get("fingerprint") != fingerprint: return None spec = spec_from_payload(payload["spec"]) summary = payload["summary"] except (OSError, json.JSONDecodeError, KeyError, TypeError, ValueError): return None return spec, summary def _write_spec_cache( self, version: str, fingerprint: dict[str, Any], spec: Any, summary: dict[str, Any], ) -> None: path = self._cache_path(version) path.parent.mkdir(parents=True, exist_ok=True) payload = { "version": version, "cache_version": SPEC_CACHE_VERSION, "fingerprint": fingerprint, "summary": summary, "spec": spec_to_payload(spec), } temporary = path.with_name(path.name + ".tmp") temporary.write_text(json.dumps(payload, ensure_ascii=False) + "\n", encoding="utf-8") temporary.replace(path)
def _fingerprint(workbook: Path, model: Path) -> dict[str, Any]: def stat(path: Path) -> dict[str, Any]: info = path.stat() return {"size_bytes": info.st_size, "modified_ns": info.st_mtime_ns} return {"workbook": stat(workbook), "model": stat(model)} def _frame_payload(frame: pd.DataFrame, *, name: str) -> dict[str, Any]: attrs = frame.attrs rows = [ {str(column): jsonable(value) for column, value in zip(frame.columns, row, strict=False)} for row in frame.itertuples(index=False, name=None) ] return { "name": name, "label": attrs.get("label"), "description": attrs.get("description"), "sheet": attrs.get("sheet"), "range_ref": attrs.get("range_ref"), "columns": [str(column) for column in frame.columns], "index": [jsonable(value) for value in frame.index], "rows": rows, "selected_column_flavour_tags": jsonable(attrs.get("selected_column_flavour_tags")), "matched_column_flavour_tags": jsonable(attrs.get("matched_column_flavour_tags")), "column_flavour_tags": jsonable(attrs.get("column_flavour_tags")), "display_repair": attrs.get("display_repair"), "extension_note": attrs.get("extension_note"), } def _repair_known_output_table( version: str, table_name: str, frame: pd.DataFrame, ) -> pd.DataFrame: """Repair known broken display values without changing generated-model semantics. The 2024 generated model still contains the pre-repair ``LAND!AA`` ``#REF!`` expressions. Until that artifact is regenerated, the portal replaces only the displayed LAND TOTAL column with the workbook's ``TotallandFin`` component sum and records the repair in ``DataFrame.attrs``. """ if version != "2024" or table_name != "land_resultsland": return frame components = ("Cropland", "Pasture", "Urban", "ForPlantations", "OtherForest", "OtherLand") if "TOTAL" not in frame.columns or not all(component in frame.columns for component in components): return frame total = frame["TOTAL"] if not any(isinstance(value, str) and "#REF!" in value for value in total): return frame repaired = frame.copy() numeric = repaired.loc[:, list(components)].apply(pd.to_numeric, errors="coerce") repaired["TOTAL"] = numeric.sum(axis=1, min_count=len(components)) repaired.attrs.update(frame.attrs) repaired.attrs["display_repair"] = ( "The generated model's LAND TOTAL column is #REF!; this table displays the workbook's " "TotallandFin component sum instead (LoggedForest is represented inside OtherForest and is " "not added separately)." ) return repaired def _quality_warnings( *, version: str, spec: Any, horizon: int, headline_frames: Mapping[str, pd.DataFrame], table_frames: Mapping[str, pd.DataFrame], table_summaries: Sequence[dict[str, Any]], ) -> list[dict[str, Any]]: """Return run-scoped data-quality notes for the Results panel.""" warnings: list[dict[str, Any]] = [] if horizon == 2070: warnings.append( { "code": "trend_extension_2070", "severity": "warn", "title": "2070 values are trend extensions", "message": EXTENSION_NOTE, "scope": "headline series and extended output tables", } ) land = next((series for series in spec.headline_series if series.name == "land_total_area"), None) if land is not None and ( "component sum" in (land.label or "").casefold() or "totallandfin" in (land.description or "").casefold() ): warnings.append( { "code": "land_component_sum", "severity": "info", "title": "Land total uses a component sum", "message": ( "The workbook's ResultsLand[TOTAL] total is not usable as-is. The land series " "reproduces calc_land_cor[TotallandFin] from the component columns and records " "the source caveat in its description." ), "scope": "land_total_area", } ) if version == "2024" and not any(series.name == "water_total_footprint" for series in spec.headline_series): warnings.append( { "code": "water_unavailable_2024", "severity": "info", "title": "Water-footprint series is unavailable", "message": ( "The 2024 Canada workbook has no TotalResultsWF table, so no water-footprint " "headline series is exposed." ), "scope": "WATER", } ) mismatch = _ghg_cached_mismatch(headline_frames, table_frames, spec=spec) if mismatch is not None: years, maximum, max_year = mismatch warnings.append( { "code": "ghg_cached_mismatch", "severity": "warn", "title": "GHG components differ from cached workbook totals", "message": ( f"The derived GHG series differs from cached ResultsGHG[TotalCO2e] at " f"{len(years)} native year(s); the largest difference is {maximum:.3g} MtCO2e " f"in {max_year}. Cached values may reflect a different scenario/GWP, or an " "individual generated-model component may differ." ), "scope": "ghg_total_co2e vs cached TotalCO2e", } ) skipped = [ str(summary.get("label") or summary.get("name")) for summary in table_summaries if "not applied" in str(summary.get("extension_note") or "").casefold() ] if skipped: warnings.append( { "code": "tables_not_extended", "severity": "info", "title": "Some tables were not extended to 2070", "message": ( "These tables have no usable identity/trend grouping and remain at native 2050: " + ", ".join(skipped) ), "scope": "output tables", } ) return warnings def _cached_ghg_from_spec(spec: Any | None) -> dict[int, float]: if spec is None: return {} tables = getattr(spec, "output_tables", ()) table = next((item for item in tables if item.name == "ghg_resultsghg"), None) if table is None or not table.values: return {} labels = [str(label).casefold() for label in table.column_labels] try: year_index = labels.index("year") total_index = labels.index("totalco2e") except ValueError: return {} cached: dict[int, float] = {} for row in table.values: if len(row) <= max(year_index, total_index): continue year = _coerce_year(row[year_index]) value = row[total_index] if year is not None and isinstance(value, int | float) and not isinstance(value, bool): cached[year] = float(value) return cached def _ghg_cached_mismatch( headline_frames: Mapping[str, pd.DataFrame], table_frames: Mapping[str, pd.DataFrame], *, spec: Any | None = None, ) -> tuple[list[int], float, int] | None: headline = headline_frames.get("ghg_total_co2e") if headline is None: return None cached = _cached_ghg_from_spec(spec) if not cached: table = table_frames.get("ghg_resultsghg") if table is None or table.empty: return None year_column = next((column for column in table.columns if str(column).casefold() == "year"), None) if year_column is None or "TotalCO2e" not in table.columns: return None for _, row in table.iterrows(): year = _coerce_year(row.get(year_column)) value = row.get("TotalCO2e") if year is not None and isinstance(value, int | float) and not isinstance(value, bool): cached[year] = float(value) mismatches: list[tuple[int, float]] = [] for _, row in headline.iterrows(): year = _coerce_year(row.get("year")) value = row.get("value") if ( year is None or year > NATIVE_LAST_YEAR or year not in cached or not isinstance(value, int | float) or isinstance(value, bool) ): continue difference = abs(float(value) - cached[year]) tolerance = max(1e-6, abs(cached[year]) * 1e-6) if difference > tolerance: mismatches.append((year, difference)) if not mismatches: return None max_year, maximum = max(mismatches, key=lambda item: item[1]) return [year for year, _ in mismatches], maximum, max_year def _extend_frame_to_2070(frame: pd.DataFrame) -> pd.DataFrame: """Extend a year-indexed or year-column table using the 2035-2050 linear trend.""" if frame.empty: return frame copied = frame.copy() copied.attrs.update(frame.attrs) year_column = next( (column for column in copied.columns if str(column).casefold() == "year"), None, ) if year_column is not None: extended = _extend_year_rows(copied, year_column) if extended is not None: extended.attrs.update(copied.attrs) extended.attrs["extension_note"] = EXTENSION_NOTE return extended copied.attrs["extension_note"] = ( "This table has multiple rows per year; 2055-2070 trend extension was not applied." ) return copied if _extend_year_columns(copied): copied.attrs["extension_note"] = EXTENSION_NOTE return copied return frame def _extend_year_rows(frame: pd.DataFrame, year_column: Any) -> pd.DataFrame | None: years = [_coerce_year(value) for value in frame[year_column]] numeric_years = [year for year in years if year is not None] if not numeric_years: return None native_max = max(numeric_years) targets = [year for year in EXTENSION_YEARS if year > native_max] if not targets: return None if len(set(numeric_years)) == len(numeric_years): return _extend_series_rows(frame, year_column, years, native_max, targets) return _extend_grouped_rows(frame, year_column, native_max, targets) def _extend_series_rows( frame: pd.DataFrame, year_column: Any, years: list[int | None], native_max: int, targets: list[int], ) -> pd.DataFrame: """Extend a frame where each row is one year.""" trend_floor = max(2035, native_max - 15) last_index = frame.index[max(range(len(years)), key=lambda index: years[index] or -1)] appended: list[dict[Any, Any]] = [] for target in targets: row: dict[Any, Any] = {} for column in frame.columns: if column == year_column: row[column] = target continue if column == "cell_refs": row[column] = () continue if column == "description": row[column] = "Trend extension (non-native)" continue values = [ (year, value) for year, value in zip(years, frame[column], strict=False) if year is not None and trend_floor <= year <= native_max ] projected = _trend_value(values, target) row[column] = projected if projected is not None else frame.loc[last_index, column] appended.append(row) new = pd.DataFrame(appended, index=targets) extended = pd.concat([frame, new]) extended.attrs.update(frame.attrs) return extended def _extend_grouped_rows( frame: pd.DataFrame, year_column: Any, native_max: int, targets: list[int], ) -> pd.DataFrame | None: """Extend a multi-row-per-year frame by grouping on its text identity columns.""" group_columns = [ column for column in frame.columns if column != year_column and (pd.api.types.is_object_dtype(frame[column]) or pd.api.types.is_string_dtype(frame[column])) ] if not group_columns: return None trend_floor = max(2035, native_max - 15) appended: list[dict[Any, Any]] = [] extended_groups = 0 for _, group in frame.groupby(group_columns, dropna=False, sort=False): years = [_coerce_year(value) for value in group[year_column]] numeric_years = [year for year in years if year is not None] trend_years = [year for year in numeric_years if trend_floor <= year <= native_max] if len(trend_years) < 2 or len(set(numeric_years)) != len(numeric_years): continue last_index = group.index[max(range(len(years)), key=lambda index: years[index] or -1)] for target in targets: row: dict[Any, Any] = {} for column in frame.columns: if column == year_column: row[column] = target elif column == "cell_refs": row[column] = () elif column == "description": row[column] = "Trend extension (non-native)" else: values = [ (year, value) for year, value in zip(years, group[column], strict=False) if year is not None and trend_floor <= year <= native_max ] projected = _trend_value(values, target) row[column] = projected if projected is not None else group.loc[last_index, column] appended.append(row) extended_groups += 1 if not appended: return None new = pd.DataFrame(appended, index=[target for _ in range(extended_groups) for target in targets]) extended = pd.concat([frame, new]) extended.attrs.update(frame.attrs) return extended def _extend_year_columns(frame: pd.DataFrame) -> bool: year_columns: list[tuple[Any, int]] = [] for column in frame.columns: text = str(column) if text.isdigit() and 2000 <= int(text) <= 2100: year_columns.append((column, int(text))) if len(year_columns) < 2: return False native_max = max(year for _, year in year_columns) targets = [year for year in EXTENSION_YEARS if year > native_max] if not targets: return False trend_floor = max(2035, native_max - 15) style = type(year_columns[0][0]) for target in targets: frame[style(target)] = [ _trend_value( [ (year, row[column]) for column, year in year_columns if year is not None and trend_floor <= year <= native_max ], target, ) for _, row in frame.iterrows() ] return True def _trend_value(points: Sequence[tuple[int, Any]], target: int) -> float | None: usable = [ (float(year), float(value)) for year, value in points if isinstance(value, int | float) and not isinstance(value, bool) ] if not usable: return None if len(usable) == 1: return usable[0][1] count = len(usable) sum_x = sum(year for year, _ in usable) sum_y = sum(value for _, value in usable) sum_xy = sum(year * value for year, value in usable) sum_xx = sum(year * year for year, _ in usable) denominator = count * sum_xx - sum_x * sum_x if denominator == 0: return usable[-1][1] slope = (count * sum_xy - sum_x * sum_y) / denominator intercept = (sum_y - slope * sum_x) / count return intercept + slope * target def _coerce_year(value: Any) -> int | None: if isinstance(value, bool): return None if isinstance(value, int | float): return int(value) text = str(value).strip() return int(text) if text.isdigit() else None def _headline_summary(spec: Any, frames: Mapping[str, pd.DataFrame]) -> list[dict[str, Any]]: series_by_name = {series.name: series for series in spec.headline_series} summary: list[dict[str, Any]] = [] for name, frame in frames.items(): if "year" not in frame.columns or "value" not in frame.columns: continue points = [ {"year": jsonable(row["year"]), "value": jsonable(row["value"])} for _, row in frame.iterrows() if row.get("value") is not None ] if not points: continue first = points[0] last = points[-1] first_value = first["value"] last_value = last["value"] delta = None percent_change = None if isinstance(first_value, int | float) and isinstance(last_value, int | float): delta = last_value - first_value if first_value not in (0, None): percent_change = (delta / abs(first_value)) * 100 series = series_by_name.get(name) summary.append( { "name": name, "label": series.label if series else name, "group": series.group if series else None, "unit": series.unit if series else None, "first_year": first["year"], "first_value": first_value, "last_year": last["year"], "last_value": last_value, "delta": jsonable(delta), "percent_change": jsonable(percent_change), "point_count": len(points), } ) return sorted(summary, key=lambda item: item["name"]) def _run_id(name: str) -> str: stamp = datetime.now(UTC).strftime("%Y%m%dT%H%M%SZ") return f"{stamp}-{_slug(name)}-{uuid.uuid4().hex[:4]}" def _slug(value: str) -> str: slug = re.sub(r"[^A-Za-z0-9._-]+", "-", str(value)).strip("-").lower() return slug or "item" def _year_key(value: Any) -> tuple[int, Any]: try: return (0, int(value)) except (TypeError, ValueError): return (1, str(value))