Source code for fable_pyculator.web.grounding_evals

"""Grounding evaluations for the FABLE portal assistant.

The structural checks always run and verify that the generated cheat sheet points at real,
existing docs/source files and contains the expected API facts. The live checks are opt-in: they run
the installed OpenCode CLI with the same new-session grounding prompt and check that each answer
cites an expected source and contains the expected facts.
"""

from __future__ import annotations

import json
import time
from dataclasses import dataclass, field
from pathlib import Path
from typing import Any

from fable_pyculator.web.artifacts import DEFAULT_VERSIONS, artifact_summary
from fable_pyculator.web.assistant import DEFAULT_AGENT_MODEL, AgentError, AgentRequest, run_agent
from fable_pyculator.web.cheatsheet import CheatSheet, ensure_cheat_sheet

__all__ = [
    "DEFAULT_CASES",
    "GroundingCase",
    "GroundingCaseResult",
    "GroundingEvalSummary",
    "run_grounding_evals",
]

AGENT_COMMAND: tuple[str, ...] = ("opencode", "run", "--format", "json", "--auto")


[docs] @dataclass(frozen=True) class GroundingCase: """One assistant grounding question and its required evidence.""" case_id: str question: str expected_paths: tuple[str, ...] required_terms: tuple[str, ...] forbidden_terms: tuple[str, ...] = ()
DEFAULT_CASES: tuple[GroundingCase, ...] = ( GroundingCase( case_id="notebook_signature", question=( "What is the exact signature of build_notebook_spec in the current fable-pyculator " "source, including the optional headline-curation flag? Cite the exact file path." ), expected_paths=("src/fable_pyculator/notebook.py",), required_terms=("build_notebook_spec", "strict_headlines"), forbidden_terms=("no optional", "does not exist"), ), GroundingCase( case_id="modelwright_facade", question=("Which Modelwright module defines ModelFacade, and what does it wrap? Cite the exact file path."), expected_paths=("src/modelwright/wrappers.py",), required_terms=("ModelFacade", "generated"), ), GroundingCase( case_id="fable_2024_water", question=( "Which default FABLE headline series is unavailable in the 2024 Canada workbook, and " "why? Cite the workbook table name and a file path you used." ), expected_paths=("docs/guides/workbook-structure.rst", "src/fable_pyculator/discovery.py"), required_terms=("TotalResultsWF", "2024"), ), GroundingCase( case_id="portal_read_only", question=("Does the portal recompute FABLE results in the browser? Answer directly and cite a file path."), expected_paths=("docs/guides/web-portal.rst",), required_terms=("does not", "recompute"), ), GroundingCase( case_id="github_issue_filing", question=( "How would you file a bug report for this repository on the user's behalf? Which " "template, issue level, labels, validation command, and gh command would you use? " "Cite the workflow doc and template path." ), expected_paths=( "docs/guides/github-issue-workflow.rst", ".github/ISSUE_TEMPLATE/task.md", ), required_terms=("gh issue create", "--body-file", "check_issue_body.py", "bug", "task"), forbidden_terms=("invent the issue number", "without confirmation"), ), )
[docs] @dataclass class GroundingCaseResult: """Outcome of one structural or live grounding case.""" case_id: str status: str seconds: float = 0.0 failures: list[str] = field(default_factory=list) response: str = "" actions: list[dict[str, str]] = field(default_factory=list)
[docs] def to_dict(self) -> dict[str, Any]: """Return a JSON-ready result.""" return { "case_id": self.case_id, "status": self.status, "seconds": round(self.seconds, 2), "failures": list(self.failures), "response": self.response, "actions": list(self.actions), }
[docs] @dataclass class GroundingEvalSummary: """Combined structural and optional live evaluation summary.""" repo_root: str cheat_sheet_path: str structural: list[str] cases: list[GroundingCaseResult] live: bool @property def passed(self) -> bool: """Return whether every check passed.""" return not self.structural and all(case.status in {"pass", "skipped"} for case in self.cases)
[docs] def to_dict(self) -> dict[str, Any]: """Return a JSON-ready summary.""" return { "repo_root": self.repo_root, "cheat_sheet_path": self.cheat_sheet_path, "structural": list(self.structural), "cases": [case.to_dict() for case in self.cases], "live": self.live, "passed": self.passed, }
[docs] def run_grounding_evals( repo_root: str | Path, *, runtime_dir: str | Path | None = None, cases: tuple[GroundingCase, ...] = DEFAULT_CASES, include_live: bool = False, timeout: float = 600.0, ) -> GroundingEvalSummary: """Run structural checks and, when requested, live assistant evaluations.""" root = Path(repo_root).expanduser().resolve() runtime = ( Path(runtime_dir).expanduser().resolve() if runtime_dir is not None else root / "runtime" / "assistant-evals" ) artifacts = [artifact_summary(root, version) for version in DEFAULT_VERSIONS] cheat_sheet = ensure_cheat_sheet( repo_root=root, runtime_dir=runtime, artifacts=artifacts, prepared_summaries={}, runs=[], ) structural = structural_failures(root, cheat_sheet, cases) results: list[GroundingCaseResult] = [] for case in cases: if not include_live: results.append( GroundingCaseResult(case_id=case.case_id, status="skipped", failures=["live checks not requested"]) ) continue results.append(_evaluate_live_case(root, cheat_sheet, case, timeout=timeout)) return GroundingEvalSummary( repo_root=str(root), cheat_sheet_path=str(cheat_sheet.path), structural=structural, cases=results, live=include_live, )
def structural_failures( repo_root: Path, cheat_sheet: CheatSheet, cases: tuple[GroundingCase, ...] = DEFAULT_CASES, ) -> list[str]: """Check that the sheet and required-reading list point at real files and facts.""" failures: list[str] = [] required_paths = [Path(path) for path in cheat_sheet.required_reading] for case in cases: for relative in case.expected_paths: matches = [path for path in required_paths if _path_matches(path, relative)] if not matches: failures.append(f"{case.case_id}: expected path is not in required reading: {relative}") if not any(path.is_file() for path in matches): failures.append(f"{case.case_id}: expected path is missing: {relative}") for relative in ( "docs/guides/web-portal.rst", "docs/guides/workbook-structure.rst", "src/fable_pyculator/notebook.py", "src/fable_pyculator/discovery.py", ): if not (repo_root / relative).is_file(): failures.append(f"required FABLE source/doc is missing: {repo_root / relative}") for term in ( "Documentation digest", "Source API index", "build_notebook_spec", "class ModelFacade", "TotalResultsWF", "GitHub issue filing", "check_issue_body.py", "2024", ): if term not in cheat_sheet.markdown: failures.append(f"cheat sheet is missing expected term: {term!r}") for path in cheat_sheet.required_reading: if not Path(path).is_file(): failures.append(f"required reading path does not exist: {path}") return failures def _evaluate_live_case( repo_root: Path, cheat_sheet: CheatSheet, case: GroundingCase, *, timeout: float, ) -> GroundingCaseResult: started = time.perf_counter() try: result = run_agent( AgentRequest( message=case.question, model=DEFAULT_AGENT_MODEL, new_session=True, cheat_sheet_path=str(cheat_sheet.path), required_reading=cheat_sheet.required_reading, context={ "repo_root": str(repo_root), "cheat_sheet_path": str(cheat_sheet.path), }, ), workspace=repo_root, command=AGENT_COMMAND, timeout=timeout, ) except AgentError as error: return GroundingCaseResult( case_id=case.case_id, status="fail", seconds=time.perf_counter() - started, failures=[str(error)], ) response = str(result.get("response", "")) actions = [action for action in result.get("actions", []) if isinstance(action, dict)] haystack = response + "\n" + json.dumps(actions) lowered = haystack.casefold() failures: list[str] = [] if not any(path in haystack for path in case.expected_paths): failures.append("answer/actions do not cite any expected path: " + ", ".join(case.expected_paths)) for term in case.required_terms: if term.casefold() not in lowered: failures.append(f"answer/actions do not contain expected term: {term!r}") for term in case.forbidden_terms: if term.casefold() in lowered: failures.append(f"answer/actions contain forbidden term: {term!r}") return GroundingCaseResult( case_id=case.case_id, status="pass" if not failures else "fail", seconds=time.perf_counter() - started, failures=failures, response=response, actions=actions, ) def _path_matches(path: Path, relative: str) -> bool: """Match repo-relative paths and installed-package paths with the same package/name.""" if path.as_posix() == relative or path.as_posix().endswith("/" + relative): return True expected = Path(relative) return path.name == expected.name and path.parent.name == expected.parent.name