Source code for fresh_hectaresbc.jobs

"""In-process bundle jobs with progress and best-effort cancellation.

The demo service runs one bundle job at a time so a six-layer clip cannot
starve tile rendering or the assistant. Jobs are persisted as JSON under the
bundle root so API clients can inspect recent work and service restarts leave a
clear ``interrupted`` record instead of a stuck ``running`` job.
"""

from __future__ import annotations

import copy
import json
import re
import threading
import uuid
from concurrent.futures import ThreadPoolExecutor
from datetime import UTC, datetime
from pathlib import Path
from typing import Any

from fresh_hectaresbc.bundle import (
    BundleCancelled,
    BundleError,
    prepare_bundle,
)
from fresh_hectaresbc.bundle_retention import prune_bundle_outputs

TERMINAL_STATUSES = {
    "complete",
    "partial",
    "failed",
    "canceled",
    "needs_name",
    "ambiguous",
    "interrupted",
}

_JOB_ID_RE = re.compile(r"^[0-9A-Za-z_-]+$")


def _now() -> str:
    return datetime.now(UTC).isoformat()


def _job_time(value: Any) -> datetime | None:
    if not isinstance(value, str) or not value:
        return None
    try:
        parsed = datetime.fromisoformat(value)
    except ValueError:
        return None
    if parsed.tzinfo is None:
        parsed = parsed.replace(tzinfo=UTC)
    return parsed.astimezone(UTC)


[docs] class BundleJobManager: """Queue and track clipped-raster bundle jobs.""" def __init__( self, *, output_root: str | Path, archive_root: str | Path | None = None, femic_src: str | Path | None = None, max_layers: int = 6, max_area_ha: float = 10_000_000.0, public_url: str | None = None, max_workers: int = 1, job_history_limit: int = 100, job_history_max_age_days: float = 30.0, bundle_history_limit: int = 50, bundle_history_max_age_days: float = 30.0, bundle_aoi_max_age_days: float = 7.0, bundle_aoi_max_files: int = 500, ) -> None: self.output_root = Path(output_root).expanduser().resolve() self.jobs_dir = self.output_root / "jobs" self.jobs_dir.mkdir(parents=True, exist_ok=True) self.archive_root = archive_root self.femic_src = femic_src self.max_layers = max_layers self.max_area_ha = max_area_ha self.public_url = public_url self.job_history_limit = max(1, int(job_history_limit)) self.job_history_max_age_days = float(job_history_max_age_days) self.bundle_history_limit = max(1, int(bundle_history_limit)) self.bundle_history_max_age_days = float(bundle_history_max_age_days) self.bundle_aoi_max_age_days = float(bundle_aoi_max_age_days) self.bundle_aoi_max_files = max(1, int(bundle_aoi_max_files)) self._lock = threading.RLock() self._jobs: dict[str, dict[str, Any]] = {} self._cancels: dict[str, threading.Event] = {} self._executor = ThreadPoolExecutor( max_workers=max_workers, thread_name_prefix="hbc-bundle", ) self._load_existing() self.prune_jobs() self._prune_outputs() def _job_path(self, job_id: str) -> Path: if not _JOB_ID_RE.fullmatch(job_id): raise ValueError("invalid bundle job id") return self.jobs_dir / f"{job_id}.json" def _load_existing(self) -> None: for path in sorted(self.jobs_dir.glob("*.json")): try: payload = json.loads(path.read_text(encoding="utf-8")) except (OSError, json.JSONDecodeError): continue job_id = payload.get("job_id") if ( not isinstance(job_id, str) or not _JOB_ID_RE.fullmatch(job_id) or path.stem != job_id ): continue if payload.get("status") in {"queued", "running"}: payload.update( { "status": "interrupted", "progress": 0, "message": "Service restarted before the bundle job finished.", "finished_utc": _now(), "error": "Service restarted before the job finished.", } ) self._write(payload) self._jobs[job_id] = payload def _write(self, payload: dict[str, Any]) -> None: job_id = payload.get("job_id") if not isinstance(job_id, str): return try: path = self._job_path(job_id) except ValueError: return tmp = self.jobs_dir / f".{job_id}.{uuid.uuid4().hex}.tmp" try: tmp.write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8") tmp.replace(path) except OSError: try: tmp.unlink(missing_ok=True) except OSError: pass
[docs] def prune_jobs( self, *, limit: int | None = None, max_age_days: float | None = None, ) -> dict[str, int]: """Remove old terminal job records while keeping active jobs. A terminal job is kept only when it is within both the newest-record limit and the age limit. Setting ``max_age_days`` to zero or a negative value disables age pruning. """ keep_limit = max(1, int(limit if limit is not None else self.job_history_limit)) age_limit = float( max_age_days if max_age_days is not None else self.job_history_max_age_days ) now = datetime.now(UTC) removed = 0 with self._lock: terminal: list[tuple[datetime, str]] = [] for job_id, job in self._jobs.items(): if job.get("status") not in TERMINAL_STATUSES: continue timestamp = _job_time(job.get("finished_utc")) or _job_time( job.get("created_utc") ) terminal.append((timestamp or datetime.min.replace(tzinfo=UTC), job_id)) terminal.sort(reverse=True) for index, (timestamp, job_id) in enumerate(terminal): too_old = False if age_limit > 0: age_days = (now - timestamp).total_seconds() / 86400 too_old = age_days > age_limit if index < keep_limit and not too_old: continue self._jobs.pop(job_id, None) self._cancels.pop(job_id, None) try: self._job_path(job_id).unlink(missing_ok=True) except (OSError, ValueError): pass removed += 1 return {"removed": removed, "kept": len(self._jobs)}
def stats(self) -> dict[str, int]: with self._lock: counts: dict[str, int] = {} for job in self._jobs.values(): status = str(job.get("status") or "unknown") counts[status] = counts.get(status, 0) + 1 counts["total"] = len(self._jobs) return counts
[docs] def prune_outputs(self) -> dict[str, int]: """Prune expired bundle outputs and temporary AOI files.""" return prune_bundle_outputs( self.output_root, max_bundles=self.bundle_history_limit, max_age_days=self.bundle_history_max_age_days, aoi_max_age_days=self.bundle_aoi_max_age_days, aoi_max_files=self.bundle_aoi_max_files, )
def _prune_outputs(self) -> dict[str, int]: return self.prune_outputs() def _update(self, job_id: str, **changes: Any) -> dict[str, Any] | None: with self._lock: job = self._jobs.get(job_id) if job is None: return None job.update(changes) self._write(job) return copy.deepcopy(job) def submit( self, *, layer_ids: list[str], aoi_query: str | None = None, aoi_name: str | None = None, aoi_geojson: str | Path | None = None, ) -> dict[str, Any]: unique_layers = list( dict.fromkeys(str(layer).strip() for layer in layer_ids if layer) ) job_id = f"{datetime.now(UTC).strftime('%Y%m%dT%H%M%SZ')}-{uuid.uuid4().hex[:8]}" job = { "job_id": job_id, "status": "queued", "progress": 0, "stage": "queued", "message": "Bundle job queued.", "created_utc": _now(), "started_utc": None, "finished_utc": None, "layer_ids": unique_layers, "layer_total": len(unique_layers), "layer_index": 0, "current_layer": None, "completed_layers": 0, "failed_layers": 0, "aoi_name": aoi_name, "result": None, "error": None, } event = threading.Event() with self._lock: self._jobs[job_id] = job self._cancels[job_id] = event self._write(job) self._executor.submit( self._run, job_id, unique_layers, aoi_query, aoi_name, str(aoi_geojson) if aoi_geojson else None, event, ) return self.get(job_id) or copy.deepcopy(job) def _run( self, job_id: str, layer_ids: list[str], aoi_query: str | None, aoi_name: str | None, aoi_geojson: str | None, event: threading.Event, ) -> None: if event.is_set(): self._finish_canceled(job_id) return self._update( job_id, status="running", progress=1, stage="starting", message="Starting bundle job…", started_utc=_now(), ) def on_progress(payload: dict[str, Any]) -> None: if event.is_set(): return stage = payload.get("stage") total = max(int(payload.get("layer_total") or len(layer_ids)), 1) completed = int(payload.get("completed") or 0) failed = int(payload.get("failed") or 0) layer_index = int(payload.get("layer_index") or 0) current = payload.get("current_layer") if stage == "resolving_aoi": progress = 4 message = "Resolving AOI…" elif stage == "packaging": progress = 94 message = "Packaging ZIP and manifest…" elif stage == "clipping": if layer_index and current and completed + failed < layer_index: progress = 8 + int(84 * (layer_index - 1) / total) message = f"Clipping layer {layer_index} of {total}: {current}" else: progress = 8 + int(84 * completed / total) message = f"Clipped {completed} of {total} layers" if failed: message += f" ({failed} failed)" else: progress = 2 message = "Preparing bundle…" self._update( job_id, status="running", progress=max(0, min(progress, 99)), stage=str(stage or "working"), message=message, layer_total=total, layer_index=layer_index, current_layer=current, completed_layers=completed, failed_layers=failed, ) try: result = prepare_bundle( layer_ids=layer_ids, output_root=self.output_root, aoi_query=aoi_query, aoi_name=aoi_name, aoi_geojson=aoi_geojson, archive_root=self.archive_root, femic_src=self.femic_src, max_layers=self.max_layers, max_area_ha=self.max_area_ha, public_url=self.public_url, progress_callback=on_progress, cancel_check=event.is_set, ) except BundleCancelled: self._finish_canceled(job_id) return except BundleError as exc: self._finish_failed(job_id, str(exc)) return except Exception as exc: # noqa: BLE001 - report unexpected worker failures self._finish_failed(job_id, f"Unexpected bundle error: {exc}") return finally: with self._lock: self._cancels.pop(job_id, None) if event.is_set(): self._finish_canceled(job_id) return status = str(result.get("status") or "failed") if status in {"needs_name", "ambiguous"}: message = "Choose a boundary feature." progress = 0 elif status == "complete": message = "Bundle ready." progress = 100 elif status == "partial": message = "Bundle ready with some layer failures." progress = 100 else: message = "Bundle failed; inspect the failures." progress = 100 self._update( job_id, status=status, progress=progress, stage="finished", message=message, completed_layers=len(result.get("layers", []) or []), failed_layers=len(result.get("failures", []) or []), result=result, finished_utc=_now(), error=None if status != "failed" else message, ) self.prune_jobs() self._prune_outputs() def _finish_canceled(self, job_id: str) -> None: self._update( job_id, status="canceled", progress=0, stage="canceled", message="Bundle job canceled.", finished_utc=_now(), ) with self._lock: self._cancels.pop(job_id, None) self.prune_jobs() self._prune_outputs() def _finish_failed(self, job_id: str, message: str) -> None: self._update( job_id, status="failed", progress=0, stage="failed", message=message, finished_utc=_now(), error=message, ) self.prune_jobs() self._prune_outputs() def get(self, job_id: str) -> dict[str, Any] | None: with self._lock: job = self._jobs.get(job_id) return copy.deepcopy(job) if job is not None else None def list_jobs(self, limit: int = 50) -> list[dict[str, Any]]: with self._lock: jobs = sorted( self._jobs.values(), key=lambda job: str(job.get("created_utc") or ""), reverse=True, )[: max(1, int(limit))] return [copy.deepcopy(job) for job in jobs]
[docs] def retry(self, job_id: str) -> dict[str, Any] | None: """Re-queue the failed layers of a terminal job using its bundle AOI.""" with self._lock: job = copy.deepcopy(self._jobs.get(job_id)) if job is None: return None result = job.get("result") or {} failures = result.get("failures") or [] layer_ids = [ str(item.get("dataset_id")) for item in failures if isinstance(item, dict) and item.get("dataset_id") ] layer_ids = list(dict.fromkeys(layer_ids)) bundle_dir_value = result.get("bundle_dir") if not layer_ids or not bundle_dir_value: return None try: bundle_dir = Path(bundle_dir_value).expanduser().resolve() bundle_dir.relative_to(self.output_root) aoi_path = bundle_dir / "aoi.geojson" except (OSError, ValueError): return None if not aoi_path.is_file(): return None return self.submit( layer_ids=layer_ids, aoi_name=job.get("aoi_name") or result.get("aoi", {}).get("name"), aoi_geojson=aoi_path, )
def cancel(self, job_id: str) -> dict[str, Any] | None: with self._lock: job = self._jobs.get(job_id) if job is None: return None if job.get("status") in TERMINAL_STATUSES: return copy.deepcopy(job) event = self._cancels.get(job_id) if event is not None: event.set() job.update({"message": "Canceling bundle job…"}) self._write(job) return copy.deepcopy(job) def close(self) -> None: with self._lock: for event in self._cancels.values(): event.set() self._executor.shutdown(wait=False, cancel_futures=True)