"""Terrain-derived stream candidates with explicit review caveats.
P79.4 derives screening-level D8 flow paths from a native DEM, compares them
with an aligned public-hydrography reference, and writes a review manifest and
caveat report. Candidates are not authoritative streams.
"""
from __future__ import annotations
import heapq
import json
import math
from dataclasses import dataclass
from datetime import UTC, datetime
from pathlib import Path
import geopandas as gpd # type: ignore[import-untyped]
import numpy as np
import rasterio # type: ignore[import-untyped] # rasterio ships without PEP 561 stubs
import rasterio.features # type: ignore[import-untyped]
from femic.lidar.terrain import (
TerrainRasterMetadata,
read_terrain_grid,
require_projected_metre_grid,
)
STREAM_CANDIDATE_SCHEMA_VERSION = "femic.lidar.stream_candidates.v1"
STREAM_CANDIDATE_MANIFEST_NAME = "stream_candidates_manifest.json"
STREAM_CANDIDATE_REVIEW_NAME = "stream_candidates_review.md"
STREAM_CANDIDATE_RASTER_NAME = "stream_candidates.tif"
STREAM_ACCUMULATION_RASTER_NAME = "flow_accumulation.tif"
DEFAULT_DEPRESSION_EPSILON = 1e-6
DEFAULT_ACCUMULATION_THRESHOLD = 100
DEFAULT_STREAM_NODATA = 255
DEFAULT_ACCUMULATION_NODATA = -9999.0
D8_NEIGHBOURS: tuple[tuple[int, int, int, str], ...] = (
(0, 1, 1, "E"),
(1, 1, 2, "SE"),
(1, 0, 3, "S"),
(1, -1, 4, "SW"),
(0, -1, 5, "W"),
(-1, -1, 6, "NW"),
(-1, 0, 7, "N"),
(-1, 1, 8, "NE"),
)
STREAM_CAVEATS: tuple[str, ...] = (
(
"Terrain-derived candidates are a screening aid, not an authoritative "
"stream layer."
),
(
"D8 routing is sensitive to DEM artefacts, culverts, bridges, and source "
"resolution."
),
("Depression filling changes local flow paths and may create synthetic channels."),
(
"Candidates require field or public-hydrography review before riparian, "
"operability, or legal use."
),
"No reprojection or resampling was performed.",
)
[docs]
class StreamCandidateError(RuntimeError):
"""Raised when stream-candidate inputs or outputs are invalid."""
[docs]
@dataclass(frozen=True)
class StreamComparison:
"""Overlap metrics between candidate and reference stream masks."""
candidate_cells: int
reference_cells: int
true_positive: int
false_positive: int
false_negative: int
precision: float | None
recall: float | None
f1: float | None
jaccard: float | None
candidate_area_ha: float | None = None
reference_area_ha: float | None = None
overlap_area_ha: float | None = None
[docs]
def to_dict(self) -> dict[str, object]:
"""Return the comparison as a JSON-serializable mapping."""
return {
"candidate_cells": self.candidate_cells,
"reference_cells": self.reference_cells,
"true_positive": self.true_positive,
"false_positive": self.false_positive,
"false_negative": self.false_negative,
"precision": self.precision,
"recall": self.recall,
"f1": self.f1,
"jaccard": self.jaccard,
"candidate_area_ha": self.candidate_area_ha,
"reference_area_ha": self.reference_area_ha,
"overlap_area_ha": self.overlap_area_ha,
}
[docs]
@dataclass(frozen=True)
class StreamCandidateManifest:
"""Review manifest for one terrain-derived stream-candidate build."""
generated_utc: str
source: TerrainRasterMetadata
parameters: dict[str, object]
outputs: dict[str, str]
qa: dict[str, object]
comparison: StreamComparison | None
caveats: tuple[str, ...] = STREAM_CAVEATS
warnings: tuple[str, ...] = ()
schema_version: str = STREAM_CANDIDATE_SCHEMA_VERSION
[docs]
def to_dict(self) -> dict[str, object]:
"""Return the manifest as a JSON-serializable mapping."""
return {
"schema_version": self.schema_version,
"generated_utc": self.generated_utc,
"source": self.source.to_dict(),
"parameters": self.parameters,
"outputs": self.outputs,
"qa": self.qa,
"comparison": self.comparison.to_dict() if self.comparison else None,
"caveats": list(self.caveats),
"warnings": list(self.warnings),
}
[docs]
@dataclass(frozen=True)
class StreamCandidateResult:
"""Paths and manifest returned by a stream-candidate build."""
manifest: StreamCandidateManifest
output_dir: Path
stream_path: Path
accumulation_path: Path
manifest_path: Path
review_path: Path
[docs]
def to_dict(self) -> dict[str, object]:
"""Return the result as a JSON-serializable mapping."""
return {
"output_dir": str(self.output_dir),
"stream_path": str(self.stream_path),
"accumulation_path": str(self.accumulation_path),
"manifest_path": str(self.manifest_path),
"review_path": str(self.review_path),
"manifest": self.manifest.to_dict(),
}
[docs]
def fill_dem_depressions(
elevation: np.ndarray,
valid: np.ndarray,
*,
epsilon: float = DEFAULT_DEPRESSION_EPSILON,
) -> tuple[np.ndarray, int]:
"""Fill depressions with a priority-flood pass.
Returns the filled elevation grid and the count of valid cells that were
not connected to the outer boundary and therefore were left unfilled.
"""
if epsilon <= 0 or not math.isfinite(epsilon):
raise StreamCandidateError("Depression-fill epsilon must be positive.")
values = np.asarray(elevation, dtype=np.float64)
mask = np.asarray(valid, dtype=bool)
if values.shape != mask.shape:
raise StreamCandidateError("Elevation and validity arrays must match.")
filled = values.copy()
visited = np.zeros(mask.shape, dtype=bool)
height, width = mask.shape
heap: list[tuple[float, int, int]] = []
for row in range(height):
for column in (0, width - 1):
if mask[row, column] and not visited[row, column]:
visited[row, column] = True
heapq.heappush(heap, (filled[row, column], row, column))
for column in range(width):
for row in (0, height - 1):
if mask[row, column] and not visited[row, column]:
visited[row, column] = True
heapq.heappush(heap, (filled[row, column], row, column))
while heap:
current, row, column = heapq.heappop(heap)
for delta_row, delta_column, _, _ in D8_NEIGHBOURS:
neighbour_row = row + delta_row
neighbour_column = column + delta_column
if not (0 <= neighbour_row < height and 0 <= neighbour_column < width):
continue
if not mask[neighbour_row, neighbour_column]:
continue
if visited[neighbour_row, neighbour_column]:
continue
visited[neighbour_row, neighbour_column] = True
raised = max(filled[neighbour_row, neighbour_column], current + epsilon)
filled[neighbour_row, neighbour_column] = raised
heapq.heappush(heap, (raised, neighbour_row, neighbour_column))
unfilled_isolated = int(mask.sum() - visited.sum())
return filled, unfilled_isolated
[docs]
def derive_d8_flow_direction(
elevation: np.ndarray,
valid: np.ndarray,
*,
resolution_x: float,
resolution_y: float,
) -> tuple[np.ndarray, np.ndarray]:
"""Derive D8 flow directions and downstream flat indices.
Codes are 1=E, 2=SE, 3=S, 4=SW, 5=W, 6=NW, 7=N, 8=NE. Sinks and invalid
cells receive ``-1``.
"""
if resolution_x <= 0 or resolution_y <= 0:
raise StreamCandidateError("Flow routing requires positive resolution.")
values = np.asarray(elevation, dtype=np.float64)
mask = np.asarray(valid, dtype=bool)
if values.shape != mask.shape:
raise StreamCandidateError("Elevation and validity arrays must match.")
direction = np.full(values.shape, -1, dtype=np.int8)
downstream = np.full(values.shape, -1, dtype=np.int64)
height, width = values.shape
for row, column in zip(*np.nonzero(mask), strict=False):
current = values[row, column]
best_drop = 0.0
best_code = -1
best_target = -1
for delta_row, delta_column, code, _ in D8_NEIGHBOURS:
neighbour_row = row + delta_row
neighbour_column = column + delta_column
if not (0 <= neighbour_row < height and 0 <= neighbour_column < width):
continue
if not mask[neighbour_row, neighbour_column]:
continue
distance = math.hypot(
delta_row * resolution_x,
delta_column * resolution_y,
)
drop = (current - values[neighbour_row, neighbour_column]) / distance
if drop > best_drop + 1e-15:
best_drop = drop
best_code = code
best_target = neighbour_row * width + neighbour_column
direction[row, column] = best_code
downstream[row, column] = best_target
return direction, downstream
[docs]
def accumulate_d8_flow(
elevation: np.ndarray,
downstream: np.ndarray,
valid: np.ndarray,
) -> np.ndarray:
"""Accumulate one cell of area downstream over a D8 network."""
values = np.asarray(elevation, dtype=np.float64)
targets = np.asarray(downstream, dtype=np.int64)
mask = np.asarray(valid, dtype=bool)
if values.shape != targets.shape or values.shape != mask.shape:
raise StreamCandidateError("Flow arrays must share one shape.")
accumulation = np.zeros(values.size, dtype=np.float64)
valid_flat = np.flatnonzero(mask.ravel())
accumulation[valid_flat] = 1.0
order = valid_flat[np.argsort(values.ravel()[valid_flat], kind="stable")[::-1]]
targets_flat = targets.ravel()
for index in order:
target = targets_flat[index]
if target >= 0:
accumulation[target] += accumulation[index]
return accumulation.reshape(values.shape)
[docs]
def compare_stream_candidates(
candidate_mask: np.ndarray,
reference_mask: np.ndarray,
*,
pixel_area_m2: float | None = None,
) -> StreamComparison:
"""Compare candidate and reference stream masks."""
candidate = np.asarray(candidate_mask, dtype=bool)
reference = np.asarray(reference_mask, dtype=bool)
if candidate.shape != reference.shape:
raise StreamCandidateError("Candidate and reference masks must match.")
true_positive = int(np.count_nonzero(candidate & reference))
false_positive = int(np.count_nonzero(candidate & ~reference))
false_negative = int(np.count_nonzero(~candidate & reference))
candidate_cells = int(np.count_nonzero(candidate))
reference_cells = int(np.count_nonzero(reference))
precision = _safe_ratio(true_positive, true_positive + false_positive)
recall = _safe_ratio(true_positive, true_positive + false_negative)
f1 = (
_safe_ratio(2.0 * precision * recall, precision + recall)
if precision is not None and recall is not None
else None
)
jaccard = _safe_ratio(
true_positive, candidate_cells + reference_cells - true_positive
)
area_factor = (
pixel_area_m2 / 10_000.0
if pixel_area_m2 is not None and pixel_area_m2 > 0
else None
)
return StreamComparison(
candidate_cells=candidate_cells,
reference_cells=reference_cells,
true_positive=true_positive,
false_positive=false_positive,
false_negative=false_negative,
precision=precision,
recall=recall,
f1=f1,
jaccard=jaccard,
candidate_area_ha=(
candidate_cells * area_factor if area_factor is not None else None
),
reference_area_ha=(
reference_cells * area_factor if area_factor is not None else None
),
overlap_area_ha=(
true_positive * area_factor if area_factor is not None else None
),
)
[docs]
def derive_stream_candidates(
source_dem_path: Path,
output_dir: Path,
*,
accumulation_threshold: int = DEFAULT_ACCUMULATION_THRESHOLD,
fill_depressions: bool = True,
reference_path: Path | None = None,
reference_all_touched: bool = False,
generated_utc: str | None = None,
) -> StreamCandidateResult:
"""Derive screening stream candidates from a native DEM.
The workflow fills depressions (optional), routes D8 flow, thresholds flow
accumulation, writes candidate and accumulation rasters, and emits a JSON
review manifest plus a Markdown caveat report. It never reprojects or
resamples the DEM.
"""
if accumulation_threshold < 1:
raise StreamCandidateError("Accumulation threshold must be at least 1.")
grid = read_terrain_grid(source_dem_path)
require_projected_metre_grid(grid.metadata)
warnings: list[str] = []
if fill_depressions:
filled, isolated = fill_dem_depressions(grid.values, grid.valid)
if isolated:
warnings.append(
f"{isolated} valid cells were not connected to the outer boundary "
"and were left unfilled."
)
else:
filled = grid.values.copy()
warnings.append(
"Depression filling was disabled; sinks remain as local minima."
)
_, downstream = derive_d8_flow_direction(
filled,
grid.valid,
resolution_x=grid.metadata.resolution_x,
resolution_y=grid.metadata.resolution_y,
)
accumulation = accumulate_d8_flow(filled, downstream, grid.valid)
candidates = grid.valid & (accumulation >= accumulation_threshold)
if not np.any(candidates):
warnings.append(
"No cells met the accumulation threshold; the candidate raster is empty."
)
pixel_area = abs(
grid.metadata.transform[0] * grid.metadata.transform[4]
- grid.metadata.transform[1] * grid.metadata.transform[3]
)
comparison: StreamComparison | None = None
if reference_path is not None:
reference_mask, reference_warnings = _load_reference_mask(
reference_path,
grid.metadata,
all_touched=reference_all_touched,
)
warnings.extend(reference_warnings)
comparison = compare_stream_candidates(
candidates,
reference_mask,
pixel_area_m2=pixel_area,
)
resolved_output = output_dir.expanduser().resolve()
resolved_output.mkdir(parents=True, exist_ok=True)
stream_path = resolved_output / STREAM_CANDIDATE_RASTER_NAME
accumulation_path = resolved_output / STREAM_ACCUMULATION_RASTER_NAME
stream_values = np.full(candidates.shape, DEFAULT_STREAM_NODATA, dtype=np.uint8)
stream_values[grid.valid] = 0
stream_values[candidates] = 1
_write_uint8_raster(
stream_values,
stream_path,
grid.metadata,
nodata=DEFAULT_STREAM_NODATA,
)
accumulation_values = np.where(
grid.valid,
accumulation,
DEFAULT_ACCUMULATION_NODATA,
).astype(np.float32)
_write_float32_raster(
accumulation_values,
accumulation_path,
grid.metadata,
nodata=DEFAULT_ACCUMULATION_NODATA,
)
candidate_accumulation = accumulation[candidates]
qa: dict[str, object] = {
"valid_cells": int(grid.valid.sum()),
"candidate_cells": int(candidates.sum()),
"candidate_area_ha": float(candidates.sum() * pixel_area / 10_000.0),
"accumulation_min": (
float(np.min(candidate_accumulation))
if candidate_accumulation.size
else None
),
"accumulation_max": (
float(np.max(candidate_accumulation))
if candidate_accumulation.size
else None
),
"accumulation_mean": (
float(np.mean(candidate_accumulation))
if candidate_accumulation.size
else None
),
}
manifest = StreamCandidateManifest(
generated_utc=generated_utc or _now(),
source=grid.metadata,
parameters={
"accumulation_threshold": accumulation_threshold,
"fill_depressions": fill_depressions,
"depression_epsilon": DEFAULT_DEPRESSION_EPSILON,
"flow_routing": "d8",
"d8_codes": {code: label for _, _, code, label in D8_NEIGHBOURS},
"reference_path": str(reference_path) if reference_path else None,
"reference_all_touched": reference_all_touched,
"resampling": "none",
},
outputs={
"stream_candidates": STREAM_CANDIDATE_RASTER_NAME,
"flow_accumulation": STREAM_ACCUMULATION_RASTER_NAME,
},
qa=qa,
comparison=comparison,
warnings=tuple(warnings),
)
manifest_path = write_stream_candidate_manifest(
manifest,
resolved_output / STREAM_CANDIDATE_MANIFEST_NAME,
)
review_path = write_stream_candidate_review(
manifest,
resolved_output / STREAM_CANDIDATE_REVIEW_NAME,
)
return StreamCandidateResult(
manifest=manifest,
output_dir=resolved_output,
stream_path=stream_path,
accumulation_path=accumulation_path,
manifest_path=manifest_path,
review_path=review_path,
)
[docs]
def write_stream_candidate_manifest(
manifest: StreamCandidateManifest,
path: Path,
) -> Path:
"""Write the stream-candidate review manifest as JSON."""
resolved = path.expanduser().resolve()
resolved.parent.mkdir(parents=True, exist_ok=True)
resolved.write_text(
json.dumps(manifest.to_dict(), indent=2, sort_keys=True) + "\n",
encoding="utf-8",
)
return resolved
[docs]
def write_stream_candidate_review(
manifest: StreamCandidateManifest,
path: Path,
) -> Path:
"""Write a human-readable caveat and review report."""
resolved = path.expanduser().resolve()
resolved.parent.mkdir(parents=True, exist_ok=True)
lines = [
"# Terrain-derived stream candidates",
"",
f"Generated: {manifest.generated_utc}",
"",
"## Inputs",
"",
f"- DEM: `{manifest.source.path}`",
f"- CRS: `{manifest.source.crs}`",
f"- Resolution: {manifest.source.resolution_x} x {manifest.source.resolution_y} m",
"",
"## Parameters",
"",
]
for key, value in manifest.parameters.items():
lines.append(f"- {key}: `{value}`")
lines.extend(
[
"",
"## Outputs",
"",
]
)
for key, value in manifest.outputs.items():
lines.append(f"- {key}: `{value}`")
lines.extend(
[
"",
"## QA",
"",
]
)
for key, value in manifest.qa.items():
lines.append(f"- {key}: {value}")
if manifest.comparison is not None:
lines.extend(["", "## Comparison", ""])
for key, value in manifest.comparison.to_dict().items():
lines.append(f"- {key}: {value}")
lines.extend(["", "## Caveats", ""])
for caveat in manifest.caveats:
lines.append(f"- {caveat}")
if manifest.warnings:
lines.extend(["", "## Warnings", ""])
for warning in manifest.warnings:
lines.append(f"- {warning}")
lines.extend(
[
"",
"## Review checklist",
"",
"- [ ] Compare candidates against current public hydrography.",
"- [ ] Check DEM artefacts, culverts, bridges, and cutblocks.",
"- [ ] Confirm the accumulation threshold is appropriate for the AOI.",
"- [ ] Record the reviewer and decision before operational use.",
"",
]
)
resolved.write_text("\n".join(lines), encoding="utf-8")
return resolved
def _load_reference_mask(
reference_path: Path,
metadata: TerrainRasterMetadata,
*,
all_touched: bool,
) -> tuple[np.ndarray, tuple[str, ...]]:
resolved = reference_path.expanduser().resolve()
if resolved.suffix.lower() in {".tif", ".tiff"}:
reference = read_terrain_grid(resolved)
if (
reference.metadata.width != metadata.width
or reference.metadata.height != metadata.height
or reference.metadata.crs != metadata.crs
or reference.metadata.transform != metadata.transform
):
raise StreamCandidateError(
"Reference hydrography raster must share the DEM grid and CRS; "
"resampling is not allowed."
)
return reference.valid & (reference.values > 0), ()
try:
frame = gpd.read_file(resolved)
except (OSError, RuntimeError, TypeError, ValueError) as exc:
raise StreamCandidateError(
f"Unable to read reference hydrography: {resolved}: {exc}"
) from exc
if frame.crs is None:
raise StreamCandidateError("Reference hydrography vector has no CRS.")
if frame.crs.to_string() != metadata.crs:
raise StreamCandidateError(
f"Reference hydrography CRS {frame.crs.to_string()!r} does not match "
f"DEM CRS {metadata.crs!r}; reprojection is not allowed."
)
if frame.empty:
return np.zeros((metadata.height, metadata.width), dtype=bool), (
"Reference hydrography vector contains no features.",
)
mask = rasterio.features.rasterize(
list(frame.geometry),
out_shape=(metadata.height, metadata.width),
transform=rasterio.transform.Affine(*metadata.transform),
fill=0,
default_value=1,
all_touched=all_touched,
dtype="uint8",
)
return mask.astype(bool), ()
def _write_uint8_raster(
values: np.ndarray,
path: Path,
metadata: TerrainRasterMetadata,
*,
nodata: int,
) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
profile = {
"driver": "GTiff",
"dtype": "uint8",
"count": 1,
"width": metadata.width,
"height": metadata.height,
"crs": metadata.crs,
"transform": rasterio.transform.Affine(*metadata.transform),
"nodata": nodata,
"compress": "deflate",
}
with rasterio.open(path, "w", **profile) as destination:
destination.write(values.astype(np.uint8), 1)
def _write_float32_raster(
values: np.ndarray,
path: Path,
metadata: TerrainRasterMetadata,
*,
nodata: float,
) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
profile = {
"driver": "GTiff",
"dtype": "float32",
"count": 1,
"width": metadata.width,
"height": metadata.height,
"crs": metadata.crs,
"transform": rasterio.transform.Affine(*metadata.transform),
"nodata": nodata,
"compress": "deflate",
}
with rasterio.open(path, "w", **profile) as destination:
destination.write(values.astype(np.float32), 1)
def _safe_ratio(numerator: float, denominator: float) -> float | None:
if denominator <= 0:
return None
return float(numerator / denominator)
def _now() -> str:
return datetime.now(UTC).isoformat()