pipeline_miami/verify/reconcile_computed.py

"""(iv) Computed values: recompute every derived closure-track metric from the
served inputs and compare with the served derived value.

Run:  python -m pipeline_miami.verify.reconcile_computed
"""
from __future__ import annotations

import json

from pipeline_miami.verify import common
from pipeline_miami.verify.connectors import served_db


def shift(year: str, k: int) -> str:
    a = int(year[:4]) + k
    return f"{a}-{str((a + 1) % 100).zfill(2)}"


def main() -> None:
    common.ensure_qa_dir()
    enr = served_db.fetch("enrollment_oct")
    kg = served_db.fetch("enrollment_oct_gKG")
    g5 = served_db.fetch("enrollment_oct_g5")
    perm = served_db.fetch("fish_capacity_permanent")
    perm_abc = served_db.fetch("fish_capacity_permanent_abc")
    perm_gis = served_db.fetch("fish_capacity_permanent_gis")
    tfte = served_db.fetch("teacher_fte")
    fish_years = sorted({k[1] for k in perm})
    latest_fish = fish_years[-1] if fish_years else None

    def val(d, k):
        v = d.get(k)
        return v["value"] if v and not v["suppressed"] else None

    checks = {
        "utilization_permanent_pct": [], "enrollment_change_5yr_pct": [], "k_to_g5_ratio": [], "students_per_teacher": [],
    }
    served = {m: served_db.fetch(m) for m in checks}
    for (n, y), e in enr.items():
        ev = val(enr, (n, y))
        if ev is None:
            continue
        cap = val(perm, (n, y))
        if cap is None and latest_fish and y > latest_fish:
            cap = val(perm, (n, latest_fish)) or val(perm_abc, (n, y)) or (val(perm_gis, (n, "2024-25")) if y > "2023-24" else None)
        if cap:
            checks["utilization_permanent_pct"].append(((n, y), ev / cap * 100))
        prev = val(enr, (n, shift(y, -5)))
        if prev:
            checks["enrollment_change_5yr_pct"].append(((n, y), (ev / prev - 1) * 100))
        k, f = val(kg, (n, y)), val(g5, (n, y))
        if k and f:
            checks["k_to_g5_ratio"].append(((n, y), k / f))
        t = val(tfte, (n, y))
        if t:
            checks["students_per_teacher"].append(((n, y), ev / t))
    rows = []
    for m, expected in checks.items():
        s = served[m]
        matches = mism = 0
        examples = []
        for k, x in expected:
            sv = val(s, k)
            if sv is not None and abs(sv - x) <= 1e-6 * max(1, abs(x)):
                matches += 1
            else:
                mism += 1
                if len(examples) < 5:
                    examples.append({"key": k, "expected": x, "served": sv})
        rows.append({"name": m, "expected_total": len(expected), "served_total": len(s), "common_keys": len([1 for k, _ in expected if k in s]), "matches": matches, "mismatches": mism,
                     "agreement_rate": matches / (matches + mism) if (matches + mism) else None, "examples": examples})
    (common.QA_REPORTS_DIR / "__computed_value.json").write_text(json.dumps(rows, indent=2, default=str))
    out = ["# Computed values (iv) — derived metrics recomputed from served inputs\n", "| metric | expected | served | matches | mismatches |", "|---|---:|---:|---:|---:|"]
    out += [f"| `{r['name']}` | {r['expected_total']} | {r['served_total']} | {r['matches']} | {r['mismatches']} |" for r in rows]
    (common.QA_REPORTS_DIR / "__computed_value.md").write_text("\n".join(out))
    for r in rows:
        print(f"  {r['name']:30s} expected={r['expected_total']:>5d} served={r['served_total']:>5d} matches={r['matches']:>5d} mismatches={r['mismatches']}")


if __name__ == "__main__":
    main()