"""(iv) Computed values: recompute every derived closure-track metric from the served inputs and compare with the served derived value. Run: python -m pipeline_miami.verify.reconcile_computed """ from __future__ import annotations import json from pipeline_miami.verify import common from pipeline_miami.verify.connectors import served_db def shift(year: str, k: int) -> str: a = int(year[:4]) + k return f"{a}-{str((a + 1) % 100).zfill(2)}" def main() -> None: common.ensure_qa_dir() enr = served_db.fetch("enrollment_oct") kg = served_db.fetch("enrollment_oct_gKG") g5 = served_db.fetch("enrollment_oct_g5") perm = served_db.fetch("fish_capacity_permanent") perm_abc = served_db.fetch("fish_capacity_permanent_abc") perm_gis = served_db.fetch("fish_capacity_permanent_gis") tfte = served_db.fetch("teacher_fte") fish_years = sorted({k[1] for k in perm}) latest_fish = fish_years[-1] if fish_years else None def val(d, k): v = d.get(k) return v["value"] if v and not v["suppressed"] else None checks = { "utilization_permanent_pct": [], "enrollment_change_5yr_pct": [], "k_to_g5_ratio": [], "students_per_teacher": [], } served = {m: served_db.fetch(m) for m in checks} for (n, y), e in enr.items(): ev = val(enr, (n, y)) if ev is None: continue cap = val(perm, (n, y)) if cap is None and latest_fish and y > latest_fish: cap = val(perm, (n, latest_fish)) or val(perm_abc, (n, y)) or (val(perm_gis, (n, "2024-25")) if y > "2023-24" else None) if cap: checks["utilization_permanent_pct"].append(((n, y), ev / cap * 100)) prev = val(enr, (n, shift(y, -5))) if prev: checks["enrollment_change_5yr_pct"].append(((n, y), (ev / prev - 1) * 100)) k, f = val(kg, (n, y)), val(g5, (n, y)) if k and f: checks["k_to_g5_ratio"].append(((n, y), k / f)) t = val(tfte, (n, y)) if t: checks["students_per_teacher"].append(((n, y), ev / t)) rows = [] for m, expected in checks.items(): s = served[m] matches = mism = 0 examples = [] for k, x in expected: sv = val(s, k) if sv is not None and abs(sv - x) <= 1e-6 * max(1, abs(x)): matches += 1 else: mism += 1 if len(examples) < 5: examples.append({"key": k, "expected": x, "served": sv}) rows.append({"name": m, "expected_total": len(expected), "served_total": len(s), "common_keys": len([1 for k, _ in expected if k in s]), "matches": matches, "mismatches": mism, "agreement_rate": matches / (matches + mism) if (matches + mism) else None, "examples": examples}) (common.QA_REPORTS_DIR / "__computed_value.json").write_text(json.dumps(rows, indent=2, default=str)) out = ["# Computed values (iv) — derived metrics recomputed from served inputs\n", "| metric | expected | served | matches | mismatches |", "|---|---:|---:|---:|---:|"] out += [f"| `{r['name']}` | {r['expected_total']} | {r['served_total']} | {r['matches']} | {r['mismatches']} |" for r in rows] (common.QA_REPORTS_DIR / "__computed_value.md").write_text("\n".join(out)) for r in rows: print(f" {r['name']:30s} expected={r['expected_total']:>5d} served={r['served_total']:>5d} matches={r['matches']:>5d} mismatches={r['mismatches']}") if __name__ == "__main__": main()