pipeline_miami/verify/completeness.py

"""(i) Completeness: for each membership year, is every active district-13
school in the MSID directory served, and is every served cell in the source?

Run:  python -m pipeline_miami.verify.completeness
"""
from __future__ import annotations

import json

from pipeline_miami.verify import common
from pipeline_miami.verify.connectors import served_db, source_file

YEARS = ["2016-17", "2017-18", "2018-19", "2019-20", "2020-21", "2021-22", "2022-23", "2023-24", "2024-25", "2025-26"]


def main() -> None:
    common.ensure_qa_dir()
    served = served_db.fetch("enrollment_oct")
    directory = served_db.directory()
    rows = []
    for y in YEARS:
        source = source_file.membership_total(y)
        sk = {k for k in served if k[1] == y}
        fk = set(source)
        source_only = fk - sk
        rows.append({
            "metric": "enrollment_oct", "year": y, "served_cells": len(sk), "source_cells": len(fk), "common_cells": len(sk & fk),
            "served_only_cells": len(sk - fk), "source_only_cells": len(source_only),
            "source_only_in_directory": sum(1 for k in source_only if k[0] in directory),
            "source_only_examples": sorted(f"{k[0]}" for k in source_only)[:10],
        })
    # Directory coverage for the latest year: active REGULAR schools without a served count.
    latest = YEARS[-1]
    have = {k[0] for k in served if k[1] == latest}
    missing = sorted(n for n, d in directory.items() if d["activity_code"] == "A" and d["school_type"] == "REGULAR" and n not in have)
    summary = {"rows": rows, "latest_year": latest, "active_regular_without_enrollment": missing}
    (common.QA_REPORTS_DIR / "__completeness.json").write_text(json.dumps(summary, indent=2))
    out = ["# Completeness (i) — served DB vs ingested source (enrollment_oct)\n", "| year | served | source | common | served_only | source_only |", "|---|---:|---:|---:|---:|---:|"]
    out += [f"| {r['year']} | {r['served_cells']} | {r['source_cells']} | {r['common_cells']} | {r['served_only_cells']} | {r['source_only_cells']} |" for r in rows]
    out.append(f"\nActive district-run schools in MSID without a {latest} count: {len(missing)}")
    out += [f"- {n} {directory[n]['name']}" for n in missing[:40]]
    (common.QA_REPORTS_DIR / "__completeness.md").write_text("\n".join(out))
    for r in rows:
        print(f"  {r['year']} served={r['served_cells']} source={r['source_cells']} common={r['common_cells']} served_only={r['served_only_cells']} source_only={r['source_only_cells']}")
    print(f"  active REGULAR without {latest} enrollment: {len(missing)}")


if __name__ == "__main__":
    main()