pipeline_philly/verify/completeness.py

"""(i) Completeness: are years or schools missing from served vs source?

Per Philly metric, compare the set of (school, year) cells in the source file
against the set in the served DB. Reports source-only cells (loader skipped),
served-only cells (where did these come from?), and overall coverage.

Output: docs/qa_reports/philly/__completeness.md

Run: python pipeline_philly/verify/completeness.py
"""
from __future__ import annotations

import json
import pathlib
import sys

sys.path.insert(0, str(pathlib.Path(__file__).resolve().parents[2]))

import yaml  # noqa: E402

from pipeline_philly.verify import common  # noqa: E402
from pipeline_philly.verify.connectors import served_db  # noqa: E402
from pipeline_philly.verify.connectors import source_file  # noqa: E402
from pipeline_philly.verify.reconcile import read_source  # noqa: E402


def main() -> None:
    common.ensure_qa_dir()
    with open(common.COMPONENTS_YAML) as f:
        components = yaml.safe_load(f)

    rows: list[dict] = []
    for c in components:
        metric = c["metric_key"]
        subgroup = c["subgroup"]
        cut = c["population_cut"]
        served = served_db.fetch(metric, subgroup=subgroup, population_cut=cut)
        source = read_source(c)
        year_filter = c.get("year")
        if year_filter:
            served = {k: v for k, v in served.items() if k[1] == year_filter}
            source = {k: v for k, v in source.items() if k[1] == year_filter}
        sk, fk = set(served.keys()), set(source.keys())
        rows.append({
            "metric": metric,
            "subgroup": subgroup,
            "population_cut": cut,
            "year": year_filter or "all",
            "served_cells": len(sk),
            "source_cells": len(fk),
            "common_cells": len(sk & fk),
            "served_only_cells": len(sk - fk),
            "source_only_cells": len(fk - sk),
            "served_only_examples": sorted([f"{k[0]} {k[1]}" for k in (sk - fk)])[:10],
            "source_only_examples": sorted([f"{k[0]} {k[1]}" for k in (fk - sk)])[:10],
        })

    out: list[str] = []
    out.append("# Completeness (i) — served DB vs ingested source\n")
    out.append("Per-metric coverage: how many (school, year) cells exist in each layer\n")
    out.append("and how many appear in only one of the two.\n")
    out.append("\n| metric | year | served | source | common | served_only | source_only |")
    out.append("|---|---|---:|---:|---:|---:|---:|")
    for r in rows:
        out.append(
            f"| `{r['metric']}` | {r['year']} | "
            f"{r['served_cells']} | {r['source_cells']} | "
            f"{r['common_cells']} | {r['served_only_cells']} | {r['source_only_cells']} |"
        )
    for r in rows:
        if r["served_only_examples"] or r["source_only_examples"]:
            out.append(f"\n## {r['metric']} — uncommon cells")
            if r["source_only_examples"]:
                out.append("### Source-only (loader didn't ingest)")
                for e in r["source_only_examples"]: out.append(f"- {e}")
            if r["served_only_examples"]:
                out.append("### Served-only (in DB but not in source)")
                for e in r["served_only_examples"]: out.append(f"- {e}")

    (common.QA_REPORTS_DIR / "__completeness.md").write_text("\n".join(out))
    (common.QA_REPORTS_DIR / "__completeness.json").write_text(json.dumps(rows, indent=2))

    print(f"completeness report: {len(rows)} components")
    for r in rows:
        print(
            f"  {r['metric']:38s} served={r['served_cells']:>4d} "
            f"source={r['source_cells']:>4d} common={r['common_cells']:>4d} "
            f"served_only={r['served_only_cells']:>3d} source_only={r['source_only_cells']:>3d}"
        )


if __name__ == "__main__":
    main()