docs/cities/miami/probe/closure_overlap.py

import os as _os; _os.chdir(_os.environ.get("MIAMI_PROBE_DIR", "data/cities/miami/sources/_probe"))
import pandas as pd, json
from scipy.stats import fisher_exact
x = pd.read_csv("ccd/mdc_directory_enrollment_2324.csv", dtype={"ST_SCHID": str})
reg = x[(x.CHARTER_TEXT=="No") & (x.SCH_TYPE_TEXT=="Regular School")].copy()
# Reference lists: board-approved closures/consolidations for 2026-27 (June 17, 2026) and the Aug 25, 2026 "reassessed"/"under review" letters.
# Exact-ish name patterns to avoid partial-name collisions (checked against CCD names).
closed_2026 = ["PARKWAY ELEMENTARY","RAINBOW PARK ELEMENTARY","PHYLLIS WHEATLEY ELEMENTARY","MANDARIN LAKES","ROBERT RUSSA MOTON","LENORA BRAYNON SMITH","MIAMI SPRINGS MIDDLE","PINE VILLA","RICHMOND HEIGHTS MIDDLE"]
review_2026 = ["BRENTWOOD ELEMENTARY","CAROL CITY MIDDLE","GOLDEN GLADES ELEMENTARY","LAKE STEVENS ELEMENTARY","MYRTLE GROVE","LORAH PARK","PAUL LAURENCE DUNBAR","GREENGLADE","HAMMOCKS MIDDLE","KENDALE ELEMENTARY","VILLAGE GREEN","NATURAL BRIDGE","NORTH COUNTY K-8","NORTH DADE MIDDLE","EMERSON ELEMENTARY","LILLIE C. EVANS","LUDLAM ELEMENTARY","KELSEY L. PHARR","ORCHARD VILLA","GLORIA FLOYD","LAMAR LOUISE CURRY","OLYMPIA HEIGHTS","MARJORY STONEMAN DOUGLAS ELEM","RIVIERA MIDDLE","TROPICAL ELEMENTARY","SOUTH MIAMI HEIGHTS"]
# Left out of the reference deliberately: "Center for International Education" (two CCD schools carry that phrase; the flagged one is the Cambridge Associate high school, outside the ES+MS universe).
def flag(name, pats): n = name.upper(); return any(p in n for p in pats)
reg["closed_2026"] = reg.SCH_NAME.apply(lambda s: flag(s, closed_2026))
reg["review_2026"] = reg.SCH_NAME.apply(lambda s: flag(s, review_2026))
reg["flagged"] = reg.closed_2026 | reg.review_2026
uni = reg[(reg.LEVEL.isin(["Elementary","Middle"])) & (reg.enr>0)].copy()
out = {"universe": "district-run regular schools, CCD 2023-24 LEVEL in (Elementary, Middle), enrollment>0", "n": int(len(uni)),
       "closed_matched": int(reg.closed_2026.sum()), "review_matched": int(reg.review_2026.sum()),
       "base_rate": round(float(uni.flagged.mean()),4), "cuts": []}
print("matched closed:", reg.closed_2026.sum(), "review:", reg.review_2026.sum(), "universe:", len(uni), "base rate:", round(uni.flagged.mean(),3))
for lvl, cut in [("Elementary",300),("Elementary",400),("Middle",500),("Middle",700)]:
    u = uni[uni.LEVEL==lvl]; small=u[u.enr<cut]; big=u[u.enr>=cut]
    a,b,c,d = int(small.flagged.sum()), int((~small.flagged).sum()), int(big.flagged.sum()), int((~big.flagged).sum())
    p = fisher_exact([[a,b],[c,d]])[1]
    rec = {"level":lvl,"cut":cut,"n_below":a+b,"flagged_below":a,"p_below":round(a/(a+b),3),"n_at_or_above":c+d,"flagged_at_or_above":c,"p_at_or_above":round(c/max(c+d,1),3),"lift":round((a/(a+b))/max(c/max(c+d,1),1e-9),1),"fisher_p":float(f"{p:.2e}")}
    out["cuts"].append(rec); print(rec)
out["flagged_schools"] = uni[uni.flagged].sort_values("enr")[["ST_SCHID","SCH_NAME","LEVEL","enr","closed_2026","review_2026"]].to_dict("records")
json.dump(out, open("closure_overlap.json","w"), indent=1, default=str)