docs/cities/miami/probe/ccd_membership_year.py
Served verbatim from the project repository. Internal working document conventions apply: documents may reference file paths, branch names, and findings-ledger anchors from the repo.
import os as _os; _os.chdir(_os.environ.get("MIAMI_PROBE_DIR", "data/cities/miami/sources/_probe"))
import zipfile, hashlib, requests, pandas as pd, sys
url, out = sys.argv[1], sys.argv[2]
path = "ccd/" + url.rsplit("/",1)[-1]
h = hashlib.sha256(); n = 0
with requests.get(url, stream=True, timeout=900) as r, open(path, "wb") as f:
r.raise_for_status()
for chunk in r.iter_content(1 << 20):
f.write(chunk); h.update(chunk); n += len(chunk)
print("downloaded", n, "bytes sha256", h.hexdigest(), flush=True)
z = zipfile.ZipFile(path)
member = [m for m in z.namelist() if m.lower().endswith(".csv")][0]
print("member", member, flush=True)
out_rows = []
with z.open(member) as fh:
for i, chunk in enumerate(pd.read_csv(fh, dtype=str, chunksize=500_000, encoding="latin1", low_memory=False)):
sel = chunk[chunk.ST_LEAID == "FL-13"]
if len(sel): out_rows.append(sel)
m = pd.concat(out_rows)
m.to_csv(out, index=False)
print("MDC membership rows:", len(m))
print("columns:", list(m.columns))
print(m.TOTAL_INDICATOR.value_counts().to_string())