scripts/loaders/miami/ccd_membership.ts

// Load NCES CCD school membership (long format) for Miami-Dade →
// enrollment_oct_ccd (the federal copy of the October count, used as the
// independent-publisher check). Race/ethnicity shares pct_* are written only
// with CCD_WRITE_RACE_SHARES=1; the canonical shares come from the state file.
//
// Sources: nces_ccd_membership_<YYYY-YY> (200 MB zips, streamed).
// Run:  npx tsx scripts/loaders/miami/ccd_membership.ts [year ...]

import fs from "node:fs";
import {
  requireSource, startDataLoad, finishDataLoad, padSchoolNumber, schoolUniverse, upsertFacts, runMain, SOURCES_ROOT, type Fact,
} from "./_lib";
import { streamZipCsv } from "./_zip";

const SCRIPT_PATH = "scripts/loaders/miami/ccd_membership.ts";
const WRITE_RACE_SHARES = process.env.CCD_WRITE_RACE_SHARES === "1";
const RACE_KEYS: Record<string, string> = {
  "Black or African American": "pct_black",
  "Hispanic/Latino": "pct_hispanic",
  "White": "pct_white",
  "Asian": "pct_asian",
  "Two or more races": "pct_two_or_more",
  "American Indian or Alaska Native": "pct_amer_indian",
  "Native Hawaiian or Other Pacific Islander": "pct_hawaiian_pac_isl",
};

function fetchedYears(): string[] {
  return fs.readdirSync(SOURCES_ROOT)
    .filter((d) => /^nces_ccd_membership_\d{4}-\d{2}$/.test(d))
    .map((d) => d.replace("nces_ccd_membership_", ""))
    .sort();
}

async function loadYear(year: string, universe: Set<string>) {
  const sourceId = `nces_ccd_membership_${year}`;
  const { filePath, prov } = requireSource(sourceId);
  console.log(`[ccd ${year}] streaming ${filePath} …`);
  const rows = await streamZipCsv(filePath, /ccd_sch_052_.*\.csv$/i, (r) => r.ST_LEAID === "FL-13");
  console.log(`[ccd ${year}] FL-13 rows: ${rows.length}`);
  const totals = new Map<string, number>();
  const race = new Map<string, Map<string, number>>();
  for (const r of rows) {
    const num = padSchoolNumber(String(r.ST_SCHID).replace("FL-13-", ""));
    if (!num || !universe.has(num)) continue;
    const n = Number.parseFloat(r.STUDENT_COUNT);
    if (!Number.isFinite(n)) continue;
    if (r.TOTAL_INDICATOR === "Education Unit Total") totals.set(num, n);
    else if (r.TOTAL_INDICATOR.startsWith("Derived - Subtotal by Race/Ethnicity and Sex")) {
      const m = race.get(num) ?? new Map<string, number>();
      m.set(r.RACE_ETHNICITY, (m.get(r.RACE_ETHNICITY) ?? 0) + n);
      race.set(num, m);
    }
  }
  const facts: Fact[] = [];
  for (const [num, t] of totals) {
    facts.push({ schoolNumber: num, year, metricKey: "enrollment_oct_ccd", value: t, suppressed: false });
    // Race shares (pct_*) come from the same-year FLDOE race file
    // (race_membership.ts); the CCD copy is read by the harness as the
    // unsuppressed cross-publisher reference (cross_publisher_race.py).
    if (!WRITE_RACE_SHARES) continue;
    const m = race.get(num);
    if (!m) continue;
    const denom = [...m.values()].reduce((a, b) => a + b, 0);
    if (denom <= 0) continue;
    for (const [label, key] of Object.entries(RACE_KEYS)) {
      facts.push({ schoolNumber: num, year, metricKey: key, value: ((m.get(label) ?? 0) / denom) * 100, suppressed: false, denominator: denom });
    }
  }
  const load = await startDataLoad({ sourceId, sourceUrl: prov.url, sha256: prov.sha256, bytes: prov.bytes, yearCovered: year, scriptPath: SCRIPT_PATH });
  const res = await upsertFacts(facts, load.id);
  await finishDataLoad(load.id, { inserted: res.written, updated: 0, suppressed: 0 }, `${totals.size} schools with totals; ${race.size} with race subtotals`);
  console.log(`[ccd ${year}] ${res.written} facts for ${totals.size} schools`);
}

async function main() {
  const years = process.argv.slice(2).length ? process.argv.slice(2) : fetchedYears();
  const universe = await schoolUniverse();
  for (const y of years) await loadYear(y, universe);
}

runMain(main);