scripts/loaders/miami/derive_peer_groups.ts

// Derived peer groups for Miami-Dade schools (K-nearest neighbours on
// demographics), decision M.9 in docs/cities/miami/00_overview.md.
//
// HARD filters: school level (Elementary / K-8 / Middle / High / 6-12 / K-12 /
// Combination), sector (district-run vs charter), and choice bucket
// (whole-school magnet vs everything else). WEIGHTED distance: Euclidean on
// z-scored features — Black, Hispanic and White shares (FLDOE race file),
// English-learner lower bound (FLDOE hidden counts are not exact zeros), economically disadvantaged (School Grades), log
// October enrollment, all for the same year. K = 40 default; k = 20 and 30 are also derived for the
// optional sensitivity analysis. Outcome-ranking sensitivity to k is not computed here.
//
// Where a school's level|sector|magnet cell holds fewer than 5 other schools
// (whole-school magnets at K-8, 6-12 and K-12; 22 schools in 2024-25), the
// magnet filter is relaxed and the school is matched inside level|sector, with
// the fallback recorded in the group's notes (review 2026-09-16, M12). Schools
// outside the default-comparison universe (ESE centers, alternative, DJJ,
// adult) get no group by decision M.9: a peer set of regular schools would
// mislead more than the missing pill.
//
// Run:  npx tsx scripts/loaders/miami/derive_peer_groups.ts

import crypto from "node:crypto";
import { prisma, runMain } from "./_lib";

const KS = [20, 30, 40] as const;
const DEFAULT_K = 40;
const YEAR = "2024-25";
// metric_key → the year to read it from (latest available per source)
// All features from the same year now that the state race and ELL files are
// loaded for every year (race_membership.ts, ell.ts); 2024-25 is the latest
// year with the School Grades economic-disadvantage share.
const FEATURES: Record<string, string> = {
  pct_black: "2024-25", pct_hispanic: "2024-25", pct_white: "2024-25",
  pct_ell_lower_bound: "2024-25", pct_econ_disadv: "2024-25", enrollment_oct: "2024-25",
};

type S = { schoolNumber: string; level: string; sector: string; choice: string; include: boolean; f: Record<string, number> };

async function main() {
  const schools = await prisma.miamiSchool.findMany({ where: { closedAt: null, activityCode: "A" }, select: { schoolNumber: true, schoolLevel: true, isCharter: true, magnetStatus: true, includeInDefaultComparisons: true } });
  const cells = await prisma.miamiSchoolYearMetric.findMany({ where: { metricKey: { in: Object.keys(FEATURES) }, subgroup: "ALL", value: { not: null }, suppressed: false }, select: { schoolNumber: true, year: true, metricKey: true, value: true } });
  const feat = new Map<string, Record<string, number>>();
  for (const c of cells) {
    if (FEATURES[c.metricKey] !== c.year) continue;
    const m = feat.get(c.schoolNumber) ?? {};
    m[c.metricKey] = c.metricKey === "enrollment_oct" ? Math.log(Math.max(1, c.value!)) : c.value!;
    feat.set(c.schoolNumber, m);
  }
  const keys = Object.keys(FEATURES);
  const working: S[] = schools
    .filter((s) => s.includeInDefaultComparisons && feat.has(s.schoolNumber) && keys.every((k) => feat.get(s.schoolNumber)![k] !== undefined))
    .map((s) => ({ schoolNumber: s.schoolNumber, level: s.schoolLevel ?? "Other", sector: s.isCharter ? "charter" : "district", choice: s.magnetStatus === "S" ? "magnet" : "other", include: s.includeInDefaultComparisons, f: feat.get(s.schoolNumber)! }));
  console.log(`[peers] ${schools.length} active schools; ${working.length} default-included with all ${keys.length} features`);
  // z-score features over the working universe
  const mean: Record<string, number> = {}, sd: Record<string, number> = {};
  for (const k of keys) { const xs = working.map((s) => s.f[k]); mean[k] = xs.reduce((a, b) => a + b, 0) / xs.length; sd[k] = Math.sqrt(xs.reduce((a, b) => a + (b - mean[k]) ** 2, 0) / xs.length) || 1; }
  const dist = (a: S, b: S) => Math.sqrt(keys.reduce((acc, k) => acc + ((a.f[k] - b.f[k]) / sd[k]) ** 2, 0));
  const byKey = new Map<string, S[]>();
  for (const s of working) { const key = `${s.level}|${s.sector}|${s.choice}`; byKey.set(key, [...(byKey.get(key) ?? []), s]); }
  console.log(`[peers] matching universes (level|sector|choice):`);
  for (const [k, v] of [...byKey.entries()].sort()) console.log(`   ${k.padEnd(36)} ${v.length}`);
  const rankIn = (me: S, group: S[]) => group.filter((p) => p.schoolNumber !== me.schoolNumber).map((p) => ({ schoolNumber: p.schoolNumber, d: dist(me, p) })).sort((a, b) => a.d - b.d);
  const byLevelSector = new Map<string, S[]>();
  for (const s of working) { const key = `${s.level}|${s.sector}`; byLevelSector.set(key, [...(byLevelSector.get(key) ?? []), s]); }
  const ranked = new Map<string, { schoolNumber: string; d: number }[]>();
  const fallback = new Set<string>();
  for (const [, group] of byKey) for (const me of group) {
    let top = rankIn(me, group);
    if (top.length < 5) { top = rankIn(me, byLevelSector.get(`${me.level}|${me.sector}`) ?? []); fallback.add(me.schoolNumber); }
    ranked.set(me.schoolNumber, top);
  }
  console.log(`[peers] magnet filter relaxed to level|sector for ${fallback.size} schools in cells with <5 candidates`);

  await prisma.miamiSchool.updateMany({ data: { comparisonGroupId: null } });
  await prisma.miamiComparisonGroup.deleteMany({});
  type G = { id: string; k: number; schoolNumber: string; notes: Record<string, unknown> & { hard_filters: string }; peers: string[] };
  const groupsByK: Record<number, Map<string, G>> = {};
  for (const k of KS) {
    groupsByK[k] = new Map();
    let dropped = 0;
    for (const me of working) {
      const top = (ranked.get(me.schoolNumber) ?? []).slice(0, k);
      if (top.length < 5) { dropped++; continue; }
      const peers = top.map((t) => t.schoolNumber);
      groupsByK[k].set(me.schoolNumber, { id: crypto.randomUUID(), k, schoolNumber: me.schoolNumber, peers, notes: { k, year: YEAR, school_level: me.level, sector: me.sector, choice: me.choice, hard_filters: fallback.has(me.schoolNumber) ? "level|sector (magnet filter relaxed: fewer than 5 candidates in the magnet cell)" : "level|sector|magnet", features: keys, feature_years: FEATURES, standardized: true, peers, max_distance: top[top.length - 1].d } });
    }
    console.log(`[peers] k=${k}: ${groupsByK[k].size} groups (${dropped} schools with <5 candidates dropped)`);
  }
  const all = Object.values(groupsByK).flatMap((m) => [...m.values()]);
  for (let i = 0; i < all.length; i += 500) {
    await prisma.miamiComparisonGroup.createMany({ data: all.slice(i, i + 500).map((g) => ({ id: g.id, year: YEAR, k: g.k, description: `KNN-${g.k} on z-scored demographics; hard filters ${g.notes.hard_filters.replace("|", " + ").replace("|", " + ")}`, methodologyNotes: g.notes as object })) });
  }
  for (const [num, g] of groupsByK[DEFAULT_K]) await prisma.miamiSchool.update({ where: { schoolNumber: num }, data: { comparisonGroupId: g.id } });
  // The first ten neighbours of the k=20 and k=40 prefixes are identical by
  // construction. Comparing them is not evidence of stable outcome rankings.
  console.log(`[peers] linked k=${DEFAULT_K} for ${groupsByK[DEFAULT_K].size} schools; k=20/30 groups stored; outcome-percentile/flag sensitivity to k has not been evaluated`);
}

runMain(main);