scripts/loaders/miami/_zip.ts

// Stream one member of a zip through a CSV parser, keeping only rows that
// pass `keep`. Used for the NCES CCD files (a 200 MB zip expands to a ~1.5 GB
// CSV; we only want the Miami-Dade rows). Relies on the system `unzip`.

import { spawn } from "node:child_process";
import { parse } from "csv-parse";

export async function streamZipCsv<T = Record<string, string>>(
  zipPath: string,
  memberPattern: RegExp,
  keep: (row: Record<string, string>) => boolean,
): Promise<T[]> {
  const list = spawn("unzip", ["-Z1", zipPath]);
  let names = "";
  for await (const chunk of list.stdout) names += chunk;
  await new Promise((res) => list.on("close", res));
  const member = names.split("\n").find((n) => memberPattern.test(n));
  if (!member) throw new Error(`no zip member matching ${memberPattern} in ${zipPath}`);

  const proc = spawn("unzip", ["-p", zipPath, member]);
  const parser = parse({ columns: true, bom: true, relax_column_count: true, skip_empty_lines: true });
  const out: T[] = [];
  proc.stdout.pipe(parser);
  for await (const row of parser as AsyncIterable<Record<string, string>>) {
    if (keep(row)) out.push(row as unknown as T);
  }
  return out;
}