/**
 * The TypeScript implementation's side of the attendance contract.
 *
 * Like `retention-emit.ts`, this computes nothing of its own: every figure comes
 * from the `src/` modules the published attendance page is built from, written
 * out in the shared format so the R Markdown and Python implementations have
 * something to be compared against.
 *
 * Run: npm run calc:ts   (writes calc/out/attendance-typescript.tsv)
 */

import { createHash } from 'node:crypto'
import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs'
import { join } from 'node:path'

import { int, real, serialize, text, type Row } from '../src/contract.ts'
import { parseCsv } from '../src/csv.ts'
import {
  chronicFromAttending90,
  chronicFromNyc,
  federalRate,
  normalizeUlcs,
  seriesFromRows,
  type NycRow,
  type YearValue,
} from '../src/attendance.ts'
import { K8_SCHOOLS, NYC_SCHOOLS } from '../src/schools.ts'
import { median, pearson } from '../src/stats.ts'

const ROOT = join(import.meta.dirname, '..')
const A = join(ROOT, 'attendance')

const sha256 = (s: string): string => createHash('sha256').update(s, 'utf8').digest('hex')

/** Cities become contract keys, so their spaces have to go. */
const cityToken = (city: string): string => city.replace(/ /g, '_')

/**
 * Sources, with the short name each is known by in the contract and the columns
 * every implementation reads out of it.
 *
 * The digest is over those columns only. Hashing the whole row would make the
 * check fail for a column nothing uses, which is noise rather than signal.
 */
const CSV_SOURCES: { name: string; file: string; columns: string[] }[] = [
  { name: 'ada_archive', file: 'ex/Student_ADA_Yearly.csv', columns: ['School Year', 'ULCS Code', 'Average Daily Attendance'] },
  { name: 'ada_current', file: 'Student_ADA_School.csv', columns: ['School Year', 'ULCS Code', 'Average Daily Attendance (YTD)'] },
  { name: 'attending90', file: 'Student_Attendance_90_School.csv', columns: ['School Year', 'ULCS Code', 'Category', '% with 90%+ Attendance (Yearly)'] },
  { name: 'attending95_archive', file: 'ex/SDP_95_Attendance_School_S.csv', columns: ['School Year', 'ULCS Code', 'Category', '% with 95%+ Attendance (Yearly)'] },
  { name: 'attending95_current', file: 'Student_Attendance_95_School.csv', columns: ['School Year', 'ULCS Code', 'Category', '% with 95%+ Attendance (Yearly)'] },
  {
    name: 'teacher_ada',
    file: 'ex/Archive_Teacher_95_ADA_2012-13_2022-23/OD_Teacher_Attendance_Schools_Suppressed_20240328.csv',
    columns: ['school_year', 'ulcscode', 'pct_att_ada'],
  },
  { name: 'seda_covariates', file: '../seda/seda_cov_school_pool_4.1.csv', columns: ['sedasch', 'sedalea', 'schnam'] },
]

const NYC_YEARS = ['2016-17', '2017-18', '2018-19', '2019-20', '2020-21']
const CRDC_LEAS = ['4218990', '0103360', '3600091', '3600092']
/** 2017-18 is the last clean pre-pandemic collection; 2022-23 the most recent. */
const CRDC_YEARS = ['2017', '2022']
const TUSCALOOSA_SEDA_LEA = '103360'

const csv = (rel: string): Record<string, string>[] => {
  const path = join(A, rel)
  if (!existsSync(path)) throw new Error(`missing ${path} — run ./fetch-attendance.sh`)
  return parseCsv(readFileSync(path, 'latin1'))
}

const allStudents = (r: Record<string, string>): boolean => r['Category'] === 'All Students'

/** Districtwide reference line for one measure: the median across every school SDP reports. */
function districtMedians(src: Map<string, YearValue>): Map<string, number[]> {
  const byYear = new Map<string, number[]>()
  for (const years of src.values()) {
    for (const [y, v] of Object.entries(years)) {
      const bucket = byYear.get(y)
      if (bucket) bucket.push(v)
      else byYear.set(y, [v])
    }
  }
  return byYear
}

export function emitRows(): Row[] {
  const rows: Row[] = []
  const add = (metric: string, key: string, value: string): void => {
    rows.push({ metric, key, value })
  }

  // ------------------------------------------------------------------ inputs
  for (const src of CSV_SOURCES) {
    const parsed = csv(src.file)
    add('input.rows', src.name, int(parsed.length))
    add('input.digest', src.name, text(sha256(parsed.map((r) => src.columns.map((c) => r[c] ?? '').join('\x1f')).join('\n'))))
  }

  // The JSON sources are hashed as raw bytes rather than as parsed columns:
  // there is no cross-language question about how to read a JSON string, and
  // what each implementation makes of the numbers is checked by the facts below.
  const jsonFiles = [
    ...NYC_YEARS.map((y) => ({ name: `nyc_${y}`, file: `nyc_attendance_${y}.json` })),
    ...CRDC_YEARS.flatMap((y) =>
      CRDC_LEAS.flatMap((lea) => [
        { name: `crdc_absent_${y}_${lea}`, file: `crdc_absent_${y}_${lea}.json` },
        { name: `crdc_enrolled_${y}_${lea}`, file: `crdc_enrolled_${y}_${lea}.json` },
      ]),
    ),
  ]
  for (const j of jsonFiles) {
    const path = join(A, j.file)
    if (!existsSync(path)) throw new Error(`missing ${path} — run ./fetch-attendance.sh and ./fetch-crdc.sh`)
    add('input.filedigest', j.name, text(sha256(readFileSync(path, 'utf8'))))
  }

  // ------------------------------------------------------------ Philadelphia
  const adaOld = seriesFromRows(csv('ex/Student_ADA_Yearly.csv'), 'School Year', 'ULCS Code', 'Average Daily Attendance')
  const adaNew = seriesFromRows(csv('Student_ADA_School.csv'), 'School Year', 'ULCS Code', 'Average Daily Attendance (YTD)')
  const phlAda = new Map(adaOld)
  for (const [k, v] of adaNew) phlAda.set(k, { ...(phlAda.get(k) ?? {}), ...v })

  const phlAttending90 = seriesFromRows(
    csv('Student_Attendance_90_School.csv'), 'School Year', 'ULCS Code', '% with 90%+ Attendance (Yearly)', allStudents,
  )
  const phlChronic = new Map(
    [...phlAttending90].map(([code, years]): [string, YearValue] => [
      code,
      Object.fromEntries(Object.entries(years).map(([y, v]) => [y, chronicFromAttending90(v)])),
    ]),
  )

  const a95old = seriesFromRows(csv('ex/SDP_95_Attendance_School_S.csv'), 'School Year', 'ULCS Code', '% with 95%+ Attendance (Yearly)', allStudents)
  const a95new = seriesFromRows(csv('Student_Attendance_95_School.csv'), 'School Year', 'ULCS Code', '% with 95%+ Attendance (Yearly)', allStudents)
  const phl95 = new Map(a95old)
  for (const [k, v] of a95new) phl95.set(k, { ...(phl95.get(k) ?? {}), ...v })

  const teacherAda = seriesFromRows(
    csv('ex/Archive_Teacher_95_ADA_2012-13_2022-23/OD_Teacher_Attendance_Schools_Suppressed_20240328.csv'),
    'school_year', 'ulcscode', 'pct_att_ada',
  )

  const series: [string, Map<string, YearValue>][] = [
    ['ada', phlAda],
    ['chronic', phlChronic],
    ['a95', phl95],
    ['teacherAda', teacherAda],
  ]

  for (const s of K8_SCHOOLS) {
    const code = normalizeUlcs(s.org)
    for (const [name, src] of series) {
      for (const [year, value] of Object.entries(src.get(code) ?? {})) {
        add(`phl.${name}`, `${s.org}|${year}`, real(value))
      }
    }
  }

  for (const [name, src] of series) {
    if (name === 'teacherAda') continue
    for (const [year, values] of districtMedians(src)) {
      add(`phlMedian.${name}`, year, real(median(values)))
      add(`phlCount.${name}`, year, int(values.length))
    }
  }

  // ---------------------------------------------------------------- New York
  const nycByYear = new Map<string, ReturnType<typeof chronicFromNyc>>()
  for (const y of NYC_YEARS) {
    nycByYear.set(y, chronicFromNyc(JSON.parse(readFileSync(join(A, `nyc_attendance_${y}.json`), 'utf8')) as NycRow[]))
  }

  for (const s of NYC_SCHOOLS) {
    for (const y of NYC_YEARS) {
      const hit = nycByYear.get(y)?.get(s.dbn)
      if (hit === undefined) continue
      add('nyc.ada', `${s.dbn}|${y}`, real(hit.ada))
      add('nyc.chronic', `${s.dbn}|${y}`, real(hit.chronic))
    }
  }

  for (const y of NYC_YEARS) {
    const all = [...(nycByYear.get(y)?.values() ?? [])]
    add('nycMedian.ada', y, real(median(all.map((r) => r.ada))))
    add('nycMedian.chronic', y, real(median(all.map((r) => r.chronic))))
    add('nycMedian.n', y, int(all.length))
  }

  // ----------------------------------------------- federal, all three cities
  type CrdcRow = { ncessch?: string; students_chronically_absent?: number; enrollment?: number; homeless?: number }
  const readCrdc = (file: string): CrdcRow[] =>
    (JSON.parse(readFileSync(join(A, file), 'utf8')) as { results: CrdcRow[] }).results ?? []

  const targets = new Map<string, { city: string; label: string }>()
  for (const s of K8_SCHOOLS) if (s.nces) targets.set(s.nces, { city: 'Philadelphia', label: s.label })
  for (const s of NYC_SCHOOLS) targets.set(s.nces, { city: 'Brooklyn', label: s.label })
  // SEDA drops Alabama's leading zero from the NCES id; CRDC keeps all twelve.
  for (const r of csv('../seda/seda_cov_school_pool_4.1.csv')) {
    if (r['sedalea'] !== TUSCALOOSA_SEDA_LEA) continue
    const id = r['sedasch']
    if (!id) continue
    targets.set(id.length === 11 ? `0${id}` : id, { city: 'Tuscaloosa City', label: (r['schnam'] ?? id).trim() })
  }

  for (const [nces, meta] of targets) {
    add('federal.city', nces, text(cityToken(meta.city)))
    add('federal.label', nces, text(meta.label))
  }

  for (const year of CRDC_YEARS) {
    const crdcAbsent = new Map<string, number>()
    const crdcEnrolled = new Map<string, number>()
    for (const lea of CRDC_LEAS) {
      for (const r of readCrdc(`crdc_absent_${year}_${lea}.json`)) {
        // CRDC returns two rows per school even with every other subgroup
        // pinned to 99: one for all students and one for the homeless subgroup.
        if (r.homeless !== undefined && r.homeless !== 99) continue
        if (r.ncessch && typeof r.students_chronically_absent === 'number') {
          crdcAbsent.set(r.ncessch, r.students_chronically_absent)
        }
      }
      for (const r of readCrdc(`crdc_enrolled_${year}_${lea}.json`)) {
        if (r.ncessch && typeof r.enrollment === 'number') crdcEnrolled.set(r.ncessch, r.enrollment)
      }
    }

    const byCity = new Map<string, number[]>()
    for (const [nces, meta] of targets) {
      const rate = federalRate({ absent: crdcAbsent.get(nces) ?? null, enrolled: crdcEnrolled.get(nces) ?? null })
      if (rate === null) continue
      add('federal.rate', `${year}|${nces}`, real(rate))
      const bucket = byCity.get(meta.city)
      if (bucket) bucket.push(rate)
      else byCity.set(meta.city, [rate])
    }

    for (const [city, rates] of byCity) {
      const key = `${year}|${cityToken(city)}`
      add('federalMedian.median', key, real(median(rates)))
      add('federalMedian.min', key, real(Math.min(...rates)))
      add('federalMedian.max', key, real(Math.max(...rates)))
      add('federalMedian.n', key, int(rates.length))
    }
  }

  // ------------------------------------------------------------ correlation
  const withBoth: { org: string; chronic: number; teacher: number }[] = []
  for (const s of K8_SCHOOLS) {
    const code = normalizeUlcs(s.org)
    const chronic = (phlChronic.get(code) ?? {})['2024-2025']
    // Year keys are compared as strings, exactly as the original does; they are
    // all ASCII digits, so every language orders them the same way.
    const teacher = median(Object.entries(teacherAda.get(code) ?? {}).filter(([y]) => y >= '2016').map(([, v]) => v))
    if (chronic === undefined || teacher === null) continue
    withBoth.push({ org: s.org, chronic, teacher })
    add('correlation.chronic', s.org, real(chronic))
    add('correlation.teacher', s.org, real(teacher))
  }
  add('correlation.n', '-', int(withBoth.length))
  add(
    'correlation.chronicVsTeacherAttendance',
    '-',
    real(pearson(withBoth.map((s) => s.chronic), withBoth.map((s) => s.teacher))),
  )

  return rows
}

if (import.meta.filename === process.argv[1]) {
  const outDir = join(ROOT, 'calc', 'out')
  mkdirSync(outDir, { recursive: true })
  const tsv = serialize(emitRows())
  writeFileSync(join(outDir, 'attendance-typescript.tsv'), tsv)
  console.log(`wrote calc/out/attendance-typescript.tsv (${tsv.split('\n').length - 2} facts)`)
}