lila/data-pipeline/replay.ts

/**
 * Re-validates saved Gemini responses from `responses/` without calling the API.
 *
 * Every batch response is written to disk before it is parsed, so a change to
 * the validation rules can be applied retroactively to everything already
 * generated. Use this after editing `validate.ts` to see what the change would
 * do, and to recover entries that the old rules rejected.
 *
 * Reports by default; pass --write to stage recovered entries into staging.db.
 * Staging is idempotent, so re-running is safe.
 */
import { readdirSync, readFileSync } from "node:fs";
import path from "node:path";
import { parseArgs } from "node:util";
import {
  SUPPORTED_LANGUAGE_CODES,
  SUPPORTED_POS,
  type SupportedLanguageCode,
  type SupportedPos,
} from "@lila/shared";
import { parseEntries } from "./gemini.js";
import { openStaging, stageEntry, countStagedRows } from "./staging.js";
import { validateEntry, type ValidationContext } from "./validate.js";

const ROOT = import.meta.dirname;
const RESPONSES_DIR = path.join(ROOT, "responses");
const STAGING_DB_PATH = path.join(ROOT, "db", "staging.db");
const STAGING_SCHEMA_PATH = path.join(ROOT, "db", "schema.sql");

type SavedResponse = {
  sourceLanguage: SupportedLanguageCode;
  pos: SupportedPos;
  targetLanguages: SupportedLanguageCode[];
  words: string[];
  rawText: string;
};

type Tally = {
  files: number;
  unreadable: number;
  entries: number;
  valid: number;
  normalized: number;
  empty: number;
  invalid: number;
  staged: number;
  alreadyStaged: number;
};

const isRecord = (value: unknown): value is Record<string, unknown> =>
  typeof value === "object" && value !== null && !Array.isArray(value);

const isStringArray = (value: unknown): value is string[] =>
  Array.isArray(value) && value.every((item) => typeof item === "string");

/** Saved files are trusted-but-verified: they are ours, but they are on disk. */
const parseSavedResponse = (raw: unknown): SavedResponse | null => {
  if (!isRecord(raw)) return null;
  const { sourceLanguage, pos, targetLanguages, words, rawText } = raw;
  if (
    typeof sourceLanguage !== "string" ||
    !(SUPPORTED_LANGUAGE_CODES as readonly string[]).includes(sourceLanguage) ||
    typeof pos !== "string" ||
    !(SUPPORTED_POS as readonly string[]).includes(pos) ||
    !isStringArray(targetLanguages) ||
    !isStringArray(words) ||
    typeof rawText !== "string"
  ) {
    return null;
  }
  return {
    sourceLanguage: sourceLanguage as SupportedLanguageCode,
    pos: pos as SupportedPos,
    targetLanguages: targetLanguages as SupportedLanguageCode[],
    words,
    rawText,
  };
};

const main = (): void => {
  const { values } = parseArgs({
    args:
      process.argv[2] === "--" ? process.argv.slice(3) : process.argv.slice(2),
    options: {
      write: { type: "boolean", default: false },
      langs: { type: "string" },
      verbose: { type: "boolean", default: false },
    },
  });

  const langFilter =
    values.langs === undefined
      ? null
      : new Set(values.langs.split(",").map((code) => code.trim()));

  let files: string[];
  try {
    files = readdirSync(RESPONSES_DIR)
      .filter((name) => name.endsWith(".json"))
      .sort();
  } catch {
    console.error(`no responses directory at ${RESPONSES_DIR}`);
    process.exitCode = 1;
    return;
  }

  if (files.length === 0) {
    console.log("no saved responses to replay");
    return;
  }

  const db = values.write
    ? openStaging(STAGING_DB_PATH, STAGING_SCHEMA_PATH)
    : null;

  const tally: Tally = {
    files: 0,
    unreadable: 0,
    entries: 0,
    valid: 0,
    normalized: 0,
    empty: 0,
    invalid: 0,
    staged: 0,
    alreadyStaged: 0,
  };
  const recovered: string[] = [];
  const stillInvalid = new Map<string, string[]>();

  for (const name of files) {
    let saved: SavedResponse | null;
    try {
      saved = parseSavedResponse(
        JSON.parse(readFileSync(path.join(RESPONSES_DIR, name), "utf-8")),
      );
    } catch {
      saved = null;
    }
    if (saved === null) {
      tally.unreadable++;
      console.warn(`  skipping unreadable response file: ${name}`);
      continue;
    }
    if (langFilter !== null && !langFilter.has(saved.sourceLanguage)) continue;
    tally.files++;

    let entries: unknown[];
    try {
      entries = parseEntries(saved.rawText);
    } catch (error) {
      tally.unreadable++;
      console.warn(
        `  skipping ${name}: ${error instanceof Error ? error.message : String(error)}`,
      );
      continue;
    }

    const ctx: ValidationContext = {
      sourceLanguage: saved.sourceLanguage,
      pos: saved.pos,
      targetLanguages: saved.targetLanguages,
      inputWords: new Set(saved.words),
    };

    for (const entry of entries) {
      tally.entries++;
      const result = validateEntry(entry, ctx);
      if (result.status === "valid") {
        tally.valid++;
        if (result.normalizations.length > 0) {
          tally.normalized++;
          recovered.push(
            `${saved.sourceLanguage}/${result.entry.headword}: ${result.normalizations.join("; ")}`,
          );
        }
        if (db !== null) {
          const outcome = stageEntry(db, result.entry);
          if (outcome === "staged") tally.staged++;
          else tally.alreadyStaged++;
        }
      } else if (result.status === "empty") {
        tally.empty++;
      } else {
        tally.invalid++;
        const key = result.errors.join(" | ");
        stillInvalid.set(key, [...(stillInvalid.get(key) ?? []), name]);
      }
    }
  }

  if (values.verbose && recovered.length > 0) {
    console.log("\n— normalized entries —");
    for (const line of recovered) console.log(`  ${line}`);
  }

  if (stillInvalid.size > 0) {
    console.log("\n— still invalid —");
    for (const [errors, sources] of stillInvalid) {
      console.log(`  (${sources.length}×) ${errors}`);
    }
  }

  console.log("\n— replay summary —");
  console.log(`  response files replayed: ${tally.files}`);
  if (tally.unreadable > 0)
    console.log(`  unreadable files: ${tally.unreadable}`);
  console.log(`  entries seen:            ${tally.entries}`);
  console.log(`  valid:                   ${tally.valid}`);
  console.log(`    of which normalized:   ${tally.normalized}`);
  console.log(`  empty (skipped):         ${tally.empty}`);
  console.log(`  invalid:                 ${tally.invalid}`);

  if (db !== null) {
    console.log(`  newly staged:            ${tally.staged}`);
    console.log(`  already staged:          ${tally.alreadyStaged}`);
    const totals = countStagedRows(db);
    console.log(
      `  staging.db totals: ${totals.words} words, ${totals.senses} senses, ${totals.translations} translations`,
    );
    db.close();
  } else {
    console.log("\n  (report only — pass --write to stage recovered entries)");
  }
};

main();