implementing phase 3 pipeline: gemini structured output, validation, sqlite staging

Prompt is now a template (fixes the hardcoded en/es leftovers in rules
2, 3, 15, 16, 26, 31). pipeline.ts replaces the pseudocode: wordlist
normalization, skip-already-staged idempotency, batches of 20 against
gemini-3.6-flash with responseSchema, raw responses persisted per batch,
per-entry validation with rejection log, one transaction per word into
db/staging.db. Flags: --langs --pos --max-batches --delay-ms --dry-run.

Smoke run: 40/40 words staged (de+es, one batch each), 0 rejections.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

data-pipeline/validate.ts

@@ -0,0 +1,239 @@
+import {
+  DIFFICULTY_LEVELS,
+  NOUN_GENDERS,
+  type DifficultyLevel,
+  type NounGender,
+  type SupportedLanguageCode,
+  type SupportedPos,
+} from "@lila/shared";
+
+export type GeminiTranslation = {
+  target_language: SupportedLanguageCode;
+  word: string;
+  gender: NounGender | null;
+  difficulty: DifficultyLevel;
+};
+
+export type GeminiSense = {
+  sense_index: number;
+  difficulty: DifficultyLevel;
+  definitions: string[];
+  examples: string[];
+  translations: GeminiTranslation[];
+};
+
+export type GeminiWordEntry = {
+  headword: string;
+  language: SupportedLanguageCode;
+  pos: SupportedPos;
+  senses: GeminiSense[];
+};
+
+export type ValidationContext = {
+  sourceLanguage: SupportedLanguageCode;
+  pos: SupportedPos;
+  targetLanguages: readonly SupportedLanguageCode[];
+  inputWords: ReadonlySet<string>;
+};
+
+/**
+ * "empty" is the contract's way of saying "not a valid word of this POS"
+ * (senses: []) — the word is skipped, not rejected.
+ */
+export type ValidationResult =
+  | { status: "valid"; entry: GeminiWordEntry }
+  | { status: "empty"; headword: string }
+  | { status: "invalid"; errors: string[] };
+
+const isRecord = (value: unknown): value is Record<string, unknown> =>
+  typeof value === "object" && value !== null && !Array.isArray(value);
+
+const isNonEmptyString = (value: unknown): value is string =>
+  typeof value === "string" && value.trim() !== "";
+
+const isDifficulty = (value: unknown): value is DifficultyLevel =>
+  (DIFFICULTY_LEVELS as readonly unknown[]).includes(value);
+
+const difficultyRank = (level: DifficultyLevel): number =>
+  DIFFICULTY_LEVELS.indexOf(level);
+
+const validGendersFor = (
+  target: SupportedLanguageCode,
+): readonly (NounGender | null)[] => {
+  if (target === "en") return [null];
+  if (target === "de") return NOUN_GENDERS;
+  return ["masculine", "feminine"];
+};
+
+const checkStringArray = (
+  value: unknown,
+  label: string,
+  errors: string[],
+): void => {
+  if (!Array.isArray(value) || value.length === 0) {
+    errors.push(`${label} must be a non-empty array`);
+    return;
+  }
+  if (!value.every(isNonEmptyString)) {
+    errors.push(`${label} must contain only non-empty strings`);
+  }
+};
+
+const checkTranslation = (
+  raw: unknown,
+  label: string,
+  senseDifficulty: DifficultyLevel | null,
+  ctx: ValidationContext,
+  errors: string[],
+): void => {
+  if (!isRecord(raw)) {
+    errors.push(`${label} must be an object`);
+    return;
+  }
+  const target = raw["target_language"];
+  if (!(ctx.targetLanguages as readonly unknown[]).includes(target)) {
+    errors.push(
+      `${label}: target_language must be one of ${ctx.targetLanguages.join(", ")}`,
+    );
+  }
+  if (!isNonEmptyString(raw["word"])) {
+    errors.push(`${label}: word must be a non-empty string`);
+  }
+  const difficulty = raw["difficulty"];
+  if (!isDifficulty(difficulty)) {
+    errors.push(
+      `${label}: difficulty must be one of ${DIFFICULTY_LEVELS.join(", ")}`,
+    );
+  } else if (
+    senseDifficulty !== null &&
+    difficultyRank(difficulty) < difficultyRank(senseDifficulty)
+  ) {
+    errors.push(
+      `${label}: difficulty "${difficulty}" is lower than sense difficulty "${senseDifficulty}"`,
+    );
+  }
+  if (
+    typeof target === "string" &&
+    (ctx.targetLanguages as readonly string[]).includes(target)
+  ) {
+    const gender = raw["gender"];
+    const allowed = validGendersFor(target as SupportedLanguageCode);
+    if (!(allowed as readonly unknown[]).includes(gender)) {
+      errors.push(
+        `${label}: gender must be ${allowed.map((g) => g ?? "null").join(" or ")} for target "${target}"`,
+      );
+    }
+  }
+};
+
+const checkSense = (
+  raw: unknown,
+  index: number,
+  ctx: ValidationContext,
+  errors: string[],
+): void => {
+  const label = `senses[${index}]`;
+  if (!isRecord(raw)) {
+    errors.push(`${label} must be an object`);
+    return;
+  }
+  if (raw["sense_index"] !== index) {
+    errors.push(`${label}: sense_index must be ${index} (sequential from 0)`);
+  }
+  const senseDifficulty = isDifficulty(raw["difficulty"])
+    ? raw["difficulty"]
+    : null;
+  if (senseDifficulty === null) {
+    errors.push(
+      `${label}: difficulty must be one of ${DIFFICULTY_LEVELS.join(", ")}`,
+    );
+  }
+  checkStringArray(raw["definitions"], `${label}.definitions`, errors);
+  checkStringArray(raw["examples"], `${label}.examples`, errors);
+
+  const translations = raw["translations"];
+  if (!Array.isArray(translations) || translations.length === 0) {
+    errors.push(`${label}.translations must be a non-empty array`);
+    return;
+  }
+  translations.forEach((translation, i) => {
+    checkTranslation(
+      translation,
+      `${label}.translations[${i}]`,
+      senseDifficulty,
+      ctx,
+      errors,
+    );
+  });
+
+  const wordsPerTarget = new Map<string, string[]>();
+  for (const translation of translations) {
+    if (!isRecord(translation)) continue;
+    const target = translation["target_language"];
+    const word = translation["word"];
+    if (typeof target !== "string" || typeof word !== "string") continue;
+    const words = wordsPerTarget.get(target) ?? [];
+    words.push(word);
+    wordsPerTarget.set(target, words);
+  }
+  for (const target of ctx.targetLanguages) {
+    const words = wordsPerTarget.get(target) ?? [];
+    if (words.length === 0) {
+      errors.push(
+        `${label}: missing translation for target language "${target}"`,
+      );
+    }
+    if (words.length > 2) {
+      errors.push(
+        `${label}: more than 2 translations for target language "${target}"`,
+      );
+    }
+    if (new Set(words).size !== words.length) {
+      errors.push(
+        `${label}: duplicate translation word for target language "${target}"`,
+      );
+    }
+  }
+};
+
+export const validateEntry = (
+  raw: unknown,
+  ctx: ValidationContext,
+): ValidationResult => {
+  const errors: string[] = [];
+  if (!isRecord(raw)) {
+    return { status: "invalid", errors: ["entry must be an object"] };
+  }
+
+  const headword = raw["headword"];
+  if (!isNonEmptyString(headword)) {
+    errors.push("headword must be a non-empty string");
+  } else if (!ctx.inputWords.has(headword)) {
+    errors.push(`headword "${headword}" is not in the input batch`);
+  }
+  if (raw["language"] !== ctx.sourceLanguage) {
+    errors.push(`language must be "${ctx.sourceLanguage}"`);
+  }
+  if (raw["pos"] !== ctx.pos) {
+    errors.push(`pos must be "${ctx.pos}"`);
+  }
+
+  const senses = raw["senses"];
+  if (!Array.isArray(senses)) {
+    errors.push("senses must be an array");
+  } else if (senses.length === 0) {
+    if (errors.length === 0 && isNonEmptyString(headword)) {
+      return { status: "empty", headword };
+    }
+  } else {
+    if (senses.length > 3) {
+      errors.push("senses must contain at most 3 entries");
+    }
+    senses.forEach((sense, i) => checkSense(sense, i, ctx, errors));
+  }
+
+  if (errors.length > 0) {
+    return { status: "invalid", errors };
+  }
+  return { status: "valid", entry: raw as GeminiWordEntry };
+};