implementing phase 3 pipeline: gemini structured output, validation, sqlite staging

Prompt is now a template (fixes the hardcoded en/es leftovers in rules
2, 3, 15, 16, 26, 31). pipeline.ts replaces the pseudocode: wordlist
normalization, skip-already-staged idempotency, batches of 20 against
gemini-3.6-flash with responseSchema, raw responses persisted per batch,
per-entry validation with rejection log, one transaction per word into
db/staging.db. Flags: --langs --pos --max-batches --delay-ms --dry-run.

Smoke run: 40/40 words staged (de+es, one batch each), 0 rejections.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

data-pipeline/tests/validate.test.ts

@@ -0,0 +1,285 @@
+import { describe, it, expect } from "vitest";
+import { validateEntry, type ValidationContext } from "../validate.js";
+
+const ctx: ValidationContext = {
+  sourceLanguage: "es",
+  pos: "noun",
+  targetLanguages: ["en", "it", "de", "fr"],
+  inputWords: new Set(["casa", "banco"]),
+};
+
+type Translation = {
+  target_language: string;
+  word: string;
+  gender: string | null;
+  difficulty: string;
+};
+
+const translations = (): Translation[] => [
+  { target_language: "en", word: "house", gender: null, difficulty: "easy" },
+  {
+    target_language: "it",
+    word: "casa",
+    gender: "feminine",
+    difficulty: "easy",
+  },
+  { target_language: "de", word: "Haus", gender: "neuter", difficulty: "easy" },
+  {
+    target_language: "fr",
+    word: "maison",
+    gender: "feminine",
+    difficulty: "easy",
+  },
+];
+
+const sense = (
+  overrides: Record<string, unknown> = {},
+): Record<string, unknown> => ({
+  sense_index: 0,
+  difficulty: "easy",
+  definitions: ["Un edificio para vivir."],
+  examples: ["Compraron una casa en la ciudad."],
+  translations: translations(),
+  ...overrides,
+});
+
+const entry = (
+  overrides: Record<string, unknown> = {},
+): Record<string, unknown> => ({
+  headword: "casa",
+  language: "es",
+  pos: "noun",
+  senses: [sense()],
+  ...overrides,
+});
+
+const errorsOf = (raw: unknown): string[] => {
+  const result = validateEntry(raw, ctx);
+  return result.status === "invalid" ? result.errors : [];
+};
+
+describe("validateEntry", () => {
+  it("accepts a fully valid entry", () => {
+    const result = validateEntry(entry(), ctx);
+    expect(result.status).toBe("valid");
+  });
+
+  it("treats senses: [] as empty (word skipped, not rejected)", () => {
+    const result = validateEntry(entry({ senses: [] }), ctx);
+    expect(result).toEqual({ status: "empty", headword: "casa" });
+  });
+
+  it("rejects senses: [] when the rest of the entry is invalid", () => {
+    const result = validateEntry(entry({ senses: [], language: "en" }), ctx);
+    expect(result.status).toBe("invalid");
+  });
+
+  it("rejects non-object input", () => {
+    expect(validateEntry("casa", ctx).status).toBe("invalid");
+    expect(validateEntry(null, ctx).status).toBe("invalid");
+    expect(validateEntry([entry()], ctx).status).toBe("invalid");
+  });
+
+  it("rejects a headword that was not in the input batch", () => {
+    expect(errorsOf(entry({ headword: "perro" }))).toContainEqual(
+      expect.stringContaining("not in the input batch"),
+    );
+  });
+
+  it("rejects a wrong source language", () => {
+    expect(errorsOf(entry({ language: "en" }))).toContainEqual(
+      expect.stringContaining('language must be "es"'),
+    );
+  });
+
+  it("rejects a wrong pos", () => {
+    expect(errorsOf(entry({ pos: "verb" }))).toContainEqual(
+      expect.stringContaining('pos must be "noun"'),
+    );
+  });
+
+  it("rejects more than 3 senses", () => {
+    const senses = [0, 1, 2, 3].map((i) => sense({ sense_index: i }));
+    expect(errorsOf(entry({ senses }))).toContainEqual(
+      expect.stringContaining("at most 3"),
+    );
+  });
+
+  it("rejects non-sequential sense_index", () => {
+    const senses = [sense({ sense_index: 0 }), sense({ sense_index: 2 })];
+    expect(errorsOf(entry({ senses }))).toContainEqual(
+      expect.stringContaining("sense_index must be 1"),
+    );
+  });
+
+  it("rejects an unknown difficulty", () => {
+    expect(
+      errorsOf(entry({ senses: [sense({ difficulty: "intermediate" })] })),
+    ).toContainEqual(expect.stringContaining("difficulty must be one of"));
+  });
+
+  it("rejects empty definitions and examples", () => {
+    expect(
+      errorsOf(entry({ senses: [sense({ definitions: [] })] })),
+    ).toContainEqual(
+      expect.stringContaining("definitions must be a non-empty array"),
+    );
+    expect(
+      errorsOf(entry({ senses: [sense({ examples: [""] })] })),
+    ).toContainEqual(
+      expect.stringContaining("examples must contain only non-empty strings"),
+    );
+  });
+
+  it("rejects a non-null gender for English targets", () => {
+    const bad = translations();
+    bad[0] = {
+      target_language: "en",
+      word: "house",
+      gender: "feminine",
+      difficulty: "easy",
+    };
+    expect(
+      errorsOf(entry({ senses: [sense({ translations: bad })] })),
+    ).toContainEqual(
+      expect.stringContaining('gender must be null for target "en"'),
+    );
+  });
+
+  it("rejects a null gender for German targets", () => {
+    const bad = translations();
+    bad[2] = {
+      target_language: "de",
+      word: "Haus",
+      gender: null,
+      difficulty: "easy",
+    };
+    expect(
+      errorsOf(entry({ senses: [sense({ translations: bad })] })),
+    ).toContainEqual(expect.stringContaining('for target "de"'));
+  });
+
+  it("rejects neuter for Romance-language targets", () => {
+    const bad = translations();
+    bad[3] = {
+      target_language: "fr",
+      word: "maison",
+      gender: "neuter",
+      difficulty: "easy",
+    };
+    expect(
+      errorsOf(entry({ senses: [sense({ translations: bad })] })),
+    ).toContainEqual(expect.stringContaining('for target "fr"'));
+  });
+
+  it("rejects an invented gender value", () => {
+    const bad = translations();
+    bad[2] = {
+      target_language: "de",
+      word: "Haus",
+      gender: "common",
+      difficulty: "easy",
+    };
+    expect(
+      errorsOf(entry({ senses: [sense({ translations: bad })] })).length,
+    ).toBeGreaterThan(0);
+  });
+
+  it("rejects a missing target language", () => {
+    const partial = translations().filter((t) => t.target_language !== "fr");
+    expect(
+      errorsOf(entry({ senses: [sense({ translations: partial })] })),
+    ).toContainEqual(
+      expect.stringContaining('missing translation for target language "fr"'),
+    );
+  });
+
+  it("rejects the source language as a target", () => {
+    const bad = [
+      ...translations(),
+      {
+        target_language: "es",
+        word: "hogar",
+        gender: "masculine",
+        difficulty: "easy",
+      },
+    ];
+    expect(
+      errorsOf(entry({ senses: [sense({ translations: bad })] })),
+    ).toContainEqual(expect.stringContaining("target_language must be one of"));
+  });
+
+  it("rejects more than 2 translations for one target language", () => {
+    const bad = [
+      ...translations(),
+      {
+        target_language: "de",
+        word: "Gebäude",
+        gender: "neuter",
+        difficulty: "medium",
+      },
+      {
+        target_language: "de",
+        word: "Heim",
+        gender: "neuter",
+        difficulty: "medium",
+      },
+    ];
+    expect(
+      errorsOf(entry({ senses: [sense({ translations: bad })] })),
+    ).toContainEqual(
+      expect.stringContaining(
+        'more than 2 translations for target language "de"',
+      ),
+    );
+  });
+
+  it("rejects duplicate translation words for one target language", () => {
+    const bad = [
+      ...translations(),
+      {
+        target_language: "de",
+        word: "Haus",
+        gender: "neuter",
+        difficulty: "medium",
+      },
+    ];
+    expect(
+      errorsOf(entry({ senses: [sense({ translations: bad })] })),
+    ).toContainEqual(
+      expect.stringContaining(
+        'duplicate translation word for target language "de"',
+      ),
+    );
+  });
+
+  it("rejects a translation difficulty below the sense difficulty", () => {
+    const easyTranslations = translations();
+    expect(
+      errorsOf(
+        entry({
+          senses: [
+            sense({ difficulty: "medium", translations: easyTranslations }),
+          ],
+        }),
+      ),
+    ).toContainEqual(
+      expect.stringContaining('is lower than sense difficulty "medium"'),
+    );
+  });
+
+  it("allows a translation difficulty above the sense difficulty", () => {
+    const harder = translations();
+    harder[2] = {
+      target_language: "de",
+      word: "Geldinstitut",
+      gender: "neuter",
+      difficulty: "medium",
+    };
+    const result = validateEntry(
+      entry({ senses: [sense({ translations: harder })] }),
+      ctx,
+    );
+    expect(result.status).toBe("valid");
+  });
+});