implementing phase 3 pipeline: gemini structured output, validation, sqlite staging
Prompt is now a template (fixes the hardcoded en/es leftovers in rules 2, 3, 15, 16, 26, 31). pipeline.ts replaces the pseudocode: wordlist normalization, skip-already-staged idempotency, batches of 20 against gemini-3.6-flash with responseSchema, raw responses persisted per batch, per-entry validation with rejection log, one transaction per word into db/staging.db. Flags: --langs --pos --max-batches --delay-ms --dry-run. Smoke run: 40/40 words staged (de+es, one batch each), 0 rejections. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
lila
parent
303bb9388c
commit
37c978e230
12 changed files with 1477 additions and 66 deletions
57
data-pipeline/tests/promptTemplate.test.ts
Normal file
57
data-pipeline/tests/promptTemplate.test.ts
Normal file
|
|
@ -0,0 +1,57 @@
|
|||
import { describe, it, expect } from "vitest";
|
||||
import {
|
||||
loadPromptTemplate,
|
||||
renderPrompt,
|
||||
type PromptParams,
|
||||
} from "../promptTemplate.js";
|
||||
|
||||
const params: PromptParams = {
|
||||
sourceLanguage: "es",
|
||||
pos: "noun",
|
||||
targetLanguages: ["en", "it", "de", "fr"],
|
||||
words: ["suelo", "pared"],
|
||||
};
|
||||
|
||||
describe("renderPrompt", () => {
|
||||
it("substitutes every placeholder from the checked-in template", () => {
|
||||
const rendered = renderPrompt(loadPromptTemplate(), params);
|
||||
expect(rendered).toContain('Source language: Spanish ("es")');
|
||||
expect(rendered).toContain('language must be "es"');
|
||||
expect(rendered).toContain("Target languages: en, it, de, fr");
|
||||
expect(rendered).toContain('"en" | "it" | "de" | "fr"');
|
||||
expect(rendered).toContain("suelo\npared");
|
||||
expect(rendered).toContain("valid Spanish noun");
|
||||
expect(rendered).not.toMatch(/\{\{[A-Z_]+\}\}/);
|
||||
});
|
||||
|
||||
it("does not tell the model to exclude a non-source language as target", () => {
|
||||
const rendered = renderPrompt(loadPromptTemplate(), params);
|
||||
expect(rendered).toContain(
|
||||
'Do not include Spanish ("es") as a target_language.',
|
||||
);
|
||||
expect(rendered).not.toContain(
|
||||
"Do not include English as a target_language",
|
||||
);
|
||||
});
|
||||
|
||||
it("throws when the source language is listed as a target", () => {
|
||||
expect(() =>
|
||||
renderPrompt(loadPromptTemplate(), {
|
||||
...params,
|
||||
targetLanguages: ["en", "es"],
|
||||
}),
|
||||
).toThrow(/exclude the source language/);
|
||||
});
|
||||
|
||||
it("throws on an empty word batch", () => {
|
||||
expect(() =>
|
||||
renderPrompt(loadPromptTemplate(), { ...params, words: [] }),
|
||||
).toThrow(/empty word batch/);
|
||||
});
|
||||
|
||||
it("throws on unreplaced placeholders", () => {
|
||||
expect(() => renderPrompt("hello {{UNKNOWN_TOKEN}}", params)).toThrow(
|
||||
/UNKNOWN_TOKEN/,
|
||||
);
|
||||
});
|
||||
});
|
||||
54
data-pipeline/tests/sourceLists.test.ts
Normal file
54
data-pipeline/tests/sourceLists.test.ts
Normal file
|
|
@ -0,0 +1,54 @@
|
|||
import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import path from "node:path";
|
||||
import { describe, it, expect, afterAll } from "vitest";
|
||||
import { discoverSourceLists, normalizeWords } from "../sourceLists.js";
|
||||
|
||||
describe("normalizeWords", () => {
|
||||
it("trims whitespace and drops empty lines", () => {
|
||||
expect(normalizeWords([" Haus ", "", " ", "Tür"])).toEqual([
|
||||
"Haus",
|
||||
"Tür",
|
||||
]);
|
||||
});
|
||||
|
||||
it("dedups while preserving first-occurrence order", () => {
|
||||
expect(normalizeWords(["centro", "borde", "centro", "suelo"])).toEqual([
|
||||
"centro",
|
||||
"borde",
|
||||
"suelo",
|
||||
]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("discoverSourceLists", () => {
|
||||
const root = mkdtempSync(path.join(tmpdir(), "lila-source-data-"));
|
||||
afterAll(() => rmSync(root, { recursive: true, force: true }));
|
||||
|
||||
it("finds only supported language/pos paths and normalizes their words", () => {
|
||||
mkdirSync(path.join(root, "de"));
|
||||
mkdirSync(path.join(root, "es"));
|
||||
mkdirSync(path.join(root, "german")); // unsupported dir name — ignored
|
||||
writeFileSync(
|
||||
path.join(root, "de", "noun"),
|
||||
"Haus\nTür\nHaus\n\n Tisch \n",
|
||||
);
|
||||
writeFileSync(path.join(root, "es", "noun"), "casa\npared\n");
|
||||
writeFileSync(path.join(root, "es", "nouns"), "ignored\n"); // unsupported pos name
|
||||
writeFileSync(path.join(root, "german", "noun"), "ignored\n");
|
||||
writeFileSync(path.join(root, "stray.txt"), "ignored\n");
|
||||
|
||||
const lists = discoverSourceLists(root);
|
||||
expect(lists).toHaveLength(2);
|
||||
expect(lists[0]).toMatchObject({
|
||||
sourceLanguage: "de",
|
||||
pos: "noun",
|
||||
words: ["Haus", "Tür", "Tisch"],
|
||||
});
|
||||
expect(lists[1]).toMatchObject({
|
||||
sourceLanguage: "es",
|
||||
pos: "noun",
|
||||
words: ["casa", "pared"],
|
||||
});
|
||||
});
|
||||
});
|
||||
106
data-pipeline/tests/staging.test.ts
Normal file
106
data-pipeline/tests/staging.test.ts
Normal file
|
|
@ -0,0 +1,106 @@
|
|||
import path from "node:path";
|
||||
import { describe, it, expect, beforeEach } from "vitest";
|
||||
import type Database from "better-sqlite3";
|
||||
import {
|
||||
countStagedRows,
|
||||
getStagedHeadwords,
|
||||
openStaging,
|
||||
stageEntry,
|
||||
} from "../staging.js";
|
||||
import type { GeminiWordEntry } from "../validate.js";
|
||||
|
||||
const SCHEMA_PATH = path.join(import.meta.dirname, "..", "db", "schema.sql");
|
||||
|
||||
const entry: GeminiWordEntry = {
|
||||
headword: "casa",
|
||||
language: "es",
|
||||
pos: "noun",
|
||||
senses: [
|
||||
{
|
||||
sense_index: 0,
|
||||
difficulty: "easy",
|
||||
definitions: ["Un edificio para vivir."],
|
||||
examples: ["Compraron una casa en la ciudad."],
|
||||
translations: [
|
||||
{
|
||||
target_language: "en",
|
||||
word: "house",
|
||||
gender: null,
|
||||
difficulty: "easy",
|
||||
},
|
||||
{
|
||||
target_language: "it",
|
||||
word: "casa",
|
||||
gender: "feminine",
|
||||
difficulty: "easy",
|
||||
},
|
||||
{
|
||||
target_language: "de",
|
||||
word: "Haus",
|
||||
gender: "neuter",
|
||||
difficulty: "easy",
|
||||
},
|
||||
{
|
||||
target_language: "fr",
|
||||
word: "maison",
|
||||
gender: "feminine",
|
||||
difficulty: "easy",
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
describe("staging", () => {
|
||||
let db: Database.Database;
|
||||
beforeEach(() => {
|
||||
db = openStaging(":memory:", SCHEMA_PATH);
|
||||
});
|
||||
|
||||
it("creates the schema from db/schema.sql on an empty database", () => {
|
||||
expect(countStagedRows(db)).toEqual({
|
||||
words: 0,
|
||||
senses: 0,
|
||||
translations: 0,
|
||||
});
|
||||
});
|
||||
|
||||
it("stages a word with its senses and translations atomically", () => {
|
||||
expect(stageEntry(db, entry)).toBe("staged");
|
||||
expect(countStagedRows(db)).toEqual({
|
||||
words: 1,
|
||||
senses: 1,
|
||||
translations: 4,
|
||||
});
|
||||
|
||||
const row = db
|
||||
.prepare<
|
||||
[],
|
||||
{ definitions: string; examples: string }
|
||||
>(`SELECT definitions, examples FROM senses`)
|
||||
.get();
|
||||
expect(JSON.parse(row?.definitions ?? "")).toEqual([
|
||||
"Un edificio para vivir.",
|
||||
]);
|
||||
expect(JSON.parse(row?.examples ?? "")).toEqual([
|
||||
"Compraron una casa en la ciudad.",
|
||||
]);
|
||||
});
|
||||
|
||||
it("is idempotent: staging the same word twice writes nothing new", () => {
|
||||
expect(stageEntry(db, entry)).toBe("staged");
|
||||
expect(stageEntry(db, entry)).toBe("already-staged");
|
||||
expect(countStagedRows(db)).toEqual({
|
||||
words: 1,
|
||||
senses: 1,
|
||||
translations: 4,
|
||||
});
|
||||
});
|
||||
|
||||
it("reports staged headwords per language and pos", () => {
|
||||
stageEntry(db, entry);
|
||||
expect(getStagedHeadwords(db, "es", "noun")).toEqual(new Set(["casa"]));
|
||||
expect(getStagedHeadwords(db, "de", "noun")).toEqual(new Set());
|
||||
expect(getStagedHeadwords(db, "es", "verb")).toEqual(new Set());
|
||||
});
|
||||
});
|
||||
285
data-pipeline/tests/validate.test.ts
Normal file
285
data-pipeline/tests/validate.test.ts
Normal file
|
|
@ -0,0 +1,285 @@
|
|||
import { describe, it, expect } from "vitest";
|
||||
import { validateEntry, type ValidationContext } from "../validate.js";
|
||||
|
||||
const ctx: ValidationContext = {
|
||||
sourceLanguage: "es",
|
||||
pos: "noun",
|
||||
targetLanguages: ["en", "it", "de", "fr"],
|
||||
inputWords: new Set(["casa", "banco"]),
|
||||
};
|
||||
|
||||
type Translation = {
|
||||
target_language: string;
|
||||
word: string;
|
||||
gender: string | null;
|
||||
difficulty: string;
|
||||
};
|
||||
|
||||
const translations = (): Translation[] => [
|
||||
{ target_language: "en", word: "house", gender: null, difficulty: "easy" },
|
||||
{
|
||||
target_language: "it",
|
||||
word: "casa",
|
||||
gender: "feminine",
|
||||
difficulty: "easy",
|
||||
},
|
||||
{ target_language: "de", word: "Haus", gender: "neuter", difficulty: "easy" },
|
||||
{
|
||||
target_language: "fr",
|
||||
word: "maison",
|
||||
gender: "feminine",
|
||||
difficulty: "easy",
|
||||
},
|
||||
];
|
||||
|
||||
const sense = (
|
||||
overrides: Record<string, unknown> = {},
|
||||
): Record<string, unknown> => ({
|
||||
sense_index: 0,
|
||||
difficulty: "easy",
|
||||
definitions: ["Un edificio para vivir."],
|
||||
examples: ["Compraron una casa en la ciudad."],
|
||||
translations: translations(),
|
||||
...overrides,
|
||||
});
|
||||
|
||||
const entry = (
|
||||
overrides: Record<string, unknown> = {},
|
||||
): Record<string, unknown> => ({
|
||||
headword: "casa",
|
||||
language: "es",
|
||||
pos: "noun",
|
||||
senses: [sense()],
|
||||
...overrides,
|
||||
});
|
||||
|
||||
const errorsOf = (raw: unknown): string[] => {
|
||||
const result = validateEntry(raw, ctx);
|
||||
return result.status === "invalid" ? result.errors : [];
|
||||
};
|
||||
|
||||
describe("validateEntry", () => {
|
||||
it("accepts a fully valid entry", () => {
|
||||
const result = validateEntry(entry(), ctx);
|
||||
expect(result.status).toBe("valid");
|
||||
});
|
||||
|
||||
it("treats senses: [] as empty (word skipped, not rejected)", () => {
|
||||
const result = validateEntry(entry({ senses: [] }), ctx);
|
||||
expect(result).toEqual({ status: "empty", headword: "casa" });
|
||||
});
|
||||
|
||||
it("rejects senses: [] when the rest of the entry is invalid", () => {
|
||||
const result = validateEntry(entry({ senses: [], language: "en" }), ctx);
|
||||
expect(result.status).toBe("invalid");
|
||||
});
|
||||
|
||||
it("rejects non-object input", () => {
|
||||
expect(validateEntry("casa", ctx).status).toBe("invalid");
|
||||
expect(validateEntry(null, ctx).status).toBe("invalid");
|
||||
expect(validateEntry([entry()], ctx).status).toBe("invalid");
|
||||
});
|
||||
|
||||
it("rejects a headword that was not in the input batch", () => {
|
||||
expect(errorsOf(entry({ headword: "perro" }))).toContainEqual(
|
||||
expect.stringContaining("not in the input batch"),
|
||||
);
|
||||
});
|
||||
|
||||
it("rejects a wrong source language", () => {
|
||||
expect(errorsOf(entry({ language: "en" }))).toContainEqual(
|
||||
expect.stringContaining('language must be "es"'),
|
||||
);
|
||||
});
|
||||
|
||||
it("rejects a wrong pos", () => {
|
||||
expect(errorsOf(entry({ pos: "verb" }))).toContainEqual(
|
||||
expect.stringContaining('pos must be "noun"'),
|
||||
);
|
||||
});
|
||||
|
||||
it("rejects more than 3 senses", () => {
|
||||
const senses = [0, 1, 2, 3].map((i) => sense({ sense_index: i }));
|
||||
expect(errorsOf(entry({ senses }))).toContainEqual(
|
||||
expect.stringContaining("at most 3"),
|
||||
);
|
||||
});
|
||||
|
||||
it("rejects non-sequential sense_index", () => {
|
||||
const senses = [sense({ sense_index: 0 }), sense({ sense_index: 2 })];
|
||||
expect(errorsOf(entry({ senses }))).toContainEqual(
|
||||
expect.stringContaining("sense_index must be 1"),
|
||||
);
|
||||
});
|
||||
|
||||
it("rejects an unknown difficulty", () => {
|
||||
expect(
|
||||
errorsOf(entry({ senses: [sense({ difficulty: "intermediate" })] })),
|
||||
).toContainEqual(expect.stringContaining("difficulty must be one of"));
|
||||
});
|
||||
|
||||
it("rejects empty definitions and examples", () => {
|
||||
expect(
|
||||
errorsOf(entry({ senses: [sense({ definitions: [] })] })),
|
||||
).toContainEqual(
|
||||
expect.stringContaining("definitions must be a non-empty array"),
|
||||
);
|
||||
expect(
|
||||
errorsOf(entry({ senses: [sense({ examples: [""] })] })),
|
||||
).toContainEqual(
|
||||
expect.stringContaining("examples must contain only non-empty strings"),
|
||||
);
|
||||
});
|
||||
|
||||
it("rejects a non-null gender for English targets", () => {
|
||||
const bad = translations();
|
||||
bad[0] = {
|
||||
target_language: "en",
|
||||
word: "house",
|
||||
gender: "feminine",
|
||||
difficulty: "easy",
|
||||
};
|
||||
expect(
|
||||
errorsOf(entry({ senses: [sense({ translations: bad })] })),
|
||||
).toContainEqual(
|
||||
expect.stringContaining('gender must be null for target "en"'),
|
||||
);
|
||||
});
|
||||
|
||||
it("rejects a null gender for German targets", () => {
|
||||
const bad = translations();
|
||||
bad[2] = {
|
||||
target_language: "de",
|
||||
word: "Haus",
|
||||
gender: null,
|
||||
difficulty: "easy",
|
||||
};
|
||||
expect(
|
||||
errorsOf(entry({ senses: [sense({ translations: bad })] })),
|
||||
).toContainEqual(expect.stringContaining('for target "de"'));
|
||||
});
|
||||
|
||||
it("rejects neuter for Romance-language targets", () => {
|
||||
const bad = translations();
|
||||
bad[3] = {
|
||||
target_language: "fr",
|
||||
word: "maison",
|
||||
gender: "neuter",
|
||||
difficulty: "easy",
|
||||
};
|
||||
expect(
|
||||
errorsOf(entry({ senses: [sense({ translations: bad })] })),
|
||||
).toContainEqual(expect.stringContaining('for target "fr"'));
|
||||
});
|
||||
|
||||
it("rejects an invented gender value", () => {
|
||||
const bad = translations();
|
||||
bad[2] = {
|
||||
target_language: "de",
|
||||
word: "Haus",
|
||||
gender: "common",
|
||||
difficulty: "easy",
|
||||
};
|
||||
expect(
|
||||
errorsOf(entry({ senses: [sense({ translations: bad })] })).length,
|
||||
).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
it("rejects a missing target language", () => {
|
||||
const partial = translations().filter((t) => t.target_language !== "fr");
|
||||
expect(
|
||||
errorsOf(entry({ senses: [sense({ translations: partial })] })),
|
||||
).toContainEqual(
|
||||
expect.stringContaining('missing translation for target language "fr"'),
|
||||
);
|
||||
});
|
||||
|
||||
it("rejects the source language as a target", () => {
|
||||
const bad = [
|
||||
...translations(),
|
||||
{
|
||||
target_language: "es",
|
||||
word: "hogar",
|
||||
gender: "masculine",
|
||||
difficulty: "easy",
|
||||
},
|
||||
];
|
||||
expect(
|
||||
errorsOf(entry({ senses: [sense({ translations: bad })] })),
|
||||
).toContainEqual(expect.stringContaining("target_language must be one of"));
|
||||
});
|
||||
|
||||
it("rejects more than 2 translations for one target language", () => {
|
||||
const bad = [
|
||||
...translations(),
|
||||
{
|
||||
target_language: "de",
|
||||
word: "Gebäude",
|
||||
gender: "neuter",
|
||||
difficulty: "medium",
|
||||
},
|
||||
{
|
||||
target_language: "de",
|
||||
word: "Heim",
|
||||
gender: "neuter",
|
||||
difficulty: "medium",
|
||||
},
|
||||
];
|
||||
expect(
|
||||
errorsOf(entry({ senses: [sense({ translations: bad })] })),
|
||||
).toContainEqual(
|
||||
expect.stringContaining(
|
||||
'more than 2 translations for target language "de"',
|
||||
),
|
||||
);
|
||||
});
|
||||
|
||||
it("rejects duplicate translation words for one target language", () => {
|
||||
const bad = [
|
||||
...translations(),
|
||||
{
|
||||
target_language: "de",
|
||||
word: "Haus",
|
||||
gender: "neuter",
|
||||
difficulty: "medium",
|
||||
},
|
||||
];
|
||||
expect(
|
||||
errorsOf(entry({ senses: [sense({ translations: bad })] })),
|
||||
).toContainEqual(
|
||||
expect.stringContaining(
|
||||
'duplicate translation word for target language "de"',
|
||||
),
|
||||
);
|
||||
});
|
||||
|
||||
it("rejects a translation difficulty below the sense difficulty", () => {
|
||||
const easyTranslations = translations();
|
||||
expect(
|
||||
errorsOf(
|
||||
entry({
|
||||
senses: [
|
||||
sense({ difficulty: "medium", translations: easyTranslations }),
|
||||
],
|
||||
}),
|
||||
),
|
||||
).toContainEqual(
|
||||
expect.stringContaining('is lower than sense difficulty "medium"'),
|
||||
);
|
||||
});
|
||||
|
||||
it("allows a translation difficulty above the sense difficulty", () => {
|
||||
const harder = translations();
|
||||
harder[2] = {
|
||||
target_language: "de",
|
||||
word: "Geldinstitut",
|
||||
gender: "neuter",
|
||||
difficulty: "medium",
|
||||
};
|
||||
const result = validateEntry(
|
||||
entry({ senses: [sense({ translations: harder })] }),
|
||||
ctx,
|
||||
);
|
||||
expect(result.status).toBe("valid");
|
||||
});
|
||||
});
|
||||
Loading…
Add table
Add a link
Reference in a new issue