implementing phase 3 pipeline: gemini structured output, validation, sqlite staging

Prompt is now a template (fixes the hardcoded en/es leftovers in rules
2, 3, 15, 16, 26, 31). pipeline.ts replaces the pseudocode: wordlist
normalization, skip-already-staged idempotency, batches of 20 against
gemini-3.6-flash with responseSchema, raw responses persisted per batch,
per-entry validation with rejection log, one transaction per word into
db/staging.db. Flags: --langs --pos --max-batches --delay-ms --dry-run.

Smoke run: 40/40 words staged (de+es, one batch each), 0 rejections.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
lila 2026-08-09 19:04:04 +02:00
parent 303bb9388c
commit 37c978e230
12 changed files with 1477 additions and 66 deletions

View file

@ -0,0 +1,56 @@
import { readFileSync } from "node:fs";
import path from "node:path";
import type { SupportedLanguageCode, SupportedPos } from "@lila/shared";
export const LANGUAGE_NAMES: Record<SupportedLanguageCode, string> = {
en: "English",
it: "Italian",
de: "German",
fr: "French",
es: "Spanish",
};
export type PromptParams = {
sourceLanguage: SupportedLanguageCode;
pos: SupportedPos;
targetLanguages: readonly SupportedLanguageCode[];
words: readonly string[];
};
export const loadPromptTemplate = (): string =>
readFileSync(path.join(import.meta.dirname, "prompt"), "utf-8");
export const renderPrompt = (
template: string,
params: PromptParams,
): string => {
const { sourceLanguage, pos, targetLanguages, words } = params;
if (words.length === 0) {
throw new Error("renderPrompt: empty word batch");
}
if (
targetLanguages.length === 0 ||
targetLanguages.includes(sourceLanguage)
) {
throw new Error(
`renderPrompt: target languages must be non-empty and exclude the source language (got ${targetLanguages.join(", ")})`,
);
}
const rendered = template
.replaceAll("{{SOURCE_LANGUAGE_NAME}}", LANGUAGE_NAMES[sourceLanguage])
.replaceAll("{{SOURCE_LANGUAGE_CODE}}", sourceLanguage)
.replaceAll("{{POS}}", pos)
.replaceAll("{{TARGET_LANGUAGE_CODES}}", targetLanguages.join(", "))
.replaceAll(
"{{TARGET_LANGUAGE_UNION}}",
targetLanguages.map((code) => `"${code}"`).join(" | "),
)
.replaceAll("{{INPUT_WORDS}}", words.join("\n"));
const leftover = rendered.match(/\{\{[A-Z_]+\}\}/);
if (leftover) {
throw new Error(`renderPrompt: unreplaced placeholder ${leftover[0]}`);
}
return rendered;
};