implementing phase 3 pipeline: gemini structured output, validation, sqlite staging
Prompt is now a template (fixes the hardcoded en/es leftovers in rules 2, 3, 15, 16, 26, 31). pipeline.ts replaces the pseudocode: wordlist normalization, skip-already-staged idempotency, batches of 20 against gemini-3.6-flash with responseSchema, raw responses persisted per batch, per-entry validation with rejection log, one transaction per word into db/staging.db. Flags: --langs --pos --max-batches --delay-ms --dry-run. Smoke run: 40/40 words staged (de+es, one batch each), 0 rejections. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
parent
303bb9388c
commit
37c978e230
12 changed files with 1477 additions and 66 deletions
56
data-pipeline/promptTemplate.ts
Normal file
56
data-pipeline/promptTemplate.ts
Normal file
|
|
@ -0,0 +1,56 @@
|
|||
import { readFileSync } from "node:fs";
|
||||
import path from "node:path";
|
||||
import type { SupportedLanguageCode, SupportedPos } from "@lila/shared";
|
||||
|
||||
export const LANGUAGE_NAMES: Record<SupportedLanguageCode, string> = {
|
||||
en: "English",
|
||||
it: "Italian",
|
||||
de: "German",
|
||||
fr: "French",
|
||||
es: "Spanish",
|
||||
};
|
||||
|
||||
export type PromptParams = {
|
||||
sourceLanguage: SupportedLanguageCode;
|
||||
pos: SupportedPos;
|
||||
targetLanguages: readonly SupportedLanguageCode[];
|
||||
words: readonly string[];
|
||||
};
|
||||
|
||||
export const loadPromptTemplate = (): string =>
|
||||
readFileSync(path.join(import.meta.dirname, "prompt"), "utf-8");
|
||||
|
||||
export const renderPrompt = (
|
||||
template: string,
|
||||
params: PromptParams,
|
||||
): string => {
|
||||
const { sourceLanguage, pos, targetLanguages, words } = params;
|
||||
if (words.length === 0) {
|
||||
throw new Error("renderPrompt: empty word batch");
|
||||
}
|
||||
if (
|
||||
targetLanguages.length === 0 ||
|
||||
targetLanguages.includes(sourceLanguage)
|
||||
) {
|
||||
throw new Error(
|
||||
`renderPrompt: target languages must be non-empty and exclude the source language (got ${targetLanguages.join(", ")})`,
|
||||
);
|
||||
}
|
||||
|
||||
const rendered = template
|
||||
.replaceAll("{{SOURCE_LANGUAGE_NAME}}", LANGUAGE_NAMES[sourceLanguage])
|
||||
.replaceAll("{{SOURCE_LANGUAGE_CODE}}", sourceLanguage)
|
||||
.replaceAll("{{POS}}", pos)
|
||||
.replaceAll("{{TARGET_LANGUAGE_CODES}}", targetLanguages.join(", "))
|
||||
.replaceAll(
|
||||
"{{TARGET_LANGUAGE_UNION}}",
|
||||
targetLanguages.map((code) => `"${code}"`).join(" | "),
|
||||
)
|
||||
.replaceAll("{{INPUT_WORDS}}", words.join("\n"));
|
||||
|
||||
const leftover = rendered.match(/\{\{[A-Z_]+\}\}/);
|
||||
if (leftover) {
|
||||
throw new Error(`renderPrompt: unreplaced placeholder ${leftover[0]}`);
|
||||
}
|
||||
return rendered;
|
||||
};
|
||||
Loading…
Add table
Add a link
Reference in a new issue