implementing phase 3 pipeline: gemini structured output, validation, sqlite staging
Prompt is now a template (fixes the hardcoded en/es leftovers in rules 2, 3, 15, 16, 26, 31). pipeline.ts replaces the pseudocode: wordlist normalization, skip-already-staged idempotency, batches of 20 against gemini-3.6-flash with responseSchema, raw responses persisted per batch, per-entry validation with rejection log, one transaction per word into db/staging.db. Flags: --langs --pos --max-batches --delay-ms --dry-run. Smoke run: 40/40 words staged (de+es, one batch each), 0 rejections. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
parent
303bb9388c
commit
37c978e230
12 changed files with 1477 additions and 66 deletions
57
data-pipeline/tests/promptTemplate.test.ts
Normal file
57
data-pipeline/tests/promptTemplate.test.ts
Normal file
|
|
@ -0,0 +1,57 @@
|
|||
import { describe, it, expect } from "vitest";
|
||||
import {
|
||||
loadPromptTemplate,
|
||||
renderPrompt,
|
||||
type PromptParams,
|
||||
} from "../promptTemplate.js";
|
||||
|
||||
const params: PromptParams = {
|
||||
sourceLanguage: "es",
|
||||
pos: "noun",
|
||||
targetLanguages: ["en", "it", "de", "fr"],
|
||||
words: ["suelo", "pared"],
|
||||
};
|
||||
|
||||
describe("renderPrompt", () => {
|
||||
it("substitutes every placeholder from the checked-in template", () => {
|
||||
const rendered = renderPrompt(loadPromptTemplate(), params);
|
||||
expect(rendered).toContain('Source language: Spanish ("es")');
|
||||
expect(rendered).toContain('language must be "es"');
|
||||
expect(rendered).toContain("Target languages: en, it, de, fr");
|
||||
expect(rendered).toContain('"en" | "it" | "de" | "fr"');
|
||||
expect(rendered).toContain("suelo\npared");
|
||||
expect(rendered).toContain("valid Spanish noun");
|
||||
expect(rendered).not.toMatch(/\{\{[A-Z_]+\}\}/);
|
||||
});
|
||||
|
||||
it("does not tell the model to exclude a non-source language as target", () => {
|
||||
const rendered = renderPrompt(loadPromptTemplate(), params);
|
||||
expect(rendered).toContain(
|
||||
'Do not include Spanish ("es") as a target_language.',
|
||||
);
|
||||
expect(rendered).not.toContain(
|
||||
"Do not include English as a target_language",
|
||||
);
|
||||
});
|
||||
|
||||
it("throws when the source language is listed as a target", () => {
|
||||
expect(() =>
|
||||
renderPrompt(loadPromptTemplate(), {
|
||||
...params,
|
||||
targetLanguages: ["en", "es"],
|
||||
}),
|
||||
).toThrow(/exclude the source language/);
|
||||
});
|
||||
|
||||
it("throws on an empty word batch", () => {
|
||||
expect(() =>
|
||||
renderPrompt(loadPromptTemplate(), { ...params, words: [] }),
|
||||
).toThrow(/empty word batch/);
|
||||
});
|
||||
|
||||
it("throws on unreplaced placeholders", () => {
|
||||
expect(() => renderPrompt("hello {{UNKNOWN_TOKEN}}", params)).toThrow(
|
||||
/UNKNOWN_TOKEN/,
|
||||
);
|
||||
});
|
||||
});
|
||||
Loading…
Add table
Add a link
Reference in a new issue