37c978e230
Prompt is now a template (fixes the hardcoded en/es leftovers in rules 2, 3, 15, 16, 26, 31). pipeline.ts replaces the pseudocode: wordlist normalization, skip-already-staged idempotency, batches of 20 against gemini-3.6-flash with responseSchema, raw responses persisted per batch, per-entry validation with rejection log, one transaction per word into db/staging.db. Flags: --langs --pos --max-batches --delay-ms --dry-run. Smoke run: 40/40 words staged (de+es, one batch each), 0 rejections. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
54 lines
1.8 KiB
TypeScript
54 lines
1.8 KiB
TypeScript
import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
|
|
import { tmpdir } from "node:os";
|
|
import path from "node:path";
|
|
import { describe, it, expect, afterAll } from "vitest";
|
|
import { discoverSourceLists, normalizeWords } from "../sourceLists.js";
|
|
|
|
describe("normalizeWords", () => {
|
|
it("trims whitespace and drops empty lines", () => {
|
|
expect(normalizeWords([" Haus ", "", " ", "Tür"])).toEqual([
|
|
"Haus",
|
|
"Tür",
|
|
]);
|
|
});
|
|
|
|
it("dedups while preserving first-occurrence order", () => {
|
|
expect(normalizeWords(["centro", "borde", "centro", "suelo"])).toEqual([
|
|
"centro",
|
|
"borde",
|
|
"suelo",
|
|
]);
|
|
});
|
|
});
|
|
|
|
describe("discoverSourceLists", () => {
|
|
const root = mkdtempSync(path.join(tmpdir(), "lila-source-data-"));
|
|
afterAll(() => rmSync(root, { recursive: true, force: true }));
|
|
|
|
it("finds only supported language/pos paths and normalizes their words", () => {
|
|
mkdirSync(path.join(root, "de"));
|
|
mkdirSync(path.join(root, "es"));
|
|
mkdirSync(path.join(root, "german")); // unsupported dir name — ignored
|
|
writeFileSync(
|
|
path.join(root, "de", "noun"),
|
|
"Haus\nTür\nHaus\n\n Tisch \n",
|
|
);
|
|
writeFileSync(path.join(root, "es", "noun"), "casa\npared\n");
|
|
writeFileSync(path.join(root, "es", "nouns"), "ignored\n"); // unsupported pos name
|
|
writeFileSync(path.join(root, "german", "noun"), "ignored\n");
|
|
writeFileSync(path.join(root, "stray.txt"), "ignored\n");
|
|
|
|
const lists = discoverSourceLists(root);
|
|
expect(lists).toHaveLength(2);
|
|
expect(lists[0]).toMatchObject({
|
|
sourceLanguage: "de",
|
|
pos: "noun",
|
|
words: ["Haus", "Tür", "Tisch"],
|
|
});
|
|
expect(lists[1]).toMatchObject({
|
|
sourceLanguage: "es",
|
|
pos: "noun",
|
|
words: ["casa", "pared"],
|
|
});
|
|
});
|
|
});
|