forgejo-lila/lila
1
0
Fork 0
Code Issues 50 Pull requests Projects 1 Releases Packages Wiki Activity Actions

wip

Browse source
This commit is contained in:
lila 2026-06-03 00:02:19 +02:00
parent 0118798e36
commit cc89f0c75c
20 changed files with 384 additions and 5600 deletions
Download patch file Download diff file Expand all files Collapse all files

8
.gitignore vendored
View file

@ -10,13 +10,7 @@ venv/
__pycache__/ __pycache__/
*.pyc *.pyc
data-pipeline/archive/ data-pipeline/kaikki-source-files/
data-pipeline/stage-1-extract/output/
data-pipeline/stage-1-extract/sources/
data-pipeline/stage-2-annotate/output/
data-pipeline/stage-3-enrich/output/
data-pipeline/stage-4-merge/output/
data-pipeline/db/pipeline.db data-pipeline/db/pipeline.db
data-pipeline/reports/
data-pipeline/.env data-pipeline/.env
.aider* .aider*

87
data-pipeline/audit.ts
View file

@ -1,87 +0,0 @@
import Database from "better-sqlite3";
import path from "node:path";
import fs from "node:fs";
import { fileURLToPath } from "node:url";
const __dirname = path.dirname(fileURLToPath(import.meta.url));
const DB_PATH = path.join(__dirname, "db/pipeline.db");
const db = new Database(DB_PATH, { readonly: true });
// Pull 50 synsets: ~12 per POS, all must have German translations
const synsets = db
.prepare(
`
SELECT DISTINCT s.source_id, s.pos
FROM synsets s
JOIN translations t ON t.source_id = s.source_id
WHERE t.language = 'de'
ORDER BY RANDOM()
LIMIT 50
`,
)
.all() as { source_id: string; pos: string }[];
const results: string[] = [];
let index = 0;
for (const synset of synsets) {
index++;
const glosses = db
.prepare("SELECT language, text FROM glosses WHERE source_id = ?")
.all(synset.source_id) as { language: string; text: string }[];
const enGloss = glosses.find((g) => g.language === "en")?.text ?? "—";
const deGloss = glosses.find((g) => g.language === "de")?.text ?? "—";
const deTranslations = db
.prepare(
"SELECT word FROM translations WHERE source_id = ? AND language = 'de'",
)
.all(synset.source_id) as { word: string }[];
const enTranslations = db
.prepare(
"SELECT word FROM translations WHERE source_id = ? AND language = 'en'",
)
.all(synset.source_id) as { word: string }[];
const deWords = deTranslations.map((t) => t.word);
const enWords = enTranslations.map((t) => t.word);
results.push(
[
`${String(index).padStart(2, " ")}. [${synset.pos}] ${synset.source_id}`,
` EN gloss: ${enGloss}`,
` DE gloss: ${deGloss}`,
` EN words: ${enWords.join(", ")}`,
` DE words: ${deWords.join(", ")}`,
` QUALITY: ___`,
``,
].join("\n"),
);
}
const output = [
"# OMW German Translation Quality Audit",
"",
"Instructions: for each entry, check if the German translations",
"match the meaning described by the English gloss.",
"",
"Mark QUALITY as:",
" OK — all German translations fit the meaning",
" PARTIAL — some fit, some don't",
" BAD — none of the German translations fit",
" USELESS — translations are correct but useless for learners",
"",
"---",
"",
...results,
].join("\n");
const outPath = path.join(__dirname, "audit.md");
fs.writeFileSync(outPath, output, "utf-8");
console.log(`Wrote ${synsets.length} entries → ${outPath}`);
db.close();

154
data-pipeline/db/import.ts
View file

@ -1,154 +0,0 @@
import fs from "node:fs/promises";
import path from "node:path";
import { fileURLToPath } from "node:url";
import { SUPPORTED_LANGUAGE_CODES } from "@lila/shared";
import { openDb } from "./index.js";
import type { ExtractedSense } from "../stage-1-extract/scripts/extract.js";
// ── Paths ─────────────────────────────────────────────────────────────────────
const __dirname = path.dirname(fileURLToPath(import.meta.url));
const OUTPUT_DIR = path.resolve(__dirname, "../stage-1-extract/output");
// ── Import ────────────────────────────────────────────────────────────────────
export async function importKaikki(): Promise<void> {
const db = openDb();
const insertEntry = db.prepare(`
INSERT INTO entries (headword, language, pos, sense_index, gloss, examples)
VALUES (?, ?, ?, ?, ?, ?)
ON CONFLICT (headword, language, pos, sense_index)
DO UPDATE SET
gloss = excluded.gloss,
examples = excluded.examples
RETURNING id
`);
const insertTranslation = db.prepare(`
INSERT INTO translations (entry_id, target_lang, word, sense_hint)
VALUES (?, ?, ?, ?)
ON CONFLICT (entry_id, target_lang, word) DO NOTHING
`);
let totalEntries = 0;
let totalTranslations = 0;
let totalSkipped = 0;
for (const lang of SUPPORTED_LANGUAGE_CODES) {
const filePath = path.join(OUTPUT_DIR, `${lang}.json`);
let senses: ExtractedSense[];
try {
const raw = await fs.readFile(filePath, "utf-8");
senses = JSON.parse(raw) as ExtractedSense[];
} catch {
console.warn(` Warning: no output file found for ${lang}, skipping`);
continue;
}
console.log(
` Importing ${lang}: ${senses.length.toLocaleString()} senses...`,
);
// Track next available sense_index per (headword, pos) to handle
// the same word appearing in multiple JSONL entries with the same POS.
const senseIndexMap = new Map<string, number>();
const importLang = db.transaction(() => {
let entries = 0;
let translations = 0;
let skipped = 0;
for (const sense of senses) {
const key = `${sense.headword}|${sense.pos}`;
const nextIndex = senseIndexMap.get(key) ?? 0;
senseIndexMap.set(key, nextIndex + 1);
const row = insertEntry.get(
sense.headword,
sense.language,
sense.pos,
nextIndex,
sense.gloss ?? null,
JSON.stringify(sense.examples),
) as { id: number } | undefined;
if (!row) {
skipped++;
continue;
}
entries++;
for (const t of sense.translations) {
insertTranslation.run(
row.id,
t.target_lang,
t.word,
t.sense_hint ?? null,
);
translations++;
}
}
return { entries, translations, skipped };
});
const counts = importLang();
totalEntries += counts.entries;
totalTranslations += counts.translations;
totalSkipped += counts.skipped;
console.log(
` entries: ${counts.entries.toLocaleString()}, translations: ${counts.translations.toLocaleString()}, skipped: ${counts.skipped.toLocaleString()}`,
);
}
db.close();
console.log(`\nImport complete:`);
console.log(` Total entries: ${totalEntries.toLocaleString()}`);
console.log(` Total translations: ${totalTranslations.toLocaleString()}`);
console.log(` Total skipped: ${totalSkipped.toLocaleString()}`);
}
// ── Check if already imported ─────────────────────────────────────────────────
export function isImported(): boolean {
const db = openDb();
const row = db.prepare("SELECT COUNT(*) as count FROM entries").get() as {
count: number;
};
db.close();
return row.count > 0;
}
// ── Main ─────────────────────────────────────────────────────────────────────
async function main(): Promise<void> {
const db = openDb();
const row = db.prepare("SELECT COUNT(*) as count FROM entries").get() as {
count: number;
};
db.close();
if (row.count > 0) {
console.log(
`pipeline.db already contains ${row.count.toLocaleString()} entries — skipping import.`,
);
console.log("Delete pipeline.db and re-run db:init to start fresh.");
process.exit(0);
}
console.log("Importing Kaikki data into pipeline.db...");
await importKaikki();
}
if (import.meta.url === `file://${process.argv[1]}`) {
main().catch((err) => {
console.error(err);
process.exit(1);
});
}

24
data-pipeline/db/index.ts
View file

@ -1,24 +0,0 @@
import path from "node:path";
import { fileURLToPath } from "node:url";
import Database from "better-sqlite3";
// ── Paths ─────────────────────────────────────────────────────────────────────
const __dirname = path.dirname(fileURLToPath(import.meta.url));
const DB_PATH = path.join(__dirname, "pipeline.db");
// ── Types ─────────────────────────────────────────────────────────────────────
export type Db = InstanceType<typeof Database>;
// ── Open ──────────────────────────────────────────────────────────────────────
export function openDb(): Db {
const db = new Database(DB_PATH);
db.pragma("journal_mode = WAL");
db.pragma("foreign_keys = ON");
return db;
}

42
data-pipeline/db/init.ts
View file

@ -1,42 +0,0 @@
import fs from "node:fs/promises";
import path from "node:path";
import { fileURLToPath } from "node:url";
import Database from "better-sqlite3";
// ── Paths ─────────────────────────────────────────────────────────────────────
const __dirname = path.dirname(fileURLToPath(import.meta.url));
const PATHS = {
schema: path.join(__dirname, "schema.sql"),
db: path.join(__dirname, "pipeline.db"),
};
// ── Init ──────────────────────────────────────────────────────────────────────
export async function initDb(): Promise<void> {
const schema = await fs.readFile(PATHS.schema, "utf-8");
const db = new Database(PATHS.db);
db.pragma("journal_mode = WAL");
db.pragma("foreign_keys = ON");
db.exec(schema);
db.close();
console.log(` pipeline.db initialised → ${PATHS.db}`);
}
// ── Main ─────────────────────────────────────────────────────────────────────
async function main(): Promise<void> {
console.log("Initialising pipeline.db...");
await initDb();
}
// after
if (import.meta.url === `file://${process.argv[1]}`) {
main().catch((err) => {
console.error(err);
process.exit(1);
});
}

BIN
data-pipeline/db/pipeline.db-shm
View file

Binary file not shown.

0
data-pipeline/db/pipeline.db-wal
View file

41
data-pipeline/db/reset.ts
View file

@ -1,41 +0,0 @@
import path from "node:path";
import { fileURLToPath } from "node:url";
import Database from "better-sqlite3";
// ── Paths ─────────────────────────────────────────────────────────────────────
const __dirname = path.dirname(fileURLToPath(import.meta.url));
const DB_PATH = path.join(__dirname, "pipeline.db");
// ── Main ──────────────────────────────────────────────────────────────────────
function main(): void {
const mode = process.argv[2];
if (!mode || (mode !== "round1" && mode !== "all")) {
console.error("Usage: pnpm db:reset round1 | all");
console.error(" round1 — delete all round1 sub-stage rows");
console.error(" all — delete all run_status rows except reverse_link");
process.exit(1);
}
const db = new Database(DB_PATH);
let result: { changes: number };
if (mode === "round1") {
result = db
.prepare("DELETE FROM run_status WHERE stage LIKE 'round1%'")
.run();
console.log(`Deleted ${result.changes} round1 rows from run_status`);
} else {
result = db
.prepare("DELETE FROM run_status WHERE stage NOT IN ('reverse_link')")
.run();
console.log(`Deleted ${result.changes} rows from run_status`);
}
db.close();
}
main();

164
data-pipeline/db/schema.sql
View file

@ -1,164 +0,0 @@
-- ── Base data ─────────────────────────────────────────────────────────────────
-- Imported from Kaikki on first run. Never mutated after import.
CREATE TABLE IF NOT EXISTS entries (
id INTEGER PRIMARY KEY,
headword TEXT NOT NULL,
language TEXT NOT NULL,
pos TEXT NOT NULL,
sense_index INTEGER NOT NULL DEFAULT 0,
gloss TEXT,
examples TEXT NOT NULL DEFAULT '[]', -- JSON array of strings
source TEXT NOT NULL DEFAULT 'kaikki',
UNIQUE (headword, language, pos, sense_index)
);
CREATE TABLE IF NOT EXISTS translations (
id INTEGER PRIMARY KEY,
entry_id INTEGER NOT NULL REFERENCES entries(id),
target_lang TEXT NOT NULL,
word TEXT NOT NULL,
sense_hint TEXT,
source TEXT NOT NULL DEFAULT 'kaikki',
UNIQUE (entry_id, target_lang, word)
);
-- ── Status tracking ───────────────────────────────────────────────────────────
-- One row per entry per model per stage. Drives resumability.
-- Sentinel rows use entry_id = 0 for one-time pipeline steps.
-- stage: round1 | round2 | tiebreak
-- status: pending | complete | needs_review | flagged
CREATE TABLE IF NOT EXISTS run_status (
id INTEGER PRIMARY KEY,
entry_id INTEGER NOT NULL,
model_name TEXT NOT NULL,
stage TEXT NOT NULL,
status TEXT NOT NULL,
created_at TEXT NOT NULL DEFAULT (datetime('now')),
updated_at TEXT NOT NULL DEFAULT (datetime('now')),
UNIQUE (entry_id, model_name, stage)
);
-- ── Round 1 output ────────────────────────────────────────────────────────────
-- Written atomically per entry per model.
-- Unique constraints enforce one model one vote.
CREATE TABLE IF NOT EXISTS model_entry_cefr_votes (
id INTEGER PRIMARY KEY,
entry_id INTEGER NOT NULL REFERENCES entries(id),
model_name TEXT NOT NULL,
cefr_level TEXT NOT NULL,
UNIQUE (entry_id, model_name)
);
CREATE TABLE IF NOT EXISTS model_translation_cefr_votes (
id INTEGER PRIMARY KEY,
translation_id INTEGER NOT NULL REFERENCES translations(id),
model_name TEXT NOT NULL,
cefr_level TEXT NOT NULL,
UNIQUE (translation_id, model_name)
);
CREATE TABLE IF NOT EXISTS model_translation_rejections (
id INTEGER PRIMARY KEY,
translation_id INTEGER NOT NULL REFERENCES translations(id),
model_name TEXT NOT NULL,
UNIQUE (translation_id, model_name)
);
CREATE TABLE IF NOT EXISTS generated_glosses (
id INTEGER PRIMARY KEY,
entry_id INTEGER NOT NULL REFERENCES entries(id),
model_name TEXT NOT NULL,
text TEXT NOT NULL,
UNIQUE (entry_id, model_name)
);
CREATE TABLE IF NOT EXISTS generated_examples (
id INTEGER PRIMARY KEY,
entry_id INTEGER NOT NULL REFERENCES entries(id),
model_name TEXT NOT NULL,
text TEXT NOT NULL,
UNIQUE (entry_id, model_name)
);
CREATE TABLE IF NOT EXISTS generated_translations (
id INTEGER PRIMARY KEY,
entry_id INTEGER NOT NULL REFERENCES entries(id),
model_name TEXT NOT NULL,
target_lang TEXT NOT NULL,
word TEXT NOT NULL,
UNIQUE (entry_id, model_name, target_lang)
);
-- ── Round 2 output ────────────────────────────────────────────────────────────
-- Each row represents one model voting for one candidate.
-- The candidate with the most votes wins in merge.
CREATE TABLE IF NOT EXISTS gloss_candidate_votes (
id INTEGER PRIMARY KEY,
gloss_id INTEGER NOT NULL REFERENCES generated_glosses(id),
model_name TEXT NOT NULL,
UNIQUE (gloss_id, model_name)
);
CREATE TABLE IF NOT EXISTS example_candidate_votes (
id INTEGER PRIMARY KEY,
example_id INTEGER NOT NULL REFERENCES generated_examples(id),
model_name TEXT NOT NULL,
UNIQUE (example_id, model_name)
);
CREATE TABLE IF NOT EXISTS translation_candidate_votes (
id INTEGER PRIMARY KEY,
translation_id INTEGER NOT NULL REFERENCES generated_translations(id),
model_name TEXT NOT NULL,
UNIQUE (translation_id, model_name)
);
-- ── Resolved output ───────────────────────────────────────────────────────────
-- Written by merge. Never updated after writing.
-- Only fully resolved records are written here — no nulls.
-- Absence of a row means unresolved. Flagged status tracked in run_status.
-- source: kaikki | model_name
CREATE TABLE IF NOT EXISTS resolved_entry_cefr (
id INTEGER PRIMARY KEY,
entry_id INTEGER NOT NULL REFERENCES entries(id),
cefr_level TEXT NOT NULL,
difficulty TEXT NOT NULL,
UNIQUE (entry_id)
);
CREATE TABLE IF NOT EXISTS resolved_translation_cefr (
id INTEGER PRIMARY KEY,
translation_id INTEGER NOT NULL REFERENCES translations(id),
cefr_level TEXT NOT NULL,
difficulty TEXT NOT NULL,
UNIQUE (translation_id)
);
CREATE TABLE IF NOT EXISTS resolved_glosses (
id INTEGER PRIMARY KEY,
entry_id INTEGER NOT NULL REFERENCES entries(id),
text TEXT NOT NULL,
source TEXT NOT NULL,
UNIQUE (entry_id)
);
CREATE TABLE IF NOT EXISTS resolved_examples (
id INTEGER PRIMARY KEY,
entry_id INTEGER NOT NULL REFERENCES entries(id),
text TEXT NOT NULL,
source TEXT NOT NULL
);
CREATE TABLE IF NOT EXISTS resolved_generated_translations (
id INTEGER PRIMARY KEY,
entry_id INTEGER NOT NULL REFERENCES entries(id),
target_lang TEXT NOT NULL,
word TEXT NOT NULL,
source TEXT NOT NULL,
UNIQUE (entry_id, target_lang)
);

616
data-pipeline/pipeline.ts
View file

@ -1,616 +0,0 @@
import fs from "node:fs/promises";
import path from "node:path";
import { fileURLToPath } from "node:url";
import { initDb } from "./db/init.js";
import { isImported, importKaikki } from "./db/import.js";
import { openDb } from "./db/index.js";
import { reverseLink } from "./stage-2-reverse-link/scripts/reverse-link.js";
import { ALL_PROVIDERS, validateProviderKey } from "./stage-3-enrich/config.js";
import type { ProviderConfig } from "./stage-3-enrich/config.js";
import { enrich } from "./stage-3-enrich/scripts/enrich.js";
// ── Types ─────────────────────────────────────────────────────────────────────
type RunStage =
| "round1"
| "compile_candidates"
| "round2"
| "compile_votes"
| "merge"
| "tiebreak"
| "compare";
type StageStatus = "complete" | "pending" | "in_progress";
type RunStats = {
startedAt: Date;
stoppedAt: Date | null;
recordsProcessed: number;
recordsSkipped: number;
needsReview: number;
modelsRun: string[];
currentStage: RunStage | null;
};
// ── Constants ─────────────────────────────────────────────────────────────────
const __dirname = path.dirname(fileURLToPath(import.meta.url));
const PATHS = {
extractedEn: path.join(__dirname, "stage-1-extract/output/en.json"),
db: path.join(__dirname, "db/pipeline.db"),
reports: path.join(__dirname, "reports"),
llamaHealth: "http://127.0.0.1:8080/health",
};
const SENTINEL = { entryId: 0, modelName: "system" };
// ── Startup checks ────────────────────────────────────────────────────────────
async function checkExtractedFilesExist(): Promise<void> {
try {
await fs.access(PATHS.extractedEn);
} catch {
console.error("\n ERROR: stage-1-extract/output/en.json not found.");
console.error(" Run the stage 1 extraction script first:");
console.error(" pnpm extract\n");
process.exit(1);
}
}
async function checkAndInitDb(): Promise<void> {
try {
await fs.access(PATHS.db);
} catch {
console.log(" pipeline.db not found — initialising...");
await initDb();
}
}
async function checkAndImportDb(): Promise<void> {
if (!isImported()) {
console.log(" Base tables empty — importing Kaikki data...");
await importKaikki();
}
}
async function checkLlamaServer(): Promise<boolean> {
try {
const res = await fetch(PATHS.llamaHealth);
return res.ok;
} catch {
return false;
}
}
function isLocalProvider(provider: ProviderConfig): boolean {
return provider.apiKey === "none";
}
async function checkProviderReady(provider: ProviderConfig): Promise<void> {
if (isLocalProvider(provider)) {
const running = await checkLlamaServer();
if (!running) {
console.error("\n ERROR: llama.cpp server is not running.");
console.error(" Start the server before running the pipeline:");
console.error(
" ./build/bin/llama-server --model models/<model>.gguf \\",
);
console.error(" --port 8080 --host 127.0.0.1");
console.error(" See llm-setup.md for full instructions.\n");
process.exit(1);
}
} else {
validateProviderKey(provider);
}
}
// ── Run name generation ───────────────────────────────────────────────────────
async function generateRunName(): Promise<string> {
await fs.mkdir(PATHS.reports, { recursive: true });
const date = new Date().toISOString().exi(0, 10);
const files = await fs.readdir(PATHS.reports);
const todaysRuns = files.filter(
(f) => f.startsWith(date) && f.endsWith(".json"),
).length;
return `${date}_run-${todaysRuns + 1}`;
}
// ── Shutdown handler ──────────────────────────────────────────────────────────
let shutdownRequested = false;
function registerShutdownHandler(stats: RunStats): void {
const handler = (): void => {
if (shutdownRequested) return;
shutdownRequested = true;
stats.stoppedAt = new Date();
console.log("\n\n Shutdown requested — finishing current record...");
};
process.on("SIGINT", handler);
process.on("SIGTERM", handler);
}
// ── Stage status helpers ──────────────────────────────────────────────────────
function getSentinelStatus(stage: RunStage): StageStatus {
const db = openDb();
const row = db
.prepare(
`SELECT status FROM run_status
WHERE entry_id = ? AND model_name = ? AND stage = ?`,
)
.get(SENTINEL.entryId, SENTINEL.modelName, stage) as
| { status: string }
| undefined;
db.close();
return row?.status === "complete" ? "complete" : "pending";
}
function markSentinelComplete(stage: RunStage): void {
const db = openDb();
db.prepare(
`INSERT INTO run_status (entry_id, model_name, stage, status)
VALUES (?, ?, ?, 'complete')
ON CONFLICT (entry_id, model_name, stage)
DO UPDATE SET status = 'complete', updated_at = datetime('now')`,
).run(SENTINEL.entryId, SENTINEL.modelName, stage);
db.close();
}
function getModelRound1Status(modelName: string): StageStatus {
const db = openDb();
const total = (
db
.prepare("SELECT COUNT(*) as count FROM entries WHERE language = 'en'")
.get() as { count: number }
).count;
const complete = (
db
.prepare(
`SELECT COUNT(*) as count FROM run_status
WHERE model_name = ? AND stage = 'round1_gloss'
AND status = 'complete'`,
)
.get(modelName) as { count: number }
).count;
db.close();
if (complete === 0) return "pending";
if (complete >= total) return "complete";
return "in_progress";
}
function getModelRound2Status(modelName: string): StageStatus {
const db = openDb();
const total = (
db
.prepare("SELECT COUNT(*) as count FROM entries WHERE language = 'en'")
.get() as { count: number }
).count;
const complete = (
db
.prepare(
`SELECT COUNT(*) as count FROM run_status
WHERE model_name = ? AND stage = 'round2' AND status = 'complete'`,
)
.get(modelName) as { count: number }
).count;
db.close();
if (complete === 0) return "pending";
if (complete >= total) return "complete";
return "in_progress";
}
function isReverseLinkDone(): boolean {
const db = openDb();
const row = db
.prepare(
`SELECT status FROM run_status
WHERE entry_id = ? AND model_name = ? AND stage = 'reverse_link'`,
)
.get(SENTINEL.entryId, SENTINEL.modelName) as
| { status: string }
| undefined;
db.close();
return row?.status === "complete";
}
function markReverseLinkComplete(): void {
const db = openDb();
db.prepare(
`INSERT INTO run_status (entry_id, model_name, stage, status)
VALUES (?, ?, 'reverse_link', 'complete')
ON CONFLICT (entry_id, model_name, stage)
DO UPDATE SET status = 'complete', updated_at = datetime('now')`,
).run(SENTINEL.entryId, SENTINEL.modelName);
db.close();
}
// ── Stage runners ─────────────────────────────────────────────────────────────
function runReverseLinkStage(): void {
if (isReverseLinkDone()) {
console.log("\n [reverse link] Already complete, skipping");
return;
}
console.log("\n [reverse link] Syncing reverse translation links...");
reverseLink();
markReverseLinkComplete();
}
async function runRound1(
provider: ProviderConfig,
stats: RunStats,
): Promise<void> {
console.log(`\n [round 1] Running ${provider.name}...`);
const counts = await enrich(provider);
stats.recordsProcessed += counts.processed;
stats.recordsSkipped += counts.skipped;
stats.needsReview += counts.needsReview;
stats.modelsRun.push(provider.name);
}
function compileCandidates(): void {
console.log("\n [compile candidates] Compiling round 1 output...");
// TODO: implement compile candidates script
console.log(" [compile candidates] not yet implemented");
markSentinelComplete("compile_candidates");
}
function runRound2(provider: ProviderConfig, stats: RunStats): void {
console.log(`\n [round 2] Running ${provider.name}...`);
// TODO: implement round 2 enrich script
console.log(` [round 2] ${provider.name} — not yet implemented`);
stats.modelsRun.push(provider.name);
}
function compileVotes(): void {
console.log("\n [compile votes] Compiling round 2 votes...");
// TODO: implement compile votes script
console.log(" [compile votes] not yet implemented");
markSentinelComplete("compile_votes");
}
function runMerge(): void {
console.log("\n [merge] Resolving votes...");
// TODO: implement merge script
console.log(" [merge] not yet implemented");
markSentinelComplete("merge");
}
function runTiebreak(stats: RunStats): void {
console.log("\n [tiebreak] Resolving flagged entries...");
// TODO: implement tiebreak logic
console.log(" [tiebreak] not yet implemented");
stats.currentStage = "tiebreak";
}
function runCompare(): void {
console.log("\n [compare] Generating COVERAGE.md...");
// TODO: implement compare script
console.log(" [compare] not yet implemented");
markSentinelComplete("compare");
}
// ── Report generation ─────────────────────────────────────────────────────────
async function generateReport(runName: string, stats: RunStats): Promise<void> {
const db = openDb();
const totalEntries = (
db.prepare("SELECT COUNT(*) as count FROM entries").get() as {
count: number;
}
).count;
const resolvedEntries = (
db.prepare("SELECT COUNT(*) as count FROM resolved_entry_cefr").get() as {
count: number;
}
).count;
const flaggedEntries = (
db
.prepare(
`SELECT COUNT(*) as count FROM run_status
WHERE stage = 'merge' AND status = 'flagged'`,
)
.get() as { count: number }
).count;
const needsReview = (
db
.prepare(
`SELECT COUNT(*) as count FROM run_status
WHERE status = 'needs_review'`,
)
.get() as { count: number }
).count;
db.close();
const stoppedAt = stats.stoppedAt ?? new Date();
const durationMs = stoppedAt.getTime() - stats.startedAt.getTime();
const durationMin = Math.round(durationMs / 60_000);
const isFinal =
getSentinelStatus("compare") === "complete" && flaggedEntries === 0;
const report = {
runName,
generatedAt: stoppedAt.toISOString(),
durationMinutes: durationMin,
isFinal,
progress: {
totalEntries,
resolvedEntries,
flaggedEntries,
needsReview,
recordsProcessedThisRun: stats.recordsProcessed,
recordsSkippedThisRun: stats.recordsSkipped,
},
modelsRun: stats.modelsRun,
stages: {
reverseLink: isReverseLinkDone() ? "complete" : "pending",
round1: ALL_PROVIDERS.map((p) => ({
model: p.name,
status: getModelRound1Status(p.name),
})),
compileCandidates: getSentinelStatus("compile_candidates"),
round2: ALL_PROVIDERS.map((p) => ({
model: p.name,
status: getModelRound2Status(p.name),
})),
compileVotes: getSentinelStatus("compile_votes"),
merge: getSentinelStatus("merge"),
compare: getSentinelStatus("compare"),
},
};
await fs.mkdir(PATHS.reports, { recursive: true });
const jsonPath = path.join(PATHS.reports, `${runName}.json`);
const mdPath = path.join(PATHS.reports, `${runName}.md`);
await fs.writeFile(jsonPath, JSON.stringify(report, null, 2), "utf-8");
const md = [
`# Pipeline run: ${runName}`,
``,
`Generated: ${stoppedAt.toISOString()}`,
`Duration: ${durationMin} minutes`,
isFinal
? `**Status: FINAL — pipeline complete**`
: `**Status: In progress**`,
``,
`## Progress`,
``,
`| Metric | Value |`,
`| ------ | ----- |`,
`| Total entries | ${totalEntries.toLocaleString()} |`,
`| Resolved entries | ${resolvedEntries.toLocaleString()} |`,
`| Flagged entries | ${flaggedEntries.toLocaleString()} |`,
`| Needs review | ${needsReview.toLocaleString()} |`,
`| Records processed this run | ${stats.recordsProcessed.toLocaleString()} |`,
`| Records skipped this run | ${stats.recordsSkipped.toLocaleString()} |`,
``,
`## Stage status`,
``,
`### Reverse link: ${report.stages.reverseLink}`,
``,
`### Round 1`,
``,
...report.stages.round1.map(
(s) =>
`- ${s.status === "complete" ? "✅" : s.status === "in_progress" ? "🔄" : "🔲"} ${s.model}`,
),
``,
`### Compile candidates: ${report.stages.compileCandidates}`,
``,
`### Round 2`,
``,
...report.stages.round2.map(
(s) =>
`- ${s.status === "complete" ? "✅" : s.status === "in_progress" ? "🔄" : "🔲"} ${s.model}`,
),
``,
`### Compile votes: ${report.stages.compileVotes}`,
`### Merge: ${report.stages.merge}`,
`### Compare: ${report.stages.compare}`,
``,
`## Models run this session`,
``,
stats.modelsRun.length > 0
? stats.modelsRun.map((m) => `- ${m}`).join("\n")
: "_none_",
].join("\n");
await fs.writeFile(mdPath, md, "utf-8");
console.log(`\n Report written → ${jsonPath}`);
console.log(` Report written → ${mdPath}`);
}
// ── Main ──────────────────────────────────────────────────────────────────────
async function main(): Promise<void> {
console.log("lila data pipeline\n");
// ── Startup checks
console.log("Checking prerequisites...");
await checkExtractedFilesExist();
await checkAndInitDb();
await checkAndImportDb();
console.log(" Prerequisites OK");
// ── Run name
const runName = await generateRunName();
console.log(`\n Run: ${runName}`);
// ── Stats
const stats: RunStats = {
startedAt: new Date(),
stoppedAt: null,
recordsProcessed: 0,
recordsSkipped: 0,
needsReview: 0,
modelsRun: [],
currentStage: null,
};
registerShutdownHandler(stats);
// ── Stage 2 — Reverse link
runReverseLinkStage();
if (shutdownRequested) {
await generateReport(runName, stats);
process.exit(0);
}
// ── Round 1
console.log("\nRound 1 — generation");
for (const provider of ALL_PROVIDERS) {
if (shutdownRequested) break;
const status = getModelRound1Status(provider.name);
if (status === "complete") {
console.log(` [round 1] ${provider.name} — already complete, skipping`);
continue;
}
await checkProviderReady(provider);
stats.currentStage = "round1";
if (status === "in_progress") {
console.log(` [round 1] ${provider.name} — resuming...`);
}
await runRound1(provider, stats);
}
if (shutdownRequested) {
await generateReport(runName, stats);
process.exit(0);
}
// ── Compile candidates
if (getSentinelStatus("compile_candidates") === "complete") {
console.log("\n [compile candidates] Already complete, skipping");
} else {
stats.currentStage = "compile_candidates";
compileCandidates();
}
if (shutdownRequested) {
await generateReport(runName, stats);
process.exit(0);
}
// ── Round 2
console.log("\nRound 2 — voting");
for (const provider of ALL_PROVIDERS) {
if (shutdownRequested) break;
const status = getModelRound2Status(provider.name);
if (status === "complete") {
console.log(` [round 2] ${provider.name} — already complete, skipping`);
continue;
}
await checkProviderReady(provider);
stats.currentStage = "round2";
if (status === "in_progress") {
console.log(` [round 2] ${provider.name} — resuming...`);
}
runRound2(provider, stats);
}
if (shutdownRequested) {
await generateReport(runName, stats);
process.exit(0);
}
// ── Compile votes
if (getSentinelStatus("compile_votes") === "complete") {
console.log("\n [compile votes] Already complete, skipping");
} else {
stats.currentStage = "compile_votes";
compileVotes();
}
if (shutdownRequested) {
await generateReport(runName, stats);
process.exit(0);
}
// ── Merge
if (getSentinelStatus("merge") === "complete") {
console.log("\n [merge] Already complete, skipping");
} else {
stats.currentStage = "merge";
runMerge();
}
if (shutdownRequested) {
await generateReport(runName, stats);
process.exit(0);
}
// ── Tiebreak
const db = openDb();
const flagged = (
db
.prepare(
`SELECT COUNT(*) as count FROM run_status
WHERE stage = 'merge' AND status = 'flagged'`,
)
.get() as { count: number }
).count;
db.close();
if (flagged > 0) {
stats.currentStage = "tiebreak";
runTiebreak(stats);
}
if (shutdownRequested) {
await generateReport(runName, stats);
process.exit(0);
}
// ── Compare
if (getSentinelStatus("compare") === "complete") {
console.log("\n [compare] Already complete, skipping");
} else {
stats.currentStage = "compare";
runCompare();
}
// ── Report (disabled until full pipeline is implemented)
// stats.stoppedAt = new Date();
// await generateReport(runName, stats);
console.log("\nPipeline complete.");
}
main().catch((err) => {
console.error(err);
process.exit(1);
});

2472
data-pipeline/sample/output/sample.json
View file

File diff suppressed because it is too large Load diff

205
data-pipeline/sample/scripts/sample.ts
View file

@ -1,205 +0,0 @@
import fs from "node:fs/promises";
import path from "node:path";
import { SUPPORTED_LANGUAGE_CODES } from "@lila/shared";
import type { SupportedLanguageCode, SupportedPos } from "@lila/shared";
// ── Types ─────────────────────────────────────────────────────────────────────
type Example = { text: string; source: "omw" | "cefr" };
type AnnotatedRecord = {
source_id: string;
pos: SupportedPos;
translations: Partial<Record<SupportedLanguageCode, string[]>>;
glosses: Partial<Record<SupportedLanguageCode, string[]>>;
examples: Partial<Record<SupportedLanguageCode, Example[]>>;
votes: Partial<
Record<SupportedLanguageCode, Record<string, { cefr_source: string }>>
>;
};
type SampleRecord = AnnotatedRecord & { _sample_bucket: string };
// ── Constants ─────────────────────────────────────────────────────────────────
const PATHS = {
annotatedDir: "stage-2-annotate/output",
output: "test/output/sample.json",
};
const BUCKET_SIZE = 20;
// ── Bucket predicates ─────────────────────────────────────────────────────────
type Bucket = { name: string; predicate: (record: AnnotatedRecord) => boolean };
const BUCKETS: Bucket[] = [
{
name: "has_cefr_vote",
predicate: (r) =>
Object.values(r.votes).some(
(langVotes) => Object.keys(langVotes ?? {}).length > 0,
),
},
{
name: "no_cefr_vote",
predicate: (r) =>
Object.values(r.votes).every(
(langVotes) => Object.keys(langVotes ?? {}).length === 0,
),
},
{
name: "has_glosses_and_examples",
predicate: (r) =>
Object.keys(r.glosses).length > 0 && Object.keys(r.examples).length > 0,
},
{
name: "no_glosses_no_examples",
predicate: (r) =>
!r.glosses["fr"] &&
!r.examples["fr"] &&
!r.votes["fr"] &&
!r.glosses["es"] &&
!r.examples["es"] &&
!r.votes["es"],
},
{
name: "pos_spread",
predicate: () => true, // sampled separately to ensure POS coverage
},
];
// ── Sampling ──────────────────────────────────────────────────────────────────
function sampleBucket(
records: AnnotatedRecord[],
predicate: (r: AnnotatedRecord) => boolean,
size: number,
exclude: Set<string>,
): AnnotatedRecord[] {
const candidates = records.filter(
(r) => !exclude.has(r.source_id) && predicate(r),
);
// Shuffle for random sampling
for (let i = candidates.length - 1; i > 0; i--) {
const j = Math.floor(Math.random() * (i + 1));
[candidates[i], candidates[j]] = [candidates[j]!, candidates[i]!];
}
return candidates.slice(0, size);
}
function samplePosBucket(
records: AnnotatedRecord[],
exclude: Set<string>,
): AnnotatedRecord[] {
const posList: SupportedPos[] = ["noun", "verb", "adjective", "adverb"];
const perPos = Math.floor(BUCKET_SIZE / posList.length);
const result: AnnotatedRecord[] = [];
for (const pos of posList) {
const sampled = sampleBucket(
records,
(r) => r.pos === pos,
perPos,
exclude,
);
result.push(...sampled);
}
return result;
}
// ── Loading ───────────────────────────────────────────────────────────────────
async function loadAnnotated(): Promise<AnnotatedRecord[]> {
// Load all language files and merge votes into a single record set.
// Use en.json as the base record structure since it has the most complete
// glosses and examples. Votes from all other languages are merged in.
const baseRaw = await fs.readFile(
path.join(PATHS.annotatedDir, "en.json"),
"utf-8",
);
const base = JSON.parse(baseRaw) as AnnotatedRecord[];
// Build a map for fast lookup by source_id
const byId = new Map<string, AnnotatedRecord>();
for (const record of base) {
byId.set(record.source_id, record);
}
// Merge votes from remaining language files
for (const lang of SUPPORTED_LANGUAGE_CODES) {
if (lang === "en") continue;
const raw = await fs.readFile(
path.join(PATHS.annotatedDir, `${lang}.json`),
"utf-8",
);
const records = JSON.parse(raw) as AnnotatedRecord[];
for (const record of records) {
const base = byId.get(record.source_id);
if (!base) continue;
// Merge votes
for (const [l, langVotes] of Object.entries(record.votes)) {
if (!base.votes[l as SupportedLanguageCode]) {
base.votes[l as SupportedLanguageCode] = {};
}
Object.assign(base.votes[l as SupportedLanguageCode]!, langVotes);
}
// Merge examples from CEFR source files not in base
for (const [l, examples] of Object.entries(record.examples)) {
const lang = l as SupportedLanguageCode;
if (!base.examples[lang]) {
base.examples[lang] = examples;
}
}
}
}
return [...byId.values()];
}
// ── Main ─────────────────────────────────────────────────────────────────────
async function main(): Promise<void> {
console.log("Loading annotated files...");
const records = await loadAnnotated();
console.log(` Loaded ${records.length.toLocaleString()} synsets`);
const sampled: SampleRecord[] = [];
const seen = new Set<string>();
// Sample each bucket except pos_spread
for (const bucket of BUCKETS.filter((b) => b.name !== "pos_spread")) {
const results = sampleBucket(records, bucket.predicate, BUCKET_SIZE, seen);
for (const r of results) {
seen.add(r.source_id);
sampled.push({ ...r, _sample_bucket: bucket.name });
}
console.log(` ${bucket.name}: ${results.length} records`);
}
// Sample pos_spread bucket
const posResults = samplePosBucket(records, seen);
for (const r of posResults) {
seen.add(r.source_id);
sampled.push({ ...r, _sample_bucket: "pos_spread" });
}
console.log(` pos_spread: ${posResults.length} records`);
console.log(`\nTotal sampled: ${sampled.length} records`);
// Write output
await fs.mkdir(path.dirname(PATHS.output), { recursive: true });
await fs.writeFile(PATHS.output, JSON.stringify(sampled, null, 2), "utf-8");
console.log(`Wrote sample → ${PATHS.output}`);
}
main().catch((err) => {
console.error(err);
process.exit(1);
});

257
data-pipeline/stage-1-extract/scripts/extract.ts
View file

@ -1,257 +0,0 @@
import fs from "node:fs";
import path from "node:path";
import readline from "node:readline";
import { fileURLToPath } from "node:url";
import { SUPPORTED_LANGUAGE_CODES } from "@lila/shared";
import type { SupportedLanguageCode, SupportedPos } from "@lila/shared";
// ── Types ─────────────────────────────────────────────────────────────────────
type KaikkiTranslation = {
code?: string;
lang_code?: string;
word?: string;
sense?: string;
};
type KaikkiSense = {
glosses?: string[];
examples?: { text?: string }[];
translations?: KaikkiTranslation[];
};
type KaikkiEntry = {
word?: string;
pos?: string;
lang_code?: string;
senses?: KaikkiSense[];
};
export type ExtractedSense = {
headword: string;
language: SupportedLanguageCode;
pos: SupportedPos;
sense_index: number;
gloss: string | null;
examples: string[];
translations: {
target_lang: SupportedLanguageCode;
word: string;
sense_hint: string | null;
}[];
};
// ── Constants ─────────────────────────────────────────────────────────────────
const __dirname = path.dirname(fileURLToPath(import.meta.url));
const SOURCES_DIR = path.resolve(__dirname, "../sources");
const OUTPUT_DIR = path.resolve(__dirname, "../output");
const LANG_TO_FILE: Record<SupportedLanguageCode, string> = {
en: "kaikki.org-dictionary-English.jsonl",
de: "kaikki.org-dictionary-German.jsonl",
it: "kaikki.org-dictionary-Italian.jsonl",
fr: "kaikki.org-dictionary-French.jsonl",
es: "kaikki.org-dictionary-Spanish.jsonl",
};
const POS_MAP: Record<string, SupportedPos> = {
noun: "noun",
verb: "verb",
adj: "adjective",
adv: "adverb",
};
const SUPPORTED_LANG_SET = new Set<string>(SUPPORTED_LANGUAGE_CODES);
// ── Helpers ───────────────────────────────────────────────────────────────────
function mapPos(kaikkiPos: string): SupportedPos | null {
return POS_MAP[kaikkiPos] ?? null;
}
function isAbbreviation(gloss: string): boolean {
return gloss.toLowerCase().startsWith("abbreviation of");
}
function extractTranslations(
sense: KaikkiSense,
sourceLang: SupportedLanguageCode,
): ExtractedSense["translations"] {
const seen = new Set<string>();
const result: ExtractedSense["translations"] = [];
for (const t of sense.translations ?? []) {
const code = t.code ?? t.lang_code;
if (!code || !SUPPORTED_LANG_SET.has(code)) continue;
if (code === sourceLang) continue; // skip same-language translations
if (!t.word?.trim()) continue;
const key = `${code}:${t.word.trim()}`;
if (seen.has(key)) continue;
seen.add(key);
result.push({
target_lang: code as SupportedLanguageCode,
word: t.word.trim(),
sense_hint: t.sense?.trim() ?? null,
});
}
return result;
}
function extractExamples(sense: KaikkiSense): string[] {
return (sense.examples ?? [])
.map((e) => e.text?.trim())
.filter((t): t is string => !!t);
}
function processEntry(
entry: KaikkiEntry,
sourceLang: SupportedLanguageCode,
): Omit<ExtractedSense, "sense_index">[] {
const pos = mapPos(entry.pos ?? "");
if (!pos) return [];
if (!entry.word?.trim()) return [];
// For non-English files, only process entries in the target language
const entryLang = (entry as Record<string, unknown>)["lang_code"] as
| string
| undefined;
if (sourceLang !== "en" && entryLang !== sourceLang) return [];
const headword = entry.word.trim();
const results: Omit<ExtractedSense, "sense_index">[] = [];
for (const sense of entry.senses ?? []) {
const gloss = sense.glosses?.[0]?.trim() ?? null;
if (gloss && isAbbreviation(gloss)) continue;
if (sourceLang === "en") {
// English: require translations in supported languages
const translations = extractTranslations(sense, sourceLang);
if (translations.length === 0) continue;
results.push({
headword,
language: sourceLang,
pos,
gloss,
examples: extractExamples(sense),
translations,
});
} else {
// Non-English: just extract the entry, no translations needed
results.push({
headword,
language: sourceLang,
pos,
gloss,
examples: extractExamples(sense),
translations: [],
});
}
}
return results;
}
// ── Extract ───────────────────────────────────────────────────────────────────
export async function extract(
lang: SupportedLanguageCode,
sampleLimit?: number,
): Promise<void> {
const filename = LANG_TO_FILE[lang];
const sourcePath = path.join(SOURCES_DIR, filename);
const outputPath = path.join(OUTPUT_DIR, `${lang}.json`);
console.log(`\nExtracting ${lang}...`);
console.log(` Source: ${sourcePath}`);
if (sampleLimit) console.log(` Sample mode: ${sampleLimit} entries`);
await fs.promises.mkdir(OUTPUT_DIR, { recursive: true });
const fileStream = fs.createReadStream(sourcePath);
const rl = readline.createInterface({
input: fileStream,
crlfDelay: Infinity,
});
const senses: ExtractedSense[] = [];
const senseIndexMap = new Map<string, number>();
let linesRead = 0;
let entriesProcessed = 0;
let entriesSkipped = 0;
for await (const line of rl) {
if (!line.trim()) continue;
if (sampleLimit && entriesProcessed >= sampleLimit) break;
linesRead++;
let entry: KaikkiEntry;
try {
entry = JSON.parse(line) as KaikkiEntry;
} catch {
console.warn(` Warning: failed to parse line ${linesRead}, skipping`);
continue;
}
const extracted = processEntry(entry, lang);
if (extracted.length === 0) {
entriesSkipped++;
continue;
}
for (const sense of extracted) {
const key = `${sense.headword}|${sense.pos}`;
const senseIndex = senseIndexMap.get(key) ?? 0;
senseIndexMap.set(key, senseIndex + 1);
senses.push({ ...sense, sense_index: senseIndex });
}
entriesProcessed++;
if (entriesProcessed % 10_000 === 0) {
console.log(
` Processed ${entriesProcessed.toLocaleString()} entries...`,
);
}
}
await fs.promises.writeFile(
outputPath,
JSON.stringify(senses, null, 2),
"utf-8",
);
console.log(` Lines read: ${linesRead.toLocaleString()}`);
console.log(` Entries processed: ${entriesProcessed.toLocaleString()}`);
console.log(` Entries skipped: ${entriesSkipped.toLocaleString()}`);
console.log(` Senses extracted: ${senses.length.toLocaleString()}`);
console.log(` Output: ${outputPath}`);
}
// ── Main ─────────────────────────────────────────────────────────────────────
async function main(): Promise<void> {
// Hardcoded sample limit for development — remove for full extraction
const SAMPLE = 500;
for (const lang of SUPPORTED_LANGUAGE_CODES) {
await extract(lang, SAMPLE);
}
console.log("\nExtraction complete.");
}
if (import.meta.url === `file://${process.argv[1]}`) {
main().catch((err) => {
console.error(err);
process.exit(1);
});
}

109
data-pipeline/stage-2-reverse-link/scripts/reverse-link.ts
View file

@ -1,109 +0,0 @@
import { openDb } from "../../db/index.js";
// ── Types ─────────────────────────────────────────────────────────────────────
type TranslationRow = {
translation_id: number;
entry_id: number;
entry_language: string;
entry_headword: string;
target_lang: string;
word: string;
sense_hint: string | null;
};
type EntryRow = { id: number };
// ── Sync ──────────────────────────────────────────────────────────────────────
export function reverseLink(): void {
const db = openDb();
// Find all translations and their source entry details
const translations = db
.prepare(
`SELECT
t.id AS translation_id,
t.entry_id,
e.language AS entry_language,
e.headword AS entry_headword,
t.target_lang,
t.word,
t.sense_hint
FROM translations t
JOIN entries e ON e.id = t.entry_id`,
)
.all() as TranslationRow[];
console.log(
` Found ${translations.length.toLocaleString()} translations to check`,
);
const findEntry = db.prepare(
`SELECT id FROM entries WHERE headword = ? AND language = ? LIMIT 1`,
);
const insertReverseLink = db.prepare(
`INSERT INTO translations (entry_id, target_lang, word, sense_hint, source)
VALUES (?, ?, ?, ?, 'reverse_link')
ON CONFLICT (entry_id, target_lang, word) DO NOTHING`,
);
const sync = db.transaction(() => {
let inserted = 0;
let skipped = 0;
let noEntry = 0;
for (const t of translations) {
// Look for an entry in the target language with the translation word as headword
const targetEntry = findEntry.get(t.word, t.target_lang) as
| EntryRow
| undefined;
if (!targetEntry) {
noEntry++;
continue;
}
// Insert reverse link: target entry → source language → source headword
const result = insertReverseLink.run(
targetEntry.id,
t.entry_language,
t.entry_headword,
t.sense_hint ?? null,
);
if (result.changes > 0) {
inserted++;
} else {
skipped++;
}
}
return { inserted, skipped, noEntry };
});
const counts = sync();
db.close();
console.log(` Inserted: ${counts.inserted.toLocaleString()} reverse links`);
console.log(
` Skipped: ${counts.skipped.toLocaleString()} (already existed)`,
);
console.log(
` No entry: ${counts.noEntry.toLocaleString()} (target word not in entries)`,
);
}
// ── Main ─────────────────────────────────────────────────────────────────────
function main(): void {
console.log("Running reverse link sync...");
reverseLink();
console.log("\nReverse link sync complete.");
}
if (import.meta.url === `file://${process.argv[1]}`) {
main();
}

123
data-pipeline/stage-3-enrich/config.ts
View file

@ -1,123 +0,0 @@
// ── Provider configuration ────────────────────────────────────────────────────
//
// Each provider + model combination counts as one vote in the final majority.
// Running the same model twice is not supported — one model, one vote.
// The `name` field is used as the model identifier in pipeline.db and must
// be unique across all runs.
//
// The pipeline iterates through ALL_PROVIDERS in order, skipping models that
// have already completed a full run and resuming models with partial progress.
//
// See llm-setup.md for full setup instructions and model recommendations.
export type ProviderConfig = {
name: string; // unique model identifier — stored in pipeline.db
baseURL: string;
apiKey: string;
model: string;
maxTokens: number;
};
// ── Local llama.cpp ───────────────────────────────────────────────────────────
export const LOCAL_QWEN35_4B: ProviderConfig = {
name: "local-qwen3.5-4b",
baseURL: "http://127.0.0.1:8080/v1",
apiKey: "none",
model: "qwen3.5-4b",
maxTokens: 1024, // no reasoning overhead so 1024 is enough
};
export const LOCAL_GEMMA4: ProviderConfig = {
name: "local-gemma4-e4b",
baseURL: "http://127.0.0.1:8080/v1",
apiKey: "none", // llama.cpp ignores this
model: "gemma4-e4b", // llama.cpp ignores model name, uses loaded model
maxTokens: 2048,
};
export const LOCAL_QWEN7B: ProviderConfig = {
name: "local-qwen2.5-7b",
baseURL: "http://127.0.0.1:8080/v1",
apiKey: "none",
model: "qwen2.5-7b",
maxTokens: 512,
};
// ── OpenRouter — free tier ────────────────────────────────────────────────────
export const OR_QWEN3_480B: ProviderConfig = {
name: "or-qwen3-480b",
baseURL: "https://openrouter.ai/api/v1",
apiKey: process.env["OPENROUTER_API_KEY"] ?? "",
model: "qwen/qwen3-coder:free",
maxTokens: 512,
};
export const OR_GEMMA4_31B: ProviderConfig = {
name: "or-gemma4-31b",
baseURL: "https://openrouter.ai/api/v1",
apiKey: process.env["OPENROUTER_API_KEY"] ?? "",
model: "google/gemma-4-31b-it:free",
maxTokens: 512,
};
export const OR_QWEN3_80B: ProviderConfig = {
name: "or-qwen3-80b",
baseURL: "https://openrouter.ai/api/v1",
apiKey: process.env["OPENROUTER_API_KEY"] ?? "",
model: "qwen/qwen3-next-80b-a3b-instruct:free",
maxTokens: 512,
};
export const OR_NEMOTRON: ProviderConfig = {
name: "or-nemotron-120b",
baseURL: "https://openrouter.ai/api/v1",
apiKey: process.env["OPENROUTER_API_KEY"] ?? "",
model: "nvidia/nemotron-3-super-120b-a12b:free",
maxTokens: 512,
};
// ── Anthropic — reference baseline ───────────────────────────────────────────
// Note: Anthropic uses a different API format. An adapter is required.
// See llm-setup.md for details.
export const ANTHROPIC_SONNET: ProviderConfig = {
name: "anthropic-sonnet-4",
baseURL: "https://api.anthropic.com/v1",
apiKey: process.env["ANTHROPIC_API_KEY"] ?? "",
model: "claude-sonnet-4-6",
maxTokens: 512,
};
// ── All configured providers ──────────────────────────────────────────────────
// The pipeline runs through these in order — local models first, then cloud.
// Add new providers here to include them in the voting pool.
export const ALL_PROVIDERS: ProviderConfig[] = [
LOCAL_QWEN35_4B,
// LOCAL_GEMMA4,
// LOCAL_QWEN7B,
// OR_QWEN3_480B,
// OR_GEMMA4_31B,
// OR_QWEN3_80B,
// OR_NEMOTRON,
// ANTHROPIC_SONNET,
];
// ── Key validation ────────────────────────────────────────────────────────────
const LOCAL_PROVIDERS = new Set(["none"]);
export function validateProviderKey(provider: ProviderConfig): void {
if (LOCAL_PROVIDERS.has(provider.apiKey)) return;
if (!provider.apiKey) {
const keyName = provider.name.startsWith("anthropic")
? "ANTHROPIC_API_KEY"
: "OPENROUTER_API_KEY";
console.error(`\n ERROR: ${keyName} is not set in .env`);
console.error(` Provider "${provider.name}" requires this key to run.\n`);
process.exit(1);
}
}

877
data-pipeline/stage-3-enrich/scripts/enrich.ts
View file

@ -1,877 +0,0 @@
import { openDb } from "../../db/index.js";
import type { ProviderConfig } from "../config.js";
import { CEFR_LEVELS, SUPPORTED_LANGUAGE_CODES } from "@lila/shared";
import type { SupportedLanguageCode } from "@lila/shared";
// ── Types ─────────────────────────────────────────────────────────────────────
type EntryRow = {
id: number;
headword: string;
language: SupportedLanguageCode;
pos: string;
gloss: string | null;
examples: string; // JSON array string
};
type TranslationRow = {
id: number;
target_lang: SupportedLanguageCode;
word: string;
};
type GlossResult = { status: "ok" } | { status: "improved"; gloss: string };
type ExampleResult = { status: "ok" } | { status: "improved"; example: string };
type TranslationResult = {
translations: Partial<
Record<SupportedLanguageCode, Record<string, "ok" | "reject">>
>;
generated?: Partial<Record<SupportedLanguageCode, string>>;
};
type CefrResult = {
headword_cefr: string;
translation_cefr: Partial<
Record<SupportedLanguageCode, Record<string, string>>
>;
};
type SubStage =
| "round1_gloss"
| "round1_example"
| "round1_translations"
| "round1_cefr";
// ── Constants ─────────────────────────────────────────────────────────────────
const SUPPORTED_LANG_SET = new Set<string>(SUPPORTED_LANGUAGE_CODES);
const CEFR_SET = new Set<string>(CEFR_LEVELS);
// ── Shutdown ──────────────────────────────────────────────────────────────────
let shutdownRequested = false;
let currentCallController: AbortController | null = null;
export function registerEnrichShutdown(): void {
const handler = (): void => {
if (shutdownRequested) return;
shutdownRequested = true;
console.log("\n\n Shutdown requested — aborting current LLM call...");
currentCallController?.abort();
};
process.on("SIGINT", handler);
process.on("SIGTERM", handler);
}
// ── Prompt builders ───────────────────────────────────────────────────────────
function buildGlossPrompt(entry: EntryRow): string {
const glossText = entry.gloss ?? "none";
const examples: string[] = JSON.parse(entry.examples) as string[];
const examplesText =
examples.length > 0 ? examples.map((e) => ` - ${e}`).join("\n") : " none";
return `You are a language learning expert.
Review this gloss for the ${entry.pos} "${entry.headword}" (sense ${entry.sense_index}).
Gloss: "${glossText}"
Examples of this specific sense:
${examplesText}
Is this gloss clear, accurate for this specific sense, and suitable for a language learner?
- If yes, respond with: {"status": "ok"}
- If no or if gloss is "none", respond with: {"status": "improved", "gloss": "your improved gloss here"}
IMPORTANT: Your improved gloss must describe THIS SPECIFIC SENSE shown by the examples above,
not a more common or general meaning of the word.
Respond ONLY with valid JSON and nothing else.`;
}
function buildTranslationsPrompt(
entry: EntryRow,
translations: TranslationRow[],
verifiedGloss: string,
): string {
const byLang = new Map<SupportedLanguageCode, string[]>();
for (const t of translations) {
if (!byLang.has(t.target_lang)) byLang.set(t.target_lang, []);
byLang.get(t.target_lang)!.push(t.word);
}
const coveredLangs = new Set(byLang.keys());
const missingLangs = SUPPORTED_LANGUAGE_CODES.filter(
(l) => l !== entry.language && !coveredLangs.has(l),
);
const translationsText =
byLang.size > 0
? [...byLang.entries()]
.map(([lang, words]) => ` ${lang}: ${words.join(", ")}`)
.join("\n")
: " none";
const missingText =
missingLangs.length > 0 ? missingLangs.join(", ") : "none";
const exampleResponse: Record<string, unknown> = {
translations: {
de: { frei: "ok", "-frei": "reject" },
it: { libero: "ok", free: "reject" },
},
};
if (missingLangs.length > 0) {
exampleResponse["generated"] = { es: "libre", fr: "libre" };
}
return `You are a language learning expert.
For the ${entry.language} ${entry.pos} "${entry.headword}" (meaning: "${verifiedGloss}"), review these translations:
${translationsText}
For each translation:
- Write "ok" if it is a valid translation for this specific meaning
- Write "reject" if it is wrong, a suffix (starts with -), garbled text, or the wrong language
Examples of correct behaviour:
- "free" listed as Italian → "reject" (it is English, not Italian)
- "-frei" listed as German → "reject" (it is a suffix, not a standalone word)
- "libre" listed as Spanish → "ok" (it is a valid Spanish word)
${missingLangs.length > 0 ? `Also generate the single best translation for these missing languages: ${missingText}` : ""}
Respond ONLY with valid JSON and nothing else:
${JSON.stringify(exampleResponse, null, 2)}`;
}
function buildCefrPrompt(
entry: EntryRow,
verifiedGloss: string,
validatedTranslations: Map<SupportedLanguageCode, string[]>,
): string {
const translationsText =
validatedTranslations.size > 0
? [...validatedTranslations.entries()]
.map(([lang, words]) => ` ${lang}: ${words.join(", ")}`)
.join("\n")
: " none";
return `You are a language learning expert.
Assign CEFR levels (A1, A2, B1, B2, C1, or C2) to this word and its validated translations.
Base your levels on how commonly a language learner at that level would encounter this specific sense.
Consider register — slang, technical, and archaic words should be rated higher.
WORD: ${entry.headword} (${entry.pos})
MEANING: ${verifiedGloss}
VALIDATED TRANSLATIONS:
${translationsText}
Respond ONLY with valid JSON and nothing else:
{
"headword_cefr": "B1",
"translation_cefr": {
"de": { "frei": "A2" },
"it": { "libero": "A2" }
}
}`;
}
// ── Validation ────────────────────────────────────────────────────────────────
function validateGloss(raw: string): GlossResult | null {
try {
const obj = JSON.parse(raw) as Record<string, unknown>;
if (obj["status"] === "ok") return { status: "ok" };
if (
obj["status"] === "improved" &&
typeof obj["gloss"] === "string" &&
obj["gloss"].trim()
) {
return { status: "improved", gloss: obj["gloss"].trim() };
}
return null;
} catch {
return null;
}
}
function validateExample(raw: string): ExampleResult | null {
try {
const obj = JSON.parse(raw) as Record<string, unknown>;
if (obj["status"] === "ok") return { status: "ok" };
if (
obj["status"] === "improved" &&
typeof obj["example"] === "string" &&
obj["example"].trim()
) {
return { status: "improved", example: obj["example"].trim() };
}
return null;
} catch {
return null;
}
}
function validateTranslations(
raw: string,
translations: TranslationRow[],
): TranslationResult | null {
try {
const obj = JSON.parse(raw) as Record<string, unknown>;
if (typeof obj["translations"] !== "object" || obj["translations"] === null)
return null;
const result: TranslationResult = { translations: {} };
const translationsObj = obj["translations"] as Record<string, unknown>;
// Validate each language's votes
for (const [lang, votes] of Object.entries(translationsObj)) {
if (!SUPPORTED_LANG_SET.has(lang)) continue;
if (typeof votes !== "object" || votes === null) continue;
result.translations[lang as SupportedLanguageCode] = {};
for (const [word, status] of Object.entries(
votes as Record<string, unknown>,
)) {
if (status === "ok" || status === "reject") {
result.translations[lang as SupportedLanguageCode]![word] = status;
}
}
}
// Validate generated translations
if (obj["generated"] !== undefined && obj["generated"] !== null) {
if (typeof obj["generated"] !== "object") return null;
result.generated = {};
for (const [lang, word] of Object.entries(
obj["generated"] as Record<string, unknown>,
)) {
if (!SUPPORTED_LANG_SET.has(lang)) continue;
if (typeof word === "string" && word.trim()) {
result.generated[lang as SupportedLanguageCode] = word.trim();
}
}
}
// Check all translations got a vote
const byLang = new Map<string, Set<string>>();
for (const t of translations) {
if (!byLang.has(t.target_lang)) byLang.set(t.target_lang, new Set());
byLang.get(t.target_lang)!.add(t.word);
}
for (const [lang, words] of byLang.entries()) {
const votes = result.translations[lang as SupportedLanguageCode];
if (!votes) return null;
for (const word of words) {
if (!votes[word]) return null;
}
}
return result;
} catch {
return null;
}
}
function validateCefr(
raw: string,
validatedTranslations: Map<SupportedLanguageCode, string[]>,
): CefrResult | null {
try {
const obj = JSON.parse(raw) as Record<string, unknown>;
if (typeof obj["headword_cefr"] !== "string") return null;
if (!CEFR_SET.has(obj["headword_cefr"])) return null;
if (
typeof obj["translation_cefr"] !== "object" ||
obj["translation_cefr"] === null
)
return null;
const translationCefr = obj["translation_cefr"] as Record<string, unknown>;
// Verify all validated translations have a CEFR vote
for (const [lang, words] of validatedTranslations.entries()) {
const votes = translationCefr[lang] as Record<string, string> | undefined;
if (!votes) return null;
for (const word of words) {
if (!votes[word] || !CEFR_SET.has(votes[word])) return null;
}
}
return {
headword_cefr: obj["headword_cefr"],
translation_cefr: translationCefr as Partial<
Record<SupportedLanguageCode, Record<string, string>>
>,
};
} catch {
return null;
}
}
// ── LLM call ──────────────────────────────────────────────────────────────────
async function callLlm(
prompt: string,
provider: ProviderConfig,
): Promise<string> {
currentCallController = new AbortController();
const timeout = setTimeout(() => currentCallController?.abort(), 120_000);
let response: Response;
try {
response = await fetch(`${provider.baseURL}/chat/completions`, {
method: "POST",
signal: currentCallController.signal,
headers: {
"Content-Type": "application/json",
Authorization: `Bearer ${provider.apiKey}`,
},
body: JSON.stringify({
model: provider.model,
max_tokens: provider.maxTokens,
messages: [{ role: "user", content: prompt }],
temperature: 0.1,
}),
});
} finally {
clearTimeout(timeout);
currentCallController = null;
}
if (!response.ok) {
throw new Error(`LLM API error: ${response.status} ${response.statusText}`);
}
const data = (await response.json()) as {
choices?: { message?: { content?: string } }[];
};
const content = data.choices?.[0]?.message?.content;
if (!content) throw new Error("LLM returned empty response");
return content
.replace(/```json\n?/g, "")
.replace(/```\n?/g, "")
.trim();
}
// ── Status helpers ────────────────────────────────────────────────────────────
function getSubStageStatus(
entryId: number,
modelName: string,
stage: SubStage,
): "complete" | "needs_review" | "pending" {
const db = openDb();
const row = db
.prepare(
`SELECT status FROM run_status
WHERE entry_id = ? AND model_name = ? AND stage = ?`,
)
.get(entryId, modelName, stage) as { status: string } | undefined;
db.close();
if (!row) return "pending";
if (row.status === "complete") return "complete";
if (row.status === "needs_review") return "needs_review";
return "pending";
}
function markSubStage(
entryId: number,
modelName: string,
stage: SubStage,
status: "complete" | "needs_review",
): void {
const db = openDb();
db.prepare(
`INSERT INTO run_status (entry_id, model_name, stage, status)
VALUES (?, ?, ?, ?)
ON CONFLICT (entry_id, model_name, stage)
DO UPDATE SET status = ?, updated_at = datetime('now')`,
).run(entryId, modelName, stage, status, status);
db.close();
}
// ── Write helpers ─────────────────────────────────────────────────────────────
function writeGloss(
entryId: number,
modelName: string,
result: GlossResult,
): void {
if (result.status === "improved") {
const db = openDb();
db.prepare(
`INSERT INTO generated_glosses (entry_id, model_name, text)
VALUES (?, ?, ?)
ON CONFLICT (entry_id, model_name) DO NOTHING`,
).run(entryId, modelName, result.gloss);
db.close();
}
}
function writeExample(
entryId: number,
modelName: string,
result: ExampleResult,
): void {
if (result.status === "improved") {
const db = openDb();
db.prepare(
`INSERT INTO generated_examples (entry_id, model_name, text)
VALUES (?, ?, ?)
ON CONFLICT (entry_id, model_name) DO NOTHING`,
).run(entryId, modelName, result.example);
db.close();
}
}
function writeTranslations(
entryId: number,
modelName: string,
result: TranslationResult,
translations: TranslationRow[],
): void {
const db = openDb();
db.transaction(() => {
// Write rejections
for (const t of translations) {
const vote = result.translations[t.target_lang]?.[t.word];
if (vote === "reject") {
db.prepare(
`INSERT INTO model_translation_rejections (translation_id, model_name)
VALUES (?, ?)
ON CONFLICT (translation_id, model_name) DO NOTHING`,
).run(t.id, modelName);
}
}
// Write generated translations
if (result.generated) {
for (const [lang, word] of Object.entries(result.generated)) {
db.prepare(
`INSERT INTO generated_translations (entry_id, model_name, target_lang, word)
VALUES (?, ?, ?, ?)
ON CONFLICT (entry_id, model_name, target_lang) DO NOTHING`,
).run(entryId, modelName, lang, word);
}
}
})();
db.close();
}
function writeCefr(
entryId: number,
modelName: string,
result: CefrResult,
translations: TranslationRow[],
): void {
const db = openDb();
db.transaction(() => {
// Headword CEFR
db.prepare(
`INSERT INTO model_entry_cefr_votes (entry_id, model_name, cefr_level)
VALUES (?, ?, ?)
ON CONFLICT (entry_id, model_name) DO NOTHING`,
).run(entryId, modelName, result.headword_cefr);
// Translation CEFR votes
for (const t of translations) {
const level = result.translation_cefr[t.target_lang]?.[t.word];
if (level && CEFR_SET.has(level)) {
db.prepare(
`INSERT INTO model_translation_cefr_votes (translation_id, model_name, cefr_level)
VALUES (?, ?, ?)
ON CONFLICT (translation_id, model_name) DO NOTHING`,
).run(t.id, modelName, level);
}
}
})();
db.close();
}
// ── Progress ──────────────────────────────────────────────────────────────────
function updateProgress(
processed: number,
needsReview: number,
total: number,
llmMs: number,
startTime: number,
): void {
const totalProcessed = processed + needsReview;
const pct = ((totalProcessed / total) * 100).toFixed(1);
const elapsed = (Date.now() - startTime) / 1000;
const rate = elapsed > 0 ? totalProcessed / elapsed : 0;
const remaining = rate > 0 ? (total - totalProcessed) / rate : 0;
const eta =
remaining === 0
? "calculating..."
: remaining < 60
? `${Math.round(remaining)}s`
: `${Math.round(remaining / 60)}m`;
const totalElapsedStr =
elapsed < 60
? `${Math.round(elapsed)}s`
: `${Math.floor(elapsed / 60)}m ${Math.round(elapsed % 60)}s`;
process.stdout.write(
`\r ${totalProcessed}/${total} (${pct}%) — entry: ${(llmMs / 1000).toFixed(1)}s — total: ${totalElapsedStr} — ETA: ${eta} `,
);
}
// ── Main enrich function ──────────────────────────────────────────────────────
export async function enrich(
provider: ProviderConfig,
): Promise<{ processed: number; skipped: number; needsReview: number }> {
registerEnrichShutdown();
const db = openDb();
const allEntries = db
.prepare(`SELECT * FROM entries WHERE language = 'en'`)
.all() as EntryRow[];
// An entry is fully complete when all 4 sub-stages are complete
const completeEntries = db
.prepare(
`SELECT entry_id FROM run_status
WHERE model_name = ? AND stage = 'round1_gloss'
AND status = 'complete'`,
)
.all(provider.name) as { entry_id: number }[];
const completeIds = new Set(completeEntries.map((r) => r.entry_id));
const pending = allEntries.filter((e) => !completeIds.has(e.id)).slice(0, 50);
db.close();
console.log(`\n Model: ${provider.name}`);
console.log(` Total entries: ${allEntries.length.toLocaleString()}`);
console.log(` Already complete: ${completeIds.size.toLocaleString()}`);
console.log(` Pending: ${pending.length.toLocaleString()}`);
if (pending.length === 0) {
console.log(" Nothing to process.");
return { processed: 0, skipped: completeIds.size, needsReview: 0 };
}
let processedCount = 0;
let needsReviewCount = 0;
let llmMs = 0;
const startTime = Date.now();
for (const entry of pending) {
if (shutdownRequested) break;
const db2 = openDb();
const translations = db2
.prepare(
`SELECT id, target_lang, word FROM translations WHERE entry_id = ? AND source = 'kaikki'`,
)
.all(entry.id) as TranslationRow[];
db2.close();
let entryFailed = false;
// ── Sub-stage 1: Gloss ────────────────────────────────────────────────────
let verifiedGloss = entry.gloss ?? "";
if (
getSubStageStatus(entry.id, provider.name, "round1_gloss") !== "complete"
) {
try {
const llmStart = Date.now();
const raw = await callLlm(buildGlossPrompt(entry), provider);
llmMs = Date.now() - llmStart;
const result = validateGloss(raw);
if (!result) {
markSubStage(entry.id, provider.name, "round1_gloss", "needs_review");
console.warn(
`\n needs_review: entry ${entry.id} round1_gloss — invalid response`,
);
entryFailed = true;
} else {
writeGloss(entry.id, provider.name, result);
if (result.status === "improved") verifiedGloss = result.gloss;
markSubStage(entry.id, provider.name, "round1_gloss", "complete");
}
} catch (err) {
llmMs = 0;
const message = err instanceof Error ? err.message : String(err);
markSubStage(entry.id, provider.name, "round1_gloss", "needs_review");
console.warn(
`\n needs_review: entry ${entry.id} round1_gloss — ${message}`,
);
entryFailed = true;
}
}
if (entryFailed) {
needsReviewCount++;
updateProgress(
processedCount,
needsReviewCount,
pending.length,
llmMs,
startTime,
);
continue;
}
/*
// ── Sub-stages 2, 3, 4 — not yet active ──────────────────────────────────
// ── Sub-stage 2: Example ──────────────────────────────────────────────────
if (
getSubStageStatus(entry.id, provider.name, "round1_example") !==
"complete"
) {
try {
const llmStart = Date.now();
const raw = await callLlm(
buildExamplePrompt(entry, verifiedGloss),
provider,
);
llmMs = Date.now() - llmStart;
const result = validateExample(raw);
if (!result) {
markSubStage(
entry.id,
provider.name,
"round1_example",
"needs_review",
);
console.warn(
`\n needs_review: entry ${entry.id} round1_example — invalid response`,
);
entryFailed = true;
} else {
writeExample(entry.id, provider.name, result);
markSubStage(entry.id, provider.name, "round1_example", "complete");
}
} catch (err) {
llmMs = 0;
const message = err instanceof Error ? err.message : String(err);
markSubStage(entry.id, provider.name, "round1_example", "needs_review");
console.warn(
`\n needs_review: entry ${entry.id} round1_example — ${message}`,
);
entryFailed = true;
}
}
if (entryFailed) {
needsReviewCount++;
updateProgress(
processedCount,
needsReviewCount,
pending.length,
llmMs,
startTime,
);
continue;
}
// ── Sub-stage 3: Translations ─────────────────────────────────────────────
const validatedTranslations = new Map<SupportedLanguageCode, string[]>();
if (
getSubStageStatus(entry.id, provider.name, "round1_translations") !==
"complete"
) {
try {
const llmStart = Date.now();
const raw = await callLlm(
buildTranslationsPrompt(entry, translations, verifiedGloss),
provider,
);
llmMs = Date.now() - llmStart;
const result = validateTranslations(raw, translations);
if (!result) {
markSubStage(
entry.id,
provider.name,
"round1_translations",
"needs_review",
);
console.warn(
`\n needs_review: entry ${entry.id} round1_translations — invalid response`,
);
entryFailed = true;
} else {
writeTranslations(entry.id, provider.name, result, translations);
markSubStage(
entry.id,
provider.name,
"round1_translations",
"complete",
);
// Build validated translations map for CEFR sub-stage
// Include kaikki translations that were ok'd + generated translations
for (const t of translations) {
const vote = result.translations[t.target_lang]?.[t.word];
if (vote === "ok") {
if (!validatedTranslations.has(t.target_lang)) {
validatedTranslations.set(t.target_lang, []);
}
validatedTranslations.get(t.target_lang)!.push(t.word);
}
}
if (result.generated) {
for (const [lang, word] of Object.entries(result.generated)) {
const l = lang as SupportedLanguageCode;
if (!validatedTranslations.has(l))
validatedTranslations.set(l, []);
validatedTranslations.get(l)!.push(word);
}
}
}
} catch (err) {
llmMs = 0;
const message = err instanceof Error ? err.message : String(err);
markSubStage(
entry.id,
provider.name,
"round1_translations",
"needs_review",
);
console.warn(
`\n needs_review: entry ${entry.id} round1_translations — ${message}`,
);
entryFailed = true;
}
} else {
// Already complete — rebuild validated translations from db
const db3 = openDb();
const rejections = new Set(
(
db3
.prepare(
`SELECT translation_id FROM model_translation_rejections WHERE model_name = ?`,
)
.all(provider.name) as { translation_id: number }[]
).map((r) => r.translation_id),
);
for (const t of translations) {
if (!rejections.has(t.id)) {
if (!validatedTranslations.has(t.target_lang)) {
validatedTranslations.set(t.target_lang, []);
}
validatedTranslations.get(t.target_lang)!.push(t.word);
}
}
const generated = db3
.prepare(
`SELECT target_lang, word FROM generated_translations WHERE entry_id = ? AND model_name = ?`,
)
.all(entry.id, provider.name) as {
target_lang: SupportedLanguageCode;
word: string;
}[];
for (const g of generated) {
if (!validatedTranslations.has(g.target_lang))
validatedTranslations.set(g.target_lang, []);
validatedTranslations.get(g.target_lang)!.push(g.word);
}
db3.close();
}
if (entryFailed) {
needsReviewCount++;
updateProgress(
processedCount,
needsReviewCount,
pending.length,
llmMs,
startTime,
);
continue;
}
// ── Sub-stage 4: CEFR ─────────────────────────────────────────────────────
if (
getSubStageStatus(entry.id, provider.name, "round1_cefr") !== "complete"
) {
try {
const llmStart = Date.now();
const raw = await callLlm(
buildCefrPrompt(entry, verifiedGloss, validatedTranslations),
provider,
);
llmMs = Date.now() - llmStart;
const result = validateCefr(raw, validatedTranslations);
if (!result) {
markSubStage(entry.id, provider.name, "round1_cefr", "needs_review");
console.warn(
`\n needs_review: entry ${entry.id} round1_cefr — invalid response`,
);
needsReviewCount++;
} else {
// Get translation rows for validated words only
const validatedRows = translations.filter((t) => {
return validatedTranslations.get(t.target_lang)?.includes(t.word);
});
writeCefr(entry.id, provider.name, result, validatedRows);
markSubStage(entry.id, provider.name, "round1_cefr", "complete");
processedCount++;
}
} catch (err) {
llmMs = 0;
const message = err instanceof Error ? err.message : String(err);
markSubStage(entry.id, provider.name, "round1_cefr", "needs_review");
console.warn(
`\n needs_review: entry ${entry.id} round1_cefr — ${message}`,
);
needsReviewCount++;
}
} else {
processedCount++;
}
*/
processedCount++;
updateProgress(
processedCount,
needsReviewCount,
pending.length,
llmMs,
startTime,
);
}
process.stdout.write("\n");
const totalMs = Date.now() - startTime;
const totalMin = Math.floor(totalMs / 60_000);
const totalSec = Math.round((totalMs % 60_000) / 1000);
console.log(` Total time: ${totalMin}m ${totalSec}s`);
console.log(
` Avg per entry: ${(totalMs / Math.max(processedCount + needsReviewCount, 1) / 1000).toFixed(1)}s`,
);
console.log(` Processed: ${processedCount.toLocaleString()}`);
console.log(` Needs review: ${needsReviewCount.toLocaleString()}`);
return {
processed: processedCount,
skipped: completeIds.size,
needsReview: needsReviewCount,
};
}

230
data-pipeline/tests/validation/db-import.validation.test.ts
View file

@ -1,230 +0,0 @@
import fs from "node:fs/promises";
import path from "node:path";
import { describe, it, expect, beforeAll } from "vitest";
import { SUPPORTED_LANGUAGE_CODES } from "@lila/shared";
import type { SupportedLanguageCode, SupportedPos } from "@lila/shared";
// ── Types ─────────────────────────────────────────────────────────────────────
type ExtractedSense = {
headword: string;
language: SupportedLanguageCode;
pos: SupportedPos;
sense_index: number;
gloss: string | null;
examples: string[];
translations: {
target_lang: SupportedLanguageCode;
word: string;
sense_hint: string | null;
}[];
};
// ── Paths ─────────────────────────────────────────────────────────────────────
const DB_PATH = path.resolve("db/pipeline.db");
const OUTPUT_DIR = path.resolve("stage-1-extract/output");
// ── Helpers ───────────────────────────────────────────────────────────────────
async function dbExists(): Promise<boolean> {
try {
await fs.access(DB_PATH);
return true;
} catch {
return false;
}
}
// ── Tests ─────────────────────────────────────────────────────────────────────
describe("pipeline.db — import validation", () => {
let db: import("better-sqlite3").Database;
let expectedEntriesByLang: Map<SupportedLanguageCode, number>;
let expectedTotalTranslations: number;
beforeAll(async () => {
if (!(await dbExists())) return;
const Database = (await import("better-sqlite3")).default;
db = new Database(DB_PATH, { readonly: true });
db.pragma("foreign_keys = ON");
expectedEntriesByLang = new Map();
expectedTotalTranslations = 0;
for (const lang of SUPPORTED_LANGUAGE_CODES) {
try {
const raw = await fs.readFile(
path.join(OUTPUT_DIR, `${lang}.json`),
"utf-8",
);
const senses = JSON.parse(raw) as ExtractedSense[];
expectedEntriesByLang.set(lang, senses.length);
if (lang === "en") {
for (const sense of senses) {
expectedTotalTranslations += sense.translations.length;
}
}
} catch {
expectedEntriesByLang.set(lang, 0);
}
}
}, 30_000);
it("pipeline.db exists — skipping all tests if not", async () => {
const exists = await dbExists();
if (!exists) {
console.warn(
"\n pipeline.db not found — run pnpm db:init and pnpm db:import first\n",
);
}
expect(exists).toBe(true);
});
it("entry count per language matches source files", () => {
if (!db) return;
const errors: string[] = [];
for (const lang of SUPPORTED_LANGUAGE_CODES) {
const expected = expectedEntriesByLang.get(lang) ?? 0;
const row = db
.prepare("SELECT COUNT(*) as count FROM entries WHERE language = ?")
.get(lang) as { count: number };
if (row.count !== expected) {
errors.push(`${lang}: expected ${expected} entries, got ${row.count}`);
}
}
expect(errors, `\n${errors.join("\n")}`).toHaveLength(0);
});
it("translation count matches source files plus reverse links", () => {
if (!db) return;
const row = db
.prepare("SELECT COUNT(*) as count FROM translations")
.get() as { count: number };
const reverseLinks = db
.prepare(
"SELECT COUNT(*) as count FROM translations WHERE source = 'reverse_link'",
)
.get() as { count: number };
expect(row.count).toBe(expectedTotalTranslations + reverseLinks.count);
});
it("every translation references a valid entry", () => {
if (!db) return;
const rows = db
.prepare(
`SELECT t.id, t.entry_id
FROM translations t
LEFT JOIN entries e ON e.id = t.entry_id
WHERE e.id IS NULL`,
)
.all() as { id: number; entry_id: number }[];
const errors = rows.map(
(r) => `translation ${r.id}: references missing entry ${r.entry_id}`,
);
expect(errors, `\n${errors.join("\n")}`).toHaveLength(0);
});
it("every entry has a valid language code", () => {
if (!db) return;
const validLangs = SUPPORTED_LANGUAGE_CODES.map((l) => `'${l}'`).join(", ");
const rows = db
.prepare(
`SELECT id, headword, language FROM entries
WHERE language NOT IN (${validLangs})`,
)
.all() as { id: number; headword: string; language: string }[];
const errors = rows.map(
(r) => `entry ${r.id} "${r.headword}": invalid language "${r.language}"`,
);
expect(errors, `\n${errors.join("\n")}`).toHaveLength(0);
});
it("every entry has a valid pos", () => {
if (!db) return;
const rows = db
.prepare(
`SELECT id, headword, pos FROM entries
WHERE pos NOT IN ('noun', 'verb', 'adjective', 'adverb')`,
)
.all() as { id: number; headword: string; pos: string }[];
const errors = rows.map(
(r) => `entry ${r.id} "${r.headword}": invalid pos "${r.pos}"`,
);
expect(errors, `\n${errors.join("\n")}`).toHaveLength(0);
});
it("sense_index is unique per headword, language, pos", () => {
if (!db) return;
const rows = db
.prepare(
`SELECT headword, language, pos, sense_index, COUNT(*) as c
FROM entries
GROUP BY headword, language, pos, sense_index
HAVING c > 1`,
)
.all() as {
headword: string;
language: string;
pos: string;
sense_index: number;
c: number;
}[];
const errors = rows.map(
(r) =>
`"${r.headword}" (${r.language} ${r.pos}): duplicate sense_index ${r.sense_index} (${r.c} rows)`,
);
expect(errors, `\n${errors.join("\n")}`).toHaveLength(0);
});
it("non-English entries have no Kaikki translations", () => {
if (!db) return;
const nonEnLangs = SUPPORTED_LANGUAGE_CODES.filter((l) => l !== "en")
.map((l) => `'${l}'`)
.join(", ");
const rows = db
.prepare(
`SELECT e.headword, e.language, COUNT(t.id) as c
FROM entries e
JOIN translations t ON t.entry_id = e.id
WHERE e.language IN (${nonEnLangs})
AND t.source = 'kaikki'
GROUP BY e.id`,
)
.all() as { headword: string; language: string; c: number }[];
const errors = rows.map(
(r) =>
`"${r.headword}" (${r.language}): unexpected ${r.c} Kaikki translations`,
);
expect(errors, `\n${errors.join("\n")}`).toHaveLength(0);
});
it("all Kaikki translation target languages are supported and not English", () => {
if (!db) return;
const validLangs = SUPPORTED_LANGUAGE_CODES.map((l) => `'${l}'`).join(", ");
const rows = db
.prepare(
`SELECT t.id, t.target_lang
FROM translations t
WHERE t.source = 'kaikki'
AND (t.target_lang NOT IN (${validLangs}) OR t.target_lang = 'en')`,
)
.all() as { id: number; target_lang: string }[];
const errors = rows.map(
(r) => `translation ${r.id}: invalid target_lang "${r.target_lang}"`,
);
expect(errors, `\n${errors.join("\n")}`).toHaveLength(0);
});
});

192
data-pipeline/tests/validation/stage-1.validation.test.ts
View file

@ -1,192 +0,0 @@
import fs from "node:fs/promises";
import path from "node:path";
import { describe, it, expect, beforeAll } from "vitest";
import { SUPPORTED_LANGUAGE_CODES, SUPPORTED_POS } from "@lila/shared";
import type { SupportedLanguageCode, SupportedPos } from "@lila/shared";
// ── Types ─────────────────────────────────────────────────────────────────────
type ExtractedSense = {
headword: string;
language: SupportedLanguageCode;
pos: SupportedPos;
sense_index: number;
gloss: string | null;
examples: string[];
translations: {
target_lang: SupportedLanguageCode;
word: string;
sense_hint: string | null;
}[];
};
// ── Paths ─────────────────────────────────────────────────────────────────────
const OUTPUT_DIR = path.resolve("stage-1-extract/output");
// ── Tests ─────────────────────────────────────────────────────────────────────
describe("stage 1 — Kaikki extraction output validation", () => {
const sensesByLang = new Map<SupportedLanguageCode, ExtractedSense[]>();
beforeAll(async () => {
for (const lang of SUPPORTED_LANGUAGE_CODES) {
const filePath = path.join(OUTPUT_DIR, `${lang}.json`);
const raw = await fs.readFile(filePath, "utf-8");
sensesByLang.set(lang, JSON.parse(raw) as ExtractedSense[]);
}
}, 30_000);
it("all five language output files exist", async () => {
const errors: string[] = [];
for (const lang of SUPPORTED_LANGUAGE_CODES) {
try {
await fs.access(path.join(OUTPUT_DIR, `${lang}.json`));
} catch {
errors.push(`missing: ${lang}.json`);
}
}
expect(errors, `\n${errors.join("\n")}`).toHaveLength(0);
});
it("every language file is a non-empty array", () => {
const errors: string[] = [];
for (const lang of SUPPORTED_LANGUAGE_CODES) {
const senses = sensesByLang.get(lang)!;
if (!Array.isArray(senses)) errors.push(`${lang}: not an array`);
else if (senses.length === 0) errors.push(`${lang}: empty array`);
}
expect(errors, `\n${errors.join("\n")}`).toHaveLength(0);
});
it("every sense has required fields", () => {
const errors: string[] = [];
for (const lang of SUPPORTED_LANGUAGE_CODES) {
for (const sense of sensesByLang.get(lang)!) {
if (!sense.headword) errors.push(`${lang}: sense missing headword`);
if (!sense.language)
errors.push(`${lang} ${sense.headword}: missing language`);
if (!sense.pos) errors.push(`${lang} ${sense.headword}: missing pos`);
if (sense.sense_index === undefined)
errors.push(`${lang} ${sense.headword}: missing sense_index`);
if (!Array.isArray(sense.examples))
errors.push(`${lang} ${sense.headword}: examples not an array`);
if (!Array.isArray(sense.translations))
errors.push(`${lang} ${sense.headword}: translations not an array`);
}
}
expect(errors, `\n${errors.join("\n")}`).toHaveLength(0);
});
it("every sense has a valid pos", () => {
const errors: string[] = [];
const validPos = new Set(SUPPORTED_POS);
for (const lang of SUPPORTED_LANGUAGE_CODES) {
for (const sense of sensesByLang.get(lang)!) {
if (!validPos.has(sense.pos)) {
errors.push(`${lang} ${sense.headword}: invalid pos "${sense.pos}"`);
}
}
}
expect(errors, `\n${errors.join("\n")}`).toHaveLength(0);
});
it("every sense language code matches its file", () => {
const errors: string[] = [];
for (const lang of SUPPORTED_LANGUAGE_CODES) {
for (const sense of sensesByLang.get(lang)!) {
if (sense.language !== lang) {
errors.push(
`${lang} ${sense.headword}: language field "${sense.language}" does not match file`,
);
}
}
}
expect(errors, `\n${errors.join("\n")}`).toHaveLength(0);
});
it("no abbreviation senses in output", () => {
const errors: string[] = [];
for (const lang of SUPPORTED_LANGUAGE_CODES) {
for (const sense of sensesByLang.get(lang)!) {
if (sense.gloss?.toLowerCase().startsWith("abbreviation of")) {
errors.push(
`${lang} ${sense.headword}: abbreviation sense not filtered`,
);
}
}
}
expect(errors, `\n${errors.join("\n")}`).toHaveLength(0);
});
it("English senses all have at least one translation", () => {
const errors: string[] = [];
for (const sense of sensesByLang.get("en")!) {
if (sense.translations.length === 0) {
errors.push(
`en ${sense.headword} (sense ${sense.sense_index}): no translations`,
);
}
}
expect(errors, `\n${errors.join("\n")}`).toHaveLength(0);
});
it("non-English senses have no translations", () => {
const errors: string[] = [];
for (const lang of SUPPORTED_LANGUAGE_CODES) {
if (lang === "en") continue;
for (const sense of sensesByLang.get(lang)!) {
if (sense.translations.length > 0) {
errors.push(
`${lang} ${sense.headword}: unexpected translations in non-English file`,
);
}
}
}
expect(errors, `\n${errors.join("\n")}`).toHaveLength(0);
});
it("all translation target languages are supported and not English", () => {
const errors: string[] = [];
const validLangs = new Set(SUPPORTED_LANGUAGE_CODES);
for (const sense of sensesByLang.get("en")!) {
for (const t of sense.translations) {
if (!validLangs.has(t.target_lang)) {
errors.push(
`en ${sense.headword}: unsupported translation language "${t.target_lang}"`,
);
}
if (t.target_lang === "en") {
errors.push(
`en ${sense.headword}: translation to same language "en"`,
);
}
if (!t.word?.trim()) {
errors.push(
`en ${sense.headword}: empty translation word for ${t.target_lang}`,
);
}
}
}
expect(errors, `\n${errors.join("\n")}`).toHaveLength(0);
});
it("sense_index is unique per headword and pos within each language", () => {
const errors: string[] = [];
for (const lang of SUPPORTED_LANGUAGE_CODES) {
const seen = new Map<string, Set<number>>();
for (const sense of sensesByLang.get(lang)!) {
const key = `${sense.headword}|${sense.pos}`;
if (!seen.has(key)) seen.set(key, new Set());
const indexes = seen.get(key)!;
if (indexes.has(sense.sense_index)) {
errors.push(
`${lang} ${sense.headword} (${sense.pos}): duplicate sense_index ${sense.sense_index}`,
);
}
indexes.add(sense.sense_index);
}
}
expect(errors, `\n${errors.join("\n")}`).toHaveLength(0);
});
});

342
documentation/pipeline/ENGLISH_NOUNS.md Normal file
View file

@ -0,0 +1,342 @@
# english nouns
## step 1a
get freqency source list + verify lemmatization + confirm language/POS tagging
example:
house
bank
asdf
## step 1b
transform list into JSON (headword, language, POS)
example:
```json
{
"headword": "house",
"language": "en",
"pos": "noun"
},
{
"headword": "bank",
"language": "en",
"pos": "noun"
},
{
"headword": "asdf",
"language": "en",
"pos": "noun"
}
```
---
## step 2a
check existence in kaikki (eg. is there a english noun "asdf" in kaikki, if not put it separate list)
example output:
```json
{
"headword": "house",
"language": "en",
"pos": "noun"
},
{
"headword": "bank",
"language": "en",
"pos": "noun"
}
```
miss list (triage queue, not trash! contains junk, slipped inflections, and real Kaikki gaps, gets handled separately, never auto-drop!)
```json
{ "headword": "asdf", "language": "en", "pos": "noun" }
```
## step 2b
for each sense, extract: glosses, translations (with gender), examples
id will be generated via: `headword:lang:pos:sense_index`
words without glosses will be dropped! (put in special list)
example output:
```json
{
"id": "house:en:noun:0",
"headword": "house",
"language": "en",
"pos": "noun",
"glosses": ["A building for human habitation."],
"examples": ["They bought a house in the city."],
"translations": {
"de": [{ "word": "Haus", "gender": "neuter" }],
"it": [{ "word": "casa", "gender": "feminine" }],
"es": [{ "word": "casa", "gender": "feminine" }],
"fr": [{ "word": "maison", "gender": "feminine" }]
}
},
{
"id": "house:en:noun:1",
"headword": "house",
"language": "en",
"pos": "noun",
"glosses": ["A noble family or lineage."],
"examples": ["The House of Tudor ruled England."],
"translations": {
"de": [
{ "word": "Adelsgeschlecht", "gender": "neuter" },
{ "word": "Haus", "gender": "neuter" }
]
}
},
{
"id": "bank:en:noun:0",
"headword": "bank",
"language": "en",
"pos": "noun",
"glosses": ["An institution where one can place and borrow money."],
"examples": ["She deposited her paycheck at the bank."],
"translations": {
"de": [{ "word": "Bank", "gender": "feminine" }],
"it": [{ "word": "banca", "gender": "feminine" }],
"es": [{ "word": "banco", "gender": "masculine" }],
"fr": [{ "word": "banque", "gender": "feminine" }]
}
},
{
"id": "bank:en:noun:1",
"headword": "bank",
"language": "en",
"pos": "noun",
"glosses": ["The land alongside a river or lake."],
"examples": ["They picnicked on the bank of the river."],
"translations": {
"de": [{ "word": "Ufer", "gender": "neuter" }],
"it": [{ "word": "riva", "gender": "feminine" }],
"es": [{ "word": "orilla", "gender": "feminine" }],
"fr": [{ "word": "rive", "gender": "feminine" }]
}
},
{
"id": "bank:en:noun:2",
"headword": "bank",
"language": "en",
"pos": "noun",
"glosses": ["A collection or store of something held in reserve."],
"examples": ["The hospital keeps a blood bank."],
"translations": {
"de": [{ "word": "Bank", "gender": "feminine" }]
}
}
```
## step 2c
fill gaps Kaikki left (LLM, 3 models, generate-then-vote)
takes 2b's partial cards. 3 different-family models (qwen, llama, gemma) generate, then vote.
separate focused sub-passes, one field at a time — never combined:
- **translations** → per missing language, generated with the gloss as context. generate → vote. no agreement → gap stays (cloud audit later)
- **examples** → for senses with no Kaikki example. generate → verify-vote ("is this a valid example of the gloss?"), since freeform sentences never exact-match. no agreement → no example (card still valid) no tiebreak: no agreement leaves the gap, never escalates to more models. vote records stored in pipeline.db for the cloud audit.
- **invariant**: the gloss is never LLM-generated — translations are filled, examples generated, difficulty graded, but the gloss must always be Kaikki's (no gloss → sense dropped in 2b)
example input:
```json
{
"id": "harbor:en:noun:0",
"headword": "harbor",
"language": "en",
"pos": "noun",
"glosses": ["A sheltered area of water where ships can dock safely."],
"examples": [],
"translations": { "de": [{ "word": "Hafen", "gender": "masculine" }] }
}
```
next substage is the translation generation:
```json
{
"id": "harbor:en:noun:0",
"headword": "harbor",
"language": "en",
"pos": "noun",
"glosses": ["A sheltered area of water where ships can dock safely."],
"examples": [],
"translations": {
"de": [{ "word": "Hafen", "gender": "masculine" }],
"it": [{ "word": "porto", "gender": null }],
"es": [{ "word": "puerto", "gender": null }],
"fr": [{ "word": "port", "gender": null }]
}
}
```
next substage is the gender modification (by looking up kaikki for it/es/fr or corresponding wiktionary, fill only nulls, not touching existing genders):
**runs only after all genders are present**
```json
{
"id": "harbor:en:noun:0",
"headword": "harbor",
"language": "en",
"pos": "noun",
"glosses": ["A sheltered area of water where ships can dock safely."],
"examples": [],
"translations": {
"de": [{ "word": "Hafen", "gender": "masculine" }],
"it": [{ "word": "porto", "gender": "masculine" }],
"es": [{ "word": "puerto", "gender": "masculine" }],
"fr": [{ "word": "port", "gender": "masculine" }]
}
}
```
next substage is the example generation:
```json
{
"id": "harbor:en:noun:0",
"headword": "harbor",
"language": "en",
"pos": "noun",
"glosses": ["A sheltered area of water where ships can dock safely."],
"examples": ["The fishing boats returned to the harbor at dusk."],
"translations": {
"de": [{ "word": "Hafen", "gender": "masculine" }],
"it": [{ "word": "porto", "gender": "masculine" }],
"es": [{ "word": "puerto", "gender": "masculine" }],
"fr": [{ "word": "port", "gender": "masculine" }]
}
}
```
---
## step 3
adding difficulty level
CEFR is mapped to three buckets:
A1/A2 → easy
B1/B2 → intermediate
C1/C2 → hard
depending on number of senses per headword:
- One sense → derive difficulty from the CEFRLex distribution (first CEFR level crossing a frequency threshold, mapped to a bucket, not the peak), for the four covered languages (en/de/es/fr). Deterministic, no LLM
- Multiple senses → the three local LLMs grade each sense (easy/intermediate/hard) from the gloss. CEFRLex not involved.
- No CEFRLex entry at all (Italian, or single-sense word that's missing) → LLMs grade from gloss
to get coverage (not quality), 3 local llms are going to be used: gemma, qwen and llama
on the llm votes:
- **Majority agrees (3-0 or 2-1)** → ship the majority value.
- **3-way split (all three differ)** → no consensus → exclude the card to the review queue. Not shipped.
example output:
```json
{
"id": "house:en:noun:0",
"headword": "house",
"language": "en",
"pos": "noun",
"difficulty_level": "easy",
"glosses": ["A building for human habitation."],
"examples": ["They bought a house in the city."],
"translations": {
"de": [{ "word": "Haus", "gender": "neuter" }],
"it": [{ "word": "casa", "gender": "feminine" }],
"es": [{ "word": "casa", "gender": "feminine" }],
"fr": [{ "word": "maison", "gender": "feminine" }]
}
},
{
"id": "house:en:noun:1",
"headword": "house",
"language": "en",
"pos": "noun",
"difficulty_level": "hard",
"glosses": ["A noble family or lineage."],
"examples": ["The House of Tudor ruled England."],
"translations": {
"de": [
{ "word": "Adelsgeschlecht", "gender": "neuter" },
{ "word": "Haus", "gender": "neuter" }
]
}
}
```
**important**: `difficulty_level` is mandatory on every shipped card (the user sets difficulty before a game)
This is coverage, not final quality: three local models make every shipped card's difficulty present and plausible, but not guaranteed correct
no cefr list: needs to be graded by online llms (see notes)
---
## coverage report (trial deliverable)
- not a data stage => reads the finished cards and summarises them
- the trial (english + italian nouns, local models) runs steps 1a→3 then emits a report
### pipeline health — did it run?
- input lemmas → output cards (ratio; multi-sense makes cards > lemmas)
- dropped per stage + why: 2a misses (junk/inflection/gap), 2b no-gloss drops, 3 no-consensus exclusions
- parse failures / errors
### coverage — how complete?
- translation completeness: all 4 langs vs gaps, per language
- examples: from Kaikki vs LLM-generated vs none
- gender: translations still null after Wiktionary fill
- difficulty distribution per bucket (lopsided = broken CEFRLex threshold or bad grading)
### local good enough? — model agreement
- difficulty votes: 3-0 / 2-1 / 3-way split rates (high 3-way = locals can't do it → need cloud)
- translation gap-fill: agreement rate, gaps left unfilled
- → go/no-go on local-for-launch
### rent vs API? — workload volume
- total LLM calls across 2c + 3 (× per-token rate = API cost)
- cards needing LLM vs handled deterministically (english: high deterministic; italian: ~0, no CEFRLex)
- per-language call counts → scale english (low) and italian (high) to estimate the middle three
### review backlog — what's deferred
- miss-list size + composition, per language
- no-consensus exclusions (cloud-audit queue size)
- unfilled translation gaps (also cloud-audit work)
**framing**: english = optimistic floor (rich Kaikki, CEFRLex exists, models strongest).
italian = pessimistic ceiling (no CEFRLex, thinner Kaikki). all five languages sit between.
read to decide, not to archive.
---
Notes:
- the miss list needs to get verified/re-worked later on
- the kaikki data contains ipas and links to audio files, add them later if needed, they are not needed now"
- use online llms to set the difficulty(cefr) of the not shipped words
- add plurals from kaikki/wiktionary
- postgres sync to prod db

41
documentation/pipeline/TRIAL_IMPLEMENTATION_ROADMAP.md Normal file
View file

@ -0,0 +1,41 @@
# trial run — implementation roadmap (pure vertical)
goal: english + italian nouns through the full pipeline on local models, emit
coverage report. build a thin end-to-end slice first, then widen each pass.
## phase 0 — foundation
- confirm @lila/shared types (lang codes, POS) exist and match
- package.json deps for what phase 1 needs
- one hardcoded test word to carry through the slice (e.g. "house")
## phase 1 — thin vertical slice (ONE word, 1a→3, crudest possible)
goal: prove a single card can travel the whole pipeline and come out the end.
allowed to be ugly — hardcode, skip voting, one model, fake CEFRLex.
- 1a: one lemma record, by hand or trivial read
- 2a: look it up in Kaikki, confirm it exists
- 2b: extract its senses → card(s) with gloss
- 2c: fill one missing translation with ONE local model, no voting
- 3: assign a difficulty crudely (even hardcoded "easy")
- OUT: one finished card. the pipeline has a shape.
## phase 2 — widen: real deterministic front (1a–2b, all words)
- 1a: real frequency list, verified
- 2a: real existence gate + miss-list triage
- 2b: real sense extraction, gender from tags, gloss-required drop
- still JSON output. inspect cards by hand.
## phase 3 — widen: real LLM back (2c + 3)
- storage seam: JSON → SQLite, schema, resumability
- 2c: real generate→vote, three families, all sub-passes
- 3: real CEFRLex single-sense + 3-model vote multi-sense + exclude-on-split
## phase 4 — coverage report + runs
- emit report
- run english, then italian
- decide: local good enough? rent vs API?