wip
This commit is contained in:
lila
parent
0118798e36
commit
cc89f0c75c
20 changed files with 384 additions and 5600 deletions
8
.gitignore
vendored
8
.gitignore
vendored
|
|
@ -10,13 +10,7 @@ venv/
|
|||
__pycache__/
|
||||
*.pyc
|
||||
|
||||
data-pipeline/archive/
|
||||
data-pipeline/stage-1-extract/output/
|
||||
data-pipeline/stage-1-extract/sources/
|
||||
data-pipeline/stage-2-annotate/output/
|
||||
data-pipeline/stage-3-enrich/output/
|
||||
data-pipeline/stage-4-merge/output/
|
||||
data-pipeline/kaikki-source-files/
|
||||
data-pipeline/db/pipeline.db
|
||||
data-pipeline/reports/
|
||||
data-pipeline/.env
|
||||
.aider*
|
||||
|
|
|
|||
|
|
@ -1,87 +0,0 @@
|
|||
import Database from "better-sqlite3";
|
||||
import path from "node:path";
|
||||
import fs from "node:fs";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
const __dirname = path.dirname(fileURLToPath(import.meta.url));
|
||||
const DB_PATH = path.join(__dirname, "db/pipeline.db");
|
||||
|
||||
const db = new Database(DB_PATH, { readonly: true });
|
||||
|
||||
// Pull 50 synsets: ~12 per POS, all must have German translations
|
||||
const synsets = db
|
||||
.prepare(
|
||||
`
|
||||
SELECT DISTINCT s.source_id, s.pos
|
||||
FROM synsets s
|
||||
JOIN translations t ON t.source_id = s.source_id
|
||||
WHERE t.language = 'de'
|
||||
ORDER BY RANDOM()
|
||||
LIMIT 50
|
||||
`,
|
||||
)
|
||||
.all() as { source_id: string; pos: string }[];
|
||||
|
||||
const results: string[] = [];
|
||||
let index = 0;
|
||||
|
||||
for (const synset of synsets) {
|
||||
index++;
|
||||
|
||||
const glosses = db
|
||||
.prepare("SELECT language, text FROM glosses WHERE source_id = ?")
|
||||
.all(synset.source_id) as { language: string; text: string }[];
|
||||
|
||||
const enGloss = glosses.find((g) => g.language === "en")?.text ?? "—";
|
||||
const deGloss = glosses.find((g) => g.language === "de")?.text ?? "—";
|
||||
|
||||
const deTranslations = db
|
||||
.prepare(
|
||||
"SELECT word FROM translations WHERE source_id = ? AND language = 'de'",
|
||||
)
|
||||
.all(synset.source_id) as { word: string }[];
|
||||
|
||||
const enTranslations = db
|
||||
.prepare(
|
||||
"SELECT word FROM translations WHERE source_id = ? AND language = 'en'",
|
||||
)
|
||||
.all(synset.source_id) as { word: string }[];
|
||||
|
||||
const deWords = deTranslations.map((t) => t.word);
|
||||
const enWords = enTranslations.map((t) => t.word);
|
||||
|
||||
results.push(
|
||||
[
|
||||
`${String(index).padStart(2, " ")}. [${synset.pos}] ${synset.source_id}`,
|
||||
` EN gloss: ${enGloss}`,
|
||||
` DE gloss: ${deGloss}`,
|
||||
` EN words: ${enWords.join(", ")}`,
|
||||
` DE words: ${deWords.join(", ")}`,
|
||||
` QUALITY: ___`,
|
||||
``,
|
||||
].join("\n"),
|
||||
);
|
||||
}
|
||||
|
||||
const output = [
|
||||
"# OMW German Translation Quality Audit",
|
||||
"",
|
||||
"Instructions: for each entry, check if the German translations",
|
||||
"match the meaning described by the English gloss.",
|
||||
"",
|
||||
"Mark QUALITY as:",
|
||||
" OK — all German translations fit the meaning",
|
||||
" PARTIAL — some fit, some don't",
|
||||
" BAD — none of the German translations fit",
|
||||
" USELESS — translations are correct but useless for learners",
|
||||
"",
|
||||
"---",
|
||||
"",
|
||||
...results,
|
||||
].join("\n");
|
||||
|
||||
const outPath = path.join(__dirname, "audit.md");
|
||||
fs.writeFileSync(outPath, output, "utf-8");
|
||||
console.log(`Wrote ${synsets.length} entries → ${outPath}`);
|
||||
|
||||
db.close();
|
||||
|
|
@ -1,154 +0,0 @@
|
|||
import fs from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { SUPPORTED_LANGUAGE_CODES } from "@lila/shared";
|
||||
import { openDb } from "./index.js";
|
||||
import type { ExtractedSense } from "../stage-1-extract/scripts/extract.js";
|
||||
|
||||
// ── Paths ─────────────────────────────────────────────────────────────────────
|
||||
|
||||
const __dirname = path.dirname(fileURLToPath(import.meta.url));
|
||||
|
||||
const OUTPUT_DIR = path.resolve(__dirname, "../stage-1-extract/output");
|
||||
|
||||
// ── Import ────────────────────────────────────────────────────────────────────
|
||||
|
||||
export async function importKaikki(): Promise<void> {
|
||||
const db = openDb();
|
||||
|
||||
const insertEntry = db.prepare(`
|
||||
INSERT INTO entries (headword, language, pos, sense_index, gloss, examples)
|
||||
VALUES (?, ?, ?, ?, ?, ?)
|
||||
ON CONFLICT (headword, language, pos, sense_index)
|
||||
DO UPDATE SET
|
||||
gloss = excluded.gloss,
|
||||
examples = excluded.examples
|
||||
RETURNING id
|
||||
`);
|
||||
|
||||
const insertTranslation = db.prepare(`
|
||||
INSERT INTO translations (entry_id, target_lang, word, sense_hint)
|
||||
VALUES (?, ?, ?, ?)
|
||||
ON CONFLICT (entry_id, target_lang, word) DO NOTHING
|
||||
`);
|
||||
|
||||
let totalEntries = 0;
|
||||
let totalTranslations = 0;
|
||||
let totalSkipped = 0;
|
||||
|
||||
for (const lang of SUPPORTED_LANGUAGE_CODES) {
|
||||
const filePath = path.join(OUTPUT_DIR, `${lang}.json`);
|
||||
|
||||
let senses: ExtractedSense[];
|
||||
try {
|
||||
const raw = await fs.readFile(filePath, "utf-8");
|
||||
senses = JSON.parse(raw) as ExtractedSense[];
|
||||
} catch {
|
||||
console.warn(` Warning: no output file found for ${lang}, skipping`);
|
||||
continue;
|
||||
}
|
||||
|
||||
console.log(
|
||||
` Importing ${lang}: ${senses.length.toLocaleString()} senses...`,
|
||||
);
|
||||
|
||||
// Track next available sense_index per (headword, pos) to handle
|
||||
// the same word appearing in multiple JSONL entries with the same POS.
|
||||
const senseIndexMap = new Map<string, number>();
|
||||
|
||||
const importLang = db.transaction(() => {
|
||||
let entries = 0;
|
||||
let translations = 0;
|
||||
let skipped = 0;
|
||||
|
||||
for (const sense of senses) {
|
||||
const key = `${sense.headword}|${sense.pos}`;
|
||||
const nextIndex = senseIndexMap.get(key) ?? 0;
|
||||
senseIndexMap.set(key, nextIndex + 1);
|
||||
|
||||
const row = insertEntry.get(
|
||||
sense.headword,
|
||||
sense.language,
|
||||
sense.pos,
|
||||
nextIndex,
|
||||
sense.gloss ?? null,
|
||||
JSON.stringify(sense.examples),
|
||||
) as { id: number } | undefined;
|
||||
|
||||
if (!row) {
|
||||
skipped++;
|
||||
continue;
|
||||
}
|
||||
|
||||
entries++;
|
||||
|
||||
for (const t of sense.translations) {
|
||||
insertTranslation.run(
|
||||
row.id,
|
||||
t.target_lang,
|
||||
t.word,
|
||||
t.sense_hint ?? null,
|
||||
);
|
||||
translations++;
|
||||
}
|
||||
}
|
||||
|
||||
return { entries, translations, skipped };
|
||||
});
|
||||
|
||||
const counts = importLang();
|
||||
totalEntries += counts.entries;
|
||||
totalTranslations += counts.translations;
|
||||
totalSkipped += counts.skipped;
|
||||
|
||||
console.log(
|
||||
` entries: ${counts.entries.toLocaleString()}, translations: ${counts.translations.toLocaleString()}, skipped: ${counts.skipped.toLocaleString()}`,
|
||||
);
|
||||
}
|
||||
|
||||
db.close();
|
||||
|
||||
console.log(`\nImport complete:`);
|
||||
console.log(` Total entries: ${totalEntries.toLocaleString()}`);
|
||||
console.log(` Total translations: ${totalTranslations.toLocaleString()}`);
|
||||
console.log(` Total skipped: ${totalSkipped.toLocaleString()}`);
|
||||
}
|
||||
|
||||
// ── Check if already imported ─────────────────────────────────────────────────
|
||||
|
||||
export function isImported(): boolean {
|
||||
const db = openDb();
|
||||
const row = db.prepare("SELECT COUNT(*) as count FROM entries").get() as {
|
||||
count: number;
|
||||
};
|
||||
db.close();
|
||||
return row.count > 0;
|
||||
}
|
||||
|
||||
// ── Main ─────────────────────────────────────────────────────────────────────
|
||||
|
||||
async function main(): Promise<void> {
|
||||
const db = openDb();
|
||||
const row = db.prepare("SELECT COUNT(*) as count FROM entries").get() as {
|
||||
count: number;
|
||||
};
|
||||
db.close();
|
||||
|
||||
if (row.count > 0) {
|
||||
console.log(
|
||||
`pipeline.db already contains ${row.count.toLocaleString()} entries — skipping import.`,
|
||||
);
|
||||
console.log("Delete pipeline.db and re-run db:init to start fresh.");
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
console.log("Importing Kaikki data into pipeline.db...");
|
||||
await importKaikki();
|
||||
}
|
||||
|
||||
if (import.meta.url === `file://${process.argv[1]}`) {
|
||||
main().catch((err) => {
|
||||
console.error(err);
|
||||
process.exit(1);
|
||||
});
|
||||
}
|
||||
|
|
@ -1,24 +0,0 @@
|
|||
import path from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import Database from "better-sqlite3";
|
||||
|
||||
// ── Paths ─────────────────────────────────────────────────────────────────────
|
||||
|
||||
const __dirname = path.dirname(fileURLToPath(import.meta.url));
|
||||
|
||||
const DB_PATH = path.join(__dirname, "pipeline.db");
|
||||
|
||||
// ── Types ─────────────────────────────────────────────────────────────────────
|
||||
|
||||
export type Db = InstanceType<typeof Database>;
|
||||
|
||||
// ── Open ──────────────────────────────────────────────────────────────────────
|
||||
|
||||
export function openDb(): Db {
|
||||
const db = new Database(DB_PATH);
|
||||
|
||||
db.pragma("journal_mode = WAL");
|
||||
db.pragma("foreign_keys = ON");
|
||||
|
||||
return db;
|
||||
}
|
||||
|
|
@ -1,42 +0,0 @@
|
|||
import fs from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import Database from "better-sqlite3";
|
||||
|
||||
// ── Paths ─────────────────────────────────────────────────────────────────────
|
||||
|
||||
const __dirname = path.dirname(fileURLToPath(import.meta.url));
|
||||
|
||||
const PATHS = {
|
||||
schema: path.join(__dirname, "schema.sql"),
|
||||
db: path.join(__dirname, "pipeline.db"),
|
||||
};
|
||||
|
||||
// ── Init ──────────────────────────────────────────────────────────────────────
|
||||
|
||||
export async function initDb(): Promise<void> {
|
||||
const schema = await fs.readFile(PATHS.schema, "utf-8");
|
||||
const db = new Database(PATHS.db);
|
||||
|
||||
db.pragma("journal_mode = WAL");
|
||||
db.pragma("foreign_keys = ON");
|
||||
db.exec(schema);
|
||||
db.close();
|
||||
|
||||
console.log(` pipeline.db initialised → ${PATHS.db}`);
|
||||
}
|
||||
|
||||
// ── Main ─────────────────────────────────────────────────────────────────────
|
||||
|
||||
async function main(): Promise<void> {
|
||||
console.log("Initialising pipeline.db...");
|
||||
await initDb();
|
||||
}
|
||||
|
||||
// after
|
||||
if (import.meta.url === `file://${process.argv[1]}`) {
|
||||
main().catch((err) => {
|
||||
console.error(err);
|
||||
process.exit(1);
|
||||
});
|
||||
}
|
||||
Binary file not shown.
|
|
@ -1,41 +0,0 @@
|
|||
import path from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import Database from "better-sqlite3";
|
||||
|
||||
// ── Paths ─────────────────────────────────────────────────────────────────────
|
||||
|
||||
const __dirname = path.dirname(fileURLToPath(import.meta.url));
|
||||
const DB_PATH = path.join(__dirname, "pipeline.db");
|
||||
|
||||
// ── Main ──────────────────────────────────────────────────────────────────────
|
||||
|
||||
function main(): void {
|
||||
const mode = process.argv[2];
|
||||
|
||||
if (!mode || (mode !== "round1" && mode !== "all")) {
|
||||
console.error("Usage: pnpm db:reset round1 | all");
|
||||
console.error(" round1 — delete all round1 sub-stage rows");
|
||||
console.error(" all — delete all run_status rows except reverse_link");
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const db = new Database(DB_PATH);
|
||||
|
||||
let result: { changes: number };
|
||||
|
||||
if (mode === "round1") {
|
||||
result = db
|
||||
.prepare("DELETE FROM run_status WHERE stage LIKE 'round1%'")
|
||||
.run();
|
||||
console.log(`Deleted ${result.changes} round1 rows from run_status`);
|
||||
} else {
|
||||
result = db
|
||||
.prepare("DELETE FROM run_status WHERE stage NOT IN ('reverse_link')")
|
||||
.run();
|
||||
console.log(`Deleted ${result.changes} rows from run_status`);
|
||||
}
|
||||
|
||||
db.close();
|
||||
}
|
||||
|
||||
main();
|
||||
|
|
@ -1,164 +0,0 @@
|
|||
-- ── Base data ─────────────────────────────────────────────────────────────────
|
||||
-- Imported from Kaikki on first run. Never mutated after import.
|
||||
|
||||
CREATE TABLE IF NOT EXISTS entries (
|
||||
id INTEGER PRIMARY KEY,
|
||||
headword TEXT NOT NULL,
|
||||
language TEXT NOT NULL,
|
||||
pos TEXT NOT NULL,
|
||||
sense_index INTEGER NOT NULL DEFAULT 0,
|
||||
gloss TEXT,
|
||||
examples TEXT NOT NULL DEFAULT '[]', -- JSON array of strings
|
||||
source TEXT NOT NULL DEFAULT 'kaikki',
|
||||
UNIQUE (headword, language, pos, sense_index)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS translations (
|
||||
id INTEGER PRIMARY KEY,
|
||||
entry_id INTEGER NOT NULL REFERENCES entries(id),
|
||||
target_lang TEXT NOT NULL,
|
||||
word TEXT NOT NULL,
|
||||
sense_hint TEXT,
|
||||
source TEXT NOT NULL DEFAULT 'kaikki',
|
||||
UNIQUE (entry_id, target_lang, word)
|
||||
);
|
||||
|
||||
-- ── Status tracking ───────────────────────────────────────────────────────────
|
||||
-- One row per entry per model per stage. Drives resumability.
|
||||
-- Sentinel rows use entry_id = 0 for one-time pipeline steps.
|
||||
-- stage: round1 | round2 | tiebreak
|
||||
-- status: pending | complete | needs_review | flagged
|
||||
|
||||
CREATE TABLE IF NOT EXISTS run_status (
|
||||
id INTEGER PRIMARY KEY,
|
||||
entry_id INTEGER NOT NULL,
|
||||
model_name TEXT NOT NULL,
|
||||
stage TEXT NOT NULL,
|
||||
status TEXT NOT NULL,
|
||||
created_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||
updated_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||
UNIQUE (entry_id, model_name, stage)
|
||||
);
|
||||
|
||||
-- ── Round 1 output ────────────────────────────────────────────────────────────
|
||||
-- Written atomically per entry per model.
|
||||
-- Unique constraints enforce one model one vote.
|
||||
|
||||
CREATE TABLE IF NOT EXISTS model_entry_cefr_votes (
|
||||
id INTEGER PRIMARY KEY,
|
||||
entry_id INTEGER NOT NULL REFERENCES entries(id),
|
||||
model_name TEXT NOT NULL,
|
||||
cefr_level TEXT NOT NULL,
|
||||
UNIQUE (entry_id, model_name)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS model_translation_cefr_votes (
|
||||
id INTEGER PRIMARY KEY,
|
||||
translation_id INTEGER NOT NULL REFERENCES translations(id),
|
||||
model_name TEXT NOT NULL,
|
||||
cefr_level TEXT NOT NULL,
|
||||
UNIQUE (translation_id, model_name)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS model_translation_rejections (
|
||||
id INTEGER PRIMARY KEY,
|
||||
translation_id INTEGER NOT NULL REFERENCES translations(id),
|
||||
model_name TEXT NOT NULL,
|
||||
UNIQUE (translation_id, model_name)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS generated_glosses (
|
||||
id INTEGER PRIMARY KEY,
|
||||
entry_id INTEGER NOT NULL REFERENCES entries(id),
|
||||
model_name TEXT NOT NULL,
|
||||
text TEXT NOT NULL,
|
||||
UNIQUE (entry_id, model_name)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS generated_examples (
|
||||
id INTEGER PRIMARY KEY,
|
||||
entry_id INTEGER NOT NULL REFERENCES entries(id),
|
||||
model_name TEXT NOT NULL,
|
||||
text TEXT NOT NULL,
|
||||
UNIQUE (entry_id, model_name)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS generated_translations (
|
||||
id INTEGER PRIMARY KEY,
|
||||
entry_id INTEGER NOT NULL REFERENCES entries(id),
|
||||
model_name TEXT NOT NULL,
|
||||
target_lang TEXT NOT NULL,
|
||||
word TEXT NOT NULL,
|
||||
UNIQUE (entry_id, model_name, target_lang)
|
||||
);
|
||||
|
||||
-- ── Round 2 output ────────────────────────────────────────────────────────────
|
||||
-- Each row represents one model voting for one candidate.
|
||||
-- The candidate with the most votes wins in merge.
|
||||
|
||||
CREATE TABLE IF NOT EXISTS gloss_candidate_votes (
|
||||
id INTEGER PRIMARY KEY,
|
||||
gloss_id INTEGER NOT NULL REFERENCES generated_glosses(id),
|
||||
model_name TEXT NOT NULL,
|
||||
UNIQUE (gloss_id, model_name)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS example_candidate_votes (
|
||||
id INTEGER PRIMARY KEY,
|
||||
example_id INTEGER NOT NULL REFERENCES generated_examples(id),
|
||||
model_name TEXT NOT NULL,
|
||||
UNIQUE (example_id, model_name)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS translation_candidate_votes (
|
||||
id INTEGER PRIMARY KEY,
|
||||
translation_id INTEGER NOT NULL REFERENCES generated_translations(id),
|
||||
model_name TEXT NOT NULL,
|
||||
UNIQUE (translation_id, model_name)
|
||||
);
|
||||
|
||||
-- ── Resolved output ───────────────────────────────────────────────────────────
|
||||
-- Written by merge. Never updated after writing.
|
||||
-- Only fully resolved records are written here — no nulls.
|
||||
-- Absence of a row means unresolved. Flagged status tracked in run_status.
|
||||
-- source: kaikki | model_name
|
||||
|
||||
CREATE TABLE IF NOT EXISTS resolved_entry_cefr (
|
||||
id INTEGER PRIMARY KEY,
|
||||
entry_id INTEGER NOT NULL REFERENCES entries(id),
|
||||
cefr_level TEXT NOT NULL,
|
||||
difficulty TEXT NOT NULL,
|
||||
UNIQUE (entry_id)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS resolved_translation_cefr (
|
||||
id INTEGER PRIMARY KEY,
|
||||
translation_id INTEGER NOT NULL REFERENCES translations(id),
|
||||
cefr_level TEXT NOT NULL,
|
||||
difficulty TEXT NOT NULL,
|
||||
UNIQUE (translation_id)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS resolved_glosses (
|
||||
id INTEGER PRIMARY KEY,
|
||||
entry_id INTEGER NOT NULL REFERENCES entries(id),
|
||||
text TEXT NOT NULL,
|
||||
source TEXT NOT NULL,
|
||||
UNIQUE (entry_id)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS resolved_examples (
|
||||
id INTEGER PRIMARY KEY,
|
||||
entry_id INTEGER NOT NULL REFERENCES entries(id),
|
||||
text TEXT NOT NULL,
|
||||
source TEXT NOT NULL
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS resolved_generated_translations (
|
||||
id INTEGER PRIMARY KEY,
|
||||
entry_id INTEGER NOT NULL REFERENCES entries(id),
|
||||
target_lang TEXT NOT NULL,
|
||||
word TEXT NOT NULL,
|
||||
source TEXT NOT NULL,
|
||||
UNIQUE (entry_id, target_lang)
|
||||
);
|
||||
|
|
@ -1,616 +0,0 @@
|
|||
import fs from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { initDb } from "./db/init.js";
|
||||
import { isImported, importKaikki } from "./db/import.js";
|
||||
import { openDb } from "./db/index.js";
|
||||
import { reverseLink } from "./stage-2-reverse-link/scripts/reverse-link.js";
|
||||
import { ALL_PROVIDERS, validateProviderKey } from "./stage-3-enrich/config.js";
|
||||
import type { ProviderConfig } from "./stage-3-enrich/config.js";
|
||||
import { enrich } from "./stage-3-enrich/scripts/enrich.js";
|
||||
|
||||
// ── Types ─────────────────────────────────────────────────────────────────────
|
||||
|
||||
type RunStage =
|
||||
| "round1"
|
||||
| "compile_candidates"
|
||||
| "round2"
|
||||
| "compile_votes"
|
||||
| "merge"
|
||||
| "tiebreak"
|
||||
| "compare";
|
||||
|
||||
type StageStatus = "complete" | "pending" | "in_progress";
|
||||
|
||||
type RunStats = {
|
||||
startedAt: Date;
|
||||
stoppedAt: Date | null;
|
||||
recordsProcessed: number;
|
||||
recordsSkipped: number;
|
||||
needsReview: number;
|
||||
modelsRun: string[];
|
||||
currentStage: RunStage | null;
|
||||
};
|
||||
|
||||
// ── Constants ─────────────────────────────────────────────────────────────────
|
||||
|
||||
const __dirname = path.dirname(fileURLToPath(import.meta.url));
|
||||
|
||||
const PATHS = {
|
||||
extractedEn: path.join(__dirname, "stage-1-extract/output/en.json"),
|
||||
db: path.join(__dirname, "db/pipeline.db"),
|
||||
reports: path.join(__dirname, "reports"),
|
||||
llamaHealth: "http://127.0.0.1:8080/health",
|
||||
};
|
||||
|
||||
const SENTINEL = { entryId: 0, modelName: "system" };
|
||||
|
||||
// ── Startup checks ────────────────────────────────────────────────────────────
|
||||
|
||||
async function checkExtractedFilesExist(): Promise<void> {
|
||||
try {
|
||||
await fs.access(PATHS.extractedEn);
|
||||
} catch {
|
||||
console.error("\n ERROR: stage-1-extract/output/en.json not found.");
|
||||
console.error(" Run the stage 1 extraction script first:");
|
||||
console.error(" pnpm extract\n");
|
||||
process.exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
async function checkAndInitDb(): Promise<void> {
|
||||
try {
|
||||
await fs.access(PATHS.db);
|
||||
} catch {
|
||||
console.log(" pipeline.db not found — initialising...");
|
||||
await initDb();
|
||||
}
|
||||
}
|
||||
|
||||
async function checkAndImportDb(): Promise<void> {
|
||||
if (!isImported()) {
|
||||
console.log(" Base tables empty — importing Kaikki data...");
|
||||
await importKaikki();
|
||||
}
|
||||
}
|
||||
|
||||
async function checkLlamaServer(): Promise<boolean> {
|
||||
try {
|
||||
const res = await fetch(PATHS.llamaHealth);
|
||||
return res.ok;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
function isLocalProvider(provider: ProviderConfig): boolean {
|
||||
return provider.apiKey === "none";
|
||||
}
|
||||
|
||||
async function checkProviderReady(provider: ProviderConfig): Promise<void> {
|
||||
if (isLocalProvider(provider)) {
|
||||
const running = await checkLlamaServer();
|
||||
if (!running) {
|
||||
console.error("\n ERROR: llama.cpp server is not running.");
|
||||
console.error(" Start the server before running the pipeline:");
|
||||
console.error(
|
||||
" ./build/bin/llama-server --model models/<model>.gguf \\",
|
||||
);
|
||||
console.error(" --port 8080 --host 127.0.0.1");
|
||||
console.error(" See llm-setup.md for full instructions.\n");
|
||||
process.exit(1);
|
||||
}
|
||||
} else {
|
||||
validateProviderKey(provider);
|
||||
}
|
||||
}
|
||||
|
||||
// ── Run name generation ───────────────────────────────────────────────────────
|
||||
|
||||
async function generateRunName(): Promise<string> {
|
||||
await fs.mkdir(PATHS.reports, { recursive: true });
|
||||
|
||||
const date = new Date().toISOString().exi(0, 10);
|
||||
const files = await fs.readdir(PATHS.reports);
|
||||
const todaysRuns = files.filter(
|
||||
(f) => f.startsWith(date) && f.endsWith(".json"),
|
||||
).length;
|
||||
|
||||
return `${date}_run-${todaysRuns + 1}`;
|
||||
}
|
||||
|
||||
// ── Shutdown handler ──────────────────────────────────────────────────────────
|
||||
|
||||
let shutdownRequested = false;
|
||||
|
||||
function registerShutdownHandler(stats: RunStats): void {
|
||||
const handler = (): void => {
|
||||
if (shutdownRequested) return;
|
||||
shutdownRequested = true;
|
||||
stats.stoppedAt = new Date();
|
||||
console.log("\n\n Shutdown requested — finishing current record...");
|
||||
};
|
||||
|
||||
process.on("SIGINT", handler);
|
||||
process.on("SIGTERM", handler);
|
||||
}
|
||||
|
||||
// ── Stage status helpers ──────────────────────────────────────────────────────
|
||||
|
||||
function getSentinelStatus(stage: RunStage): StageStatus {
|
||||
const db = openDb();
|
||||
const row = db
|
||||
.prepare(
|
||||
`SELECT status FROM run_status
|
||||
WHERE entry_id = ? AND model_name = ? AND stage = ?`,
|
||||
)
|
||||
.get(SENTINEL.entryId, SENTINEL.modelName, stage) as
|
||||
| { status: string }
|
||||
| undefined;
|
||||
db.close();
|
||||
return row?.status === "complete" ? "complete" : "pending";
|
||||
}
|
||||
|
||||
function markSentinelComplete(stage: RunStage): void {
|
||||
const db = openDb();
|
||||
db.prepare(
|
||||
`INSERT INTO run_status (entry_id, model_name, stage, status)
|
||||
VALUES (?, ?, ?, 'complete')
|
||||
ON CONFLICT (entry_id, model_name, stage)
|
||||
DO UPDATE SET status = 'complete', updated_at = datetime('now')`,
|
||||
).run(SENTINEL.entryId, SENTINEL.modelName, stage);
|
||||
db.close();
|
||||
}
|
||||
|
||||
function getModelRound1Status(modelName: string): StageStatus {
|
||||
const db = openDb();
|
||||
|
||||
const total = (
|
||||
db
|
||||
.prepare("SELECT COUNT(*) as count FROM entries WHERE language = 'en'")
|
||||
.get() as { count: number }
|
||||
).count;
|
||||
|
||||
const complete = (
|
||||
db
|
||||
.prepare(
|
||||
`SELECT COUNT(*) as count FROM run_status
|
||||
WHERE model_name = ? AND stage = 'round1_gloss'
|
||||
AND status = 'complete'`,
|
||||
)
|
||||
.get(modelName) as { count: number }
|
||||
).count;
|
||||
|
||||
db.close();
|
||||
|
||||
if (complete === 0) return "pending";
|
||||
if (complete >= total) return "complete";
|
||||
return "in_progress";
|
||||
}
|
||||
|
||||
function getModelRound2Status(modelName: string): StageStatus {
|
||||
const db = openDb();
|
||||
|
||||
const total = (
|
||||
db
|
||||
.prepare("SELECT COUNT(*) as count FROM entries WHERE language = 'en'")
|
||||
.get() as { count: number }
|
||||
).count;
|
||||
|
||||
const complete = (
|
||||
db
|
||||
.prepare(
|
||||
`SELECT COUNT(*) as count FROM run_status
|
||||
WHERE model_name = ? AND stage = 'round2' AND status = 'complete'`,
|
||||
)
|
||||
.get(modelName) as { count: number }
|
||||
).count;
|
||||
|
||||
db.close();
|
||||
|
||||
if (complete === 0) return "pending";
|
||||
if (complete >= total) return "complete";
|
||||
return "in_progress";
|
||||
}
|
||||
|
||||
function isReverseLinkDone(): boolean {
|
||||
const db = openDb();
|
||||
const row = db
|
||||
.prepare(
|
||||
`SELECT status FROM run_status
|
||||
WHERE entry_id = ? AND model_name = ? AND stage = 'reverse_link'`,
|
||||
)
|
||||
.get(SENTINEL.entryId, SENTINEL.modelName) as
|
||||
| { status: string }
|
||||
| undefined;
|
||||
db.close();
|
||||
return row?.status === "complete";
|
||||
}
|
||||
|
||||
function markReverseLinkComplete(): void {
|
||||
const db = openDb();
|
||||
db.prepare(
|
||||
`INSERT INTO run_status (entry_id, model_name, stage, status)
|
||||
VALUES (?, ?, 'reverse_link', 'complete')
|
||||
ON CONFLICT (entry_id, model_name, stage)
|
||||
DO UPDATE SET status = 'complete', updated_at = datetime('now')`,
|
||||
).run(SENTINEL.entryId, SENTINEL.modelName);
|
||||
db.close();
|
||||
}
|
||||
|
||||
// ── Stage runners ─────────────────────────────────────────────────────────────
|
||||
|
||||
function runReverseLinkStage(): void {
|
||||
if (isReverseLinkDone()) {
|
||||
console.log("\n [reverse link] Already complete, skipping");
|
||||
return;
|
||||
}
|
||||
console.log("\n [reverse link] Syncing reverse translation links...");
|
||||
reverseLink();
|
||||
markReverseLinkComplete();
|
||||
}
|
||||
|
||||
async function runRound1(
|
||||
provider: ProviderConfig,
|
||||
stats: RunStats,
|
||||
): Promise<void> {
|
||||
console.log(`\n [round 1] Running ${provider.name}...`);
|
||||
const counts = await enrich(provider);
|
||||
stats.recordsProcessed += counts.processed;
|
||||
stats.recordsSkipped += counts.skipped;
|
||||
stats.needsReview += counts.needsReview;
|
||||
stats.modelsRun.push(provider.name);
|
||||
}
|
||||
|
||||
function compileCandidates(): void {
|
||||
console.log("\n [compile candidates] Compiling round 1 output...");
|
||||
// TODO: implement compile candidates script
|
||||
console.log(" [compile candidates] not yet implemented");
|
||||
markSentinelComplete("compile_candidates");
|
||||
}
|
||||
|
||||
function runRound2(provider: ProviderConfig, stats: RunStats): void {
|
||||
console.log(`\n [round 2] Running ${provider.name}...`);
|
||||
// TODO: implement round 2 enrich script
|
||||
console.log(` [round 2] ${provider.name} — not yet implemented`);
|
||||
stats.modelsRun.push(provider.name);
|
||||
}
|
||||
|
||||
function compileVotes(): void {
|
||||
console.log("\n [compile votes] Compiling round 2 votes...");
|
||||
// TODO: implement compile votes script
|
||||
console.log(" [compile votes] not yet implemented");
|
||||
markSentinelComplete("compile_votes");
|
||||
}
|
||||
|
||||
function runMerge(): void {
|
||||
console.log("\n [merge] Resolving votes...");
|
||||
// TODO: implement merge script
|
||||
console.log(" [merge] not yet implemented");
|
||||
markSentinelComplete("merge");
|
||||
}
|
||||
|
||||
function runTiebreak(stats: RunStats): void {
|
||||
console.log("\n [tiebreak] Resolving flagged entries...");
|
||||
// TODO: implement tiebreak logic
|
||||
console.log(" [tiebreak] not yet implemented");
|
||||
stats.currentStage = "tiebreak";
|
||||
}
|
||||
|
||||
function runCompare(): void {
|
||||
console.log("\n [compare] Generating COVERAGE.md...");
|
||||
// TODO: implement compare script
|
||||
console.log(" [compare] not yet implemented");
|
||||
markSentinelComplete("compare");
|
||||
}
|
||||
|
||||
// ── Report generation ─────────────────────────────────────────────────────────
|
||||
|
||||
async function generateReport(runName: string, stats: RunStats): Promise<void> {
|
||||
const db = openDb();
|
||||
|
||||
const totalEntries = (
|
||||
db.prepare("SELECT COUNT(*) as count FROM entries").get() as {
|
||||
count: number;
|
||||
}
|
||||
).count;
|
||||
|
||||
const resolvedEntries = (
|
||||
db.prepare("SELECT COUNT(*) as count FROM resolved_entry_cefr").get() as {
|
||||
count: number;
|
||||
}
|
||||
).count;
|
||||
|
||||
const flaggedEntries = (
|
||||
db
|
||||
.prepare(
|
||||
`SELECT COUNT(*) as count FROM run_status
|
||||
WHERE stage = 'merge' AND status = 'flagged'`,
|
||||
)
|
||||
.get() as { count: number }
|
||||
).count;
|
||||
|
||||
const needsReview = (
|
||||
db
|
||||
.prepare(
|
||||
`SELECT COUNT(*) as count FROM run_status
|
||||
WHERE status = 'needs_review'`,
|
||||
)
|
||||
.get() as { count: number }
|
||||
).count;
|
||||
|
||||
db.close();
|
||||
|
||||
const stoppedAt = stats.stoppedAt ?? new Date();
|
||||
const durationMs = stoppedAt.getTime() - stats.startedAt.getTime();
|
||||
const durationMin = Math.round(durationMs / 60_000);
|
||||
|
||||
const isFinal =
|
||||
getSentinelStatus("compare") === "complete" && flaggedEntries === 0;
|
||||
|
||||
const report = {
|
||||
runName,
|
||||
generatedAt: stoppedAt.toISOString(),
|
||||
durationMinutes: durationMin,
|
||||
isFinal,
|
||||
progress: {
|
||||
totalEntries,
|
||||
resolvedEntries,
|
||||
flaggedEntries,
|
||||
needsReview,
|
||||
recordsProcessedThisRun: stats.recordsProcessed,
|
||||
recordsSkippedThisRun: stats.recordsSkipped,
|
||||
},
|
||||
modelsRun: stats.modelsRun,
|
||||
stages: {
|
||||
reverseLink: isReverseLinkDone() ? "complete" : "pending",
|
||||
round1: ALL_PROVIDERS.map((p) => ({
|
||||
model: p.name,
|
||||
status: getModelRound1Status(p.name),
|
||||
})),
|
||||
compileCandidates: getSentinelStatus("compile_candidates"),
|
||||
round2: ALL_PROVIDERS.map((p) => ({
|
||||
model: p.name,
|
||||
status: getModelRound2Status(p.name),
|
||||
})),
|
||||
compileVotes: getSentinelStatus("compile_votes"),
|
||||
merge: getSentinelStatus("merge"),
|
||||
compare: getSentinelStatus("compare"),
|
||||
},
|
||||
};
|
||||
|
||||
await fs.mkdir(PATHS.reports, { recursive: true });
|
||||
|
||||
const jsonPath = path.join(PATHS.reports, `${runName}.json`);
|
||||
const mdPath = path.join(PATHS.reports, `${runName}.md`);
|
||||
|
||||
await fs.writeFile(jsonPath, JSON.stringify(report, null, 2), "utf-8");
|
||||
|
||||
const md = [
|
||||
`# Pipeline run: ${runName}`,
|
||||
``,
|
||||
`Generated: ${stoppedAt.toISOString()}`,
|
||||
`Duration: ${durationMin} minutes`,
|
||||
isFinal
|
||||
? `**Status: FINAL — pipeline complete**`
|
||||
: `**Status: In progress**`,
|
||||
``,
|
||||
`## Progress`,
|
||||
``,
|
||||
`| Metric | Value |`,
|
||||
`| ------ | ----- |`,
|
||||
`| Total entries | ${totalEntries.toLocaleString()} |`,
|
||||
`| Resolved entries | ${resolvedEntries.toLocaleString()} |`,
|
||||
`| Flagged entries | ${flaggedEntries.toLocaleString()} |`,
|
||||
`| Needs review | ${needsReview.toLocaleString()} |`,
|
||||
`| Records processed this run | ${stats.recordsProcessed.toLocaleString()} |`,
|
||||
`| Records skipped this run | ${stats.recordsSkipped.toLocaleString()} |`,
|
||||
``,
|
||||
`## Stage status`,
|
||||
``,
|
||||
`### Reverse link: ${report.stages.reverseLink}`,
|
||||
``,
|
||||
`### Round 1`,
|
||||
``,
|
||||
...report.stages.round1.map(
|
||||
(s) =>
|
||||
`- ${s.status === "complete" ? "✅" : s.status === "in_progress" ? "🔄" : "🔲"} ${s.model}`,
|
||||
),
|
||||
``,
|
||||
`### Compile candidates: ${report.stages.compileCandidates}`,
|
||||
``,
|
||||
`### Round 2`,
|
||||
``,
|
||||
...report.stages.round2.map(
|
||||
(s) =>
|
||||
`- ${s.status === "complete" ? "✅" : s.status === "in_progress" ? "🔄" : "🔲"} ${s.model}`,
|
||||
),
|
||||
``,
|
||||
`### Compile votes: ${report.stages.compileVotes}`,
|
||||
`### Merge: ${report.stages.merge}`,
|
||||
`### Compare: ${report.stages.compare}`,
|
||||
``,
|
||||
`## Models run this session`,
|
||||
``,
|
||||
stats.modelsRun.length > 0
|
||||
? stats.modelsRun.map((m) => `- ${m}`).join("\n")
|
||||
: "_none_",
|
||||
].join("\n");
|
||||
|
||||
await fs.writeFile(mdPath, md, "utf-8");
|
||||
|
||||
console.log(`\n Report written → ${jsonPath}`);
|
||||
console.log(` Report written → ${mdPath}`);
|
||||
}
|
||||
|
||||
// ── Main ──────────────────────────────────────────────────────────────────────
|
||||
|
||||
async function main(): Promise<void> {
|
||||
console.log("lila data pipeline\n");
|
||||
|
||||
// ── Startup checks
|
||||
console.log("Checking prerequisites...");
|
||||
await checkExtractedFilesExist();
|
||||
await checkAndInitDb();
|
||||
await checkAndImportDb();
|
||||
console.log(" Prerequisites OK");
|
||||
|
||||
// ── Run name
|
||||
const runName = await generateRunName();
|
||||
console.log(`\n Run: ${runName}`);
|
||||
|
||||
// ── Stats
|
||||
const stats: RunStats = {
|
||||
startedAt: new Date(),
|
||||
stoppedAt: null,
|
||||
recordsProcessed: 0,
|
||||
recordsSkipped: 0,
|
||||
needsReview: 0,
|
||||
modelsRun: [],
|
||||
currentStage: null,
|
||||
};
|
||||
|
||||
registerShutdownHandler(stats);
|
||||
|
||||
// ── Stage 2 — Reverse link
|
||||
runReverseLinkStage();
|
||||
|
||||
if (shutdownRequested) {
|
||||
await generateReport(runName, stats);
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
// ── Round 1
|
||||
console.log("\nRound 1 — generation");
|
||||
for (const provider of ALL_PROVIDERS) {
|
||||
if (shutdownRequested) break;
|
||||
|
||||
const status = getModelRound1Status(provider.name);
|
||||
|
||||
if (status === "complete") {
|
||||
console.log(` [round 1] ${provider.name} — already complete, skipping`);
|
||||
continue;
|
||||
}
|
||||
|
||||
await checkProviderReady(provider);
|
||||
stats.currentStage = "round1";
|
||||
|
||||
if (status === "in_progress") {
|
||||
console.log(` [round 1] ${provider.name} — resuming...`);
|
||||
}
|
||||
|
||||
await runRound1(provider, stats);
|
||||
}
|
||||
|
||||
if (shutdownRequested) {
|
||||
await generateReport(runName, stats);
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
// ── Compile candidates
|
||||
if (getSentinelStatus("compile_candidates") === "complete") {
|
||||
console.log("\n [compile candidates] Already complete, skipping");
|
||||
} else {
|
||||
stats.currentStage = "compile_candidates";
|
||||
compileCandidates();
|
||||
}
|
||||
|
||||
if (shutdownRequested) {
|
||||
await generateReport(runName, stats);
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
// ── Round 2
|
||||
console.log("\nRound 2 — voting");
|
||||
for (const provider of ALL_PROVIDERS) {
|
||||
if (shutdownRequested) break;
|
||||
|
||||
const status = getModelRound2Status(provider.name);
|
||||
|
||||
if (status === "complete") {
|
||||
console.log(` [round 2] ${provider.name} — already complete, skipping`);
|
||||
continue;
|
||||
}
|
||||
|
||||
await checkProviderReady(provider);
|
||||
stats.currentStage = "round2";
|
||||
|
||||
if (status === "in_progress") {
|
||||
console.log(` [round 2] ${provider.name} — resuming...`);
|
||||
}
|
||||
|
||||
runRound2(provider, stats);
|
||||
}
|
||||
|
||||
if (shutdownRequested) {
|
||||
await generateReport(runName, stats);
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
// ── Compile votes
|
||||
if (getSentinelStatus("compile_votes") === "complete") {
|
||||
console.log("\n [compile votes] Already complete, skipping");
|
||||
} else {
|
||||
stats.currentStage = "compile_votes";
|
||||
compileVotes();
|
||||
}
|
||||
|
||||
if (shutdownRequested) {
|
||||
await generateReport(runName, stats);
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
// ── Merge
|
||||
if (getSentinelStatus("merge") === "complete") {
|
||||
console.log("\n [merge] Already complete, skipping");
|
||||
} else {
|
||||
stats.currentStage = "merge";
|
||||
runMerge();
|
||||
}
|
||||
|
||||
if (shutdownRequested) {
|
||||
await generateReport(runName, stats);
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
// ── Tiebreak
|
||||
const db = openDb();
|
||||
const flagged = (
|
||||
db
|
||||
.prepare(
|
||||
`SELECT COUNT(*) as count FROM run_status
|
||||
WHERE stage = 'merge' AND status = 'flagged'`,
|
||||
)
|
||||
.get() as { count: number }
|
||||
).count;
|
||||
db.close();
|
||||
|
||||
if (flagged > 0) {
|
||||
stats.currentStage = "tiebreak";
|
||||
runTiebreak(stats);
|
||||
}
|
||||
|
||||
if (shutdownRequested) {
|
||||
await generateReport(runName, stats);
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
// ── Compare
|
||||
if (getSentinelStatus("compare") === "complete") {
|
||||
console.log("\n [compare] Already complete, skipping");
|
||||
} else {
|
||||
stats.currentStage = "compare";
|
||||
runCompare();
|
||||
}
|
||||
|
||||
// ── Report (disabled until full pipeline is implemented)
|
||||
// stats.stoppedAt = new Date();
|
||||
// await generateReport(runName, stats);
|
||||
|
||||
console.log("\nPipeline complete.");
|
||||
}
|
||||
|
||||
main().catch((err) => {
|
||||
console.error(err);
|
||||
process.exit(1);
|
||||
});
|
||||
File diff suppressed because it is too large
Load diff
|
|
@ -1,205 +0,0 @@
|
|||
import fs from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import { SUPPORTED_LANGUAGE_CODES } from "@lila/shared";
|
||||
import type { SupportedLanguageCode, SupportedPos } from "@lila/shared";
|
||||
|
||||
// ── Types ─────────────────────────────────────────────────────────────────────
|
||||
|
||||
type Example = { text: string; source: "omw" | "cefr" };
|
||||
|
||||
type AnnotatedRecord = {
|
||||
source_id: string;
|
||||
pos: SupportedPos;
|
||||
translations: Partial<Record<SupportedLanguageCode, string[]>>;
|
||||
glosses: Partial<Record<SupportedLanguageCode, string[]>>;
|
||||
examples: Partial<Record<SupportedLanguageCode, Example[]>>;
|
||||
votes: Partial<
|
||||
Record<SupportedLanguageCode, Record<string, { cefr_source: string }>>
|
||||
>;
|
||||
};
|
||||
|
||||
type SampleRecord = AnnotatedRecord & { _sample_bucket: string };
|
||||
|
||||
// ── Constants ─────────────────────────────────────────────────────────────────
|
||||
|
||||
const PATHS = {
|
||||
annotatedDir: "stage-2-annotate/output",
|
||||
output: "test/output/sample.json",
|
||||
};
|
||||
|
||||
const BUCKET_SIZE = 20;
|
||||
|
||||
// ── Bucket predicates ─────────────────────────────────────────────────────────
|
||||
|
||||
type Bucket = { name: string; predicate: (record: AnnotatedRecord) => boolean };
|
||||
|
||||
const BUCKETS: Bucket[] = [
|
||||
{
|
||||
name: "has_cefr_vote",
|
||||
predicate: (r) =>
|
||||
Object.values(r.votes).some(
|
||||
(langVotes) => Object.keys(langVotes ?? {}).length > 0,
|
||||
),
|
||||
},
|
||||
{
|
||||
name: "no_cefr_vote",
|
||||
predicate: (r) =>
|
||||
Object.values(r.votes).every(
|
||||
(langVotes) => Object.keys(langVotes ?? {}).length === 0,
|
||||
),
|
||||
},
|
||||
{
|
||||
name: "has_glosses_and_examples",
|
||||
predicate: (r) =>
|
||||
Object.keys(r.glosses).length > 0 && Object.keys(r.examples).length > 0,
|
||||
},
|
||||
{
|
||||
name: "no_glosses_no_examples",
|
||||
predicate: (r) =>
|
||||
!r.glosses["fr"] &&
|
||||
!r.examples["fr"] &&
|
||||
!r.votes["fr"] &&
|
||||
!r.glosses["es"] &&
|
||||
!r.examples["es"] &&
|
||||
!r.votes["es"],
|
||||
},
|
||||
{
|
||||
name: "pos_spread",
|
||||
predicate: () => true, // sampled separately to ensure POS coverage
|
||||
},
|
||||
];
|
||||
|
||||
// ── Sampling ──────────────────────────────────────────────────────────────────
|
||||
|
||||
function sampleBucket(
|
||||
records: AnnotatedRecord[],
|
||||
predicate: (r: AnnotatedRecord) => boolean,
|
||||
size: number,
|
||||
exclude: Set<string>,
|
||||
): AnnotatedRecord[] {
|
||||
const candidates = records.filter(
|
||||
(r) => !exclude.has(r.source_id) && predicate(r),
|
||||
);
|
||||
|
||||
// Shuffle for random sampling
|
||||
for (let i = candidates.length - 1; i > 0; i--) {
|
||||
const j = Math.floor(Math.random() * (i + 1));
|
||||
[candidates[i], candidates[j]] = [candidates[j]!, candidates[i]!];
|
||||
}
|
||||
|
||||
return candidates.slice(0, size);
|
||||
}
|
||||
|
||||
function samplePosBucket(
|
||||
records: AnnotatedRecord[],
|
||||
exclude: Set<string>,
|
||||
): AnnotatedRecord[] {
|
||||
const posList: SupportedPos[] = ["noun", "verb", "adjective", "adverb"];
|
||||
const perPos = Math.floor(BUCKET_SIZE / posList.length);
|
||||
const result: AnnotatedRecord[] = [];
|
||||
|
||||
for (const pos of posList) {
|
||||
const sampled = sampleBucket(
|
||||
records,
|
||||
(r) => r.pos === pos,
|
||||
perPos,
|
||||
exclude,
|
||||
);
|
||||
result.push(...sampled);
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
// ── Loading ───────────────────────────────────────────────────────────────────
|
||||
|
||||
async function loadAnnotated(): Promise<AnnotatedRecord[]> {
|
||||
// Load all language files and merge votes into a single record set.
|
||||
// Use en.json as the base record structure since it has the most complete
|
||||
// glosses and examples. Votes from all other languages are merged in.
|
||||
const baseRaw = await fs.readFile(
|
||||
path.join(PATHS.annotatedDir, "en.json"),
|
||||
"utf-8",
|
||||
);
|
||||
const base = JSON.parse(baseRaw) as AnnotatedRecord[];
|
||||
|
||||
// Build a map for fast lookup by source_id
|
||||
const byId = new Map<string, AnnotatedRecord>();
|
||||
for (const record of base) {
|
||||
byId.set(record.source_id, record);
|
||||
}
|
||||
|
||||
// Merge votes from remaining language files
|
||||
for (const lang of SUPPORTED_LANGUAGE_CODES) {
|
||||
if (lang === "en") continue;
|
||||
const raw = await fs.readFile(
|
||||
path.join(PATHS.annotatedDir, `${lang}.json`),
|
||||
"utf-8",
|
||||
);
|
||||
const records = JSON.parse(raw) as AnnotatedRecord[];
|
||||
|
||||
for (const record of records) {
|
||||
const base = byId.get(record.source_id);
|
||||
if (!base) continue;
|
||||
|
||||
// Merge votes
|
||||
for (const [l, langVotes] of Object.entries(record.votes)) {
|
||||
if (!base.votes[l as SupportedLanguageCode]) {
|
||||
base.votes[l as SupportedLanguageCode] = {};
|
||||
}
|
||||
Object.assign(base.votes[l as SupportedLanguageCode]!, langVotes);
|
||||
}
|
||||
|
||||
// Merge examples from CEFR source files not in base
|
||||
for (const [l, examples] of Object.entries(record.examples)) {
|
||||
const lang = l as SupportedLanguageCode;
|
||||
if (!base.examples[lang]) {
|
||||
base.examples[lang] = examples;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return [...byId.values()];
|
||||
}
|
||||
|
||||
// ── Main ─────────────────────────────────────────────────────────────────────
|
||||
|
||||
async function main(): Promise<void> {
|
||||
console.log("Loading annotated files...");
|
||||
const records = await loadAnnotated();
|
||||
console.log(` Loaded ${records.length.toLocaleString()} synsets`);
|
||||
|
||||
const sampled: SampleRecord[] = [];
|
||||
const seen = new Set<string>();
|
||||
|
||||
// Sample each bucket except pos_spread
|
||||
for (const bucket of BUCKETS.filter((b) => b.name !== "pos_spread")) {
|
||||
const results = sampleBucket(records, bucket.predicate, BUCKET_SIZE, seen);
|
||||
for (const r of results) {
|
||||
seen.add(r.source_id);
|
||||
sampled.push({ ...r, _sample_bucket: bucket.name });
|
||||
}
|
||||
console.log(` ${bucket.name}: ${results.length} records`);
|
||||
}
|
||||
|
||||
// Sample pos_spread bucket
|
||||
const posResults = samplePosBucket(records, seen);
|
||||
for (const r of posResults) {
|
||||
seen.add(r.source_id);
|
||||
sampled.push({ ...r, _sample_bucket: "pos_spread" });
|
||||
}
|
||||
console.log(` pos_spread: ${posResults.length} records`);
|
||||
|
||||
console.log(`\nTotal sampled: ${sampled.length} records`);
|
||||
|
||||
// Write output
|
||||
await fs.mkdir(path.dirname(PATHS.output), { recursive: true });
|
||||
await fs.writeFile(PATHS.output, JSON.stringify(sampled, null, 2), "utf-8");
|
||||
console.log(`Wrote sample → ${PATHS.output}`);
|
||||
}
|
||||
|
||||
main().catch((err) => {
|
||||
console.error(err);
|
||||
process.exit(1);
|
||||
});
|
||||
|
|
@ -1,257 +0,0 @@
|
|||
import fs from "node:fs";
|
||||
import path from "node:path";
|
||||
import readline from "node:readline";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { SUPPORTED_LANGUAGE_CODES } from "@lila/shared";
|
||||
import type { SupportedLanguageCode, SupportedPos } from "@lila/shared";
|
||||
|
||||
// ── Types ─────────────────────────────────────────────────────────────────────
|
||||
|
||||
type KaikkiTranslation = {
|
||||
code?: string;
|
||||
lang_code?: string;
|
||||
word?: string;
|
||||
sense?: string;
|
||||
};
|
||||
|
||||
type KaikkiSense = {
|
||||
glosses?: string[];
|
||||
examples?: { text?: string }[];
|
||||
translations?: KaikkiTranslation[];
|
||||
};
|
||||
|
||||
type KaikkiEntry = {
|
||||
word?: string;
|
||||
pos?: string;
|
||||
lang_code?: string;
|
||||
senses?: KaikkiSense[];
|
||||
};
|
||||
|
||||
export type ExtractedSense = {
|
||||
headword: string;
|
||||
language: SupportedLanguageCode;
|
||||
pos: SupportedPos;
|
||||
sense_index: number;
|
||||
gloss: string | null;
|
||||
examples: string[];
|
||||
translations: {
|
||||
target_lang: SupportedLanguageCode;
|
||||
word: string;
|
||||
sense_hint: string | null;
|
||||
}[];
|
||||
};
|
||||
|
||||
// ── Constants ─────────────────────────────────────────────────────────────────
|
||||
|
||||
const __dirname = path.dirname(fileURLToPath(import.meta.url));
|
||||
|
||||
const SOURCES_DIR = path.resolve(__dirname, "../sources");
|
||||
const OUTPUT_DIR = path.resolve(__dirname, "../output");
|
||||
|
||||
const LANG_TO_FILE: Record<SupportedLanguageCode, string> = {
|
||||
en: "kaikki.org-dictionary-English.jsonl",
|
||||
de: "kaikki.org-dictionary-German.jsonl",
|
||||
it: "kaikki.org-dictionary-Italian.jsonl",
|
||||
fr: "kaikki.org-dictionary-French.jsonl",
|
||||
es: "kaikki.org-dictionary-Spanish.jsonl",
|
||||
};
|
||||
|
||||
const POS_MAP: Record<string, SupportedPos> = {
|
||||
noun: "noun",
|
||||
verb: "verb",
|
||||
adj: "adjective",
|
||||
adv: "adverb",
|
||||
};
|
||||
|
||||
const SUPPORTED_LANG_SET = new Set<string>(SUPPORTED_LANGUAGE_CODES);
|
||||
|
||||
// ── Helpers ───────────────────────────────────────────────────────────────────
|
||||
|
||||
function mapPos(kaikkiPos: string): SupportedPos | null {
|
||||
return POS_MAP[kaikkiPos] ?? null;
|
||||
}
|
||||
|
||||
function isAbbreviation(gloss: string): boolean {
|
||||
return gloss.toLowerCase().startsWith("abbreviation of");
|
||||
}
|
||||
|
||||
function extractTranslations(
|
||||
sense: KaikkiSense,
|
||||
sourceLang: SupportedLanguageCode,
|
||||
): ExtractedSense["translations"] {
|
||||
const seen = new Set<string>();
|
||||
const result: ExtractedSense["translations"] = [];
|
||||
|
||||
for (const t of sense.translations ?? []) {
|
||||
const code = t.code ?? t.lang_code;
|
||||
if (!code || !SUPPORTED_LANG_SET.has(code)) continue;
|
||||
if (code === sourceLang) continue; // skip same-language translations
|
||||
if (!t.word?.trim()) continue;
|
||||
|
||||
const key = `${code}:${t.word.trim()}`;
|
||||
if (seen.has(key)) continue;
|
||||
seen.add(key);
|
||||
|
||||
result.push({
|
||||
target_lang: code as SupportedLanguageCode,
|
||||
word: t.word.trim(),
|
||||
sense_hint: t.sense?.trim() ?? null,
|
||||
});
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
function extractExamples(sense: KaikkiSense): string[] {
|
||||
return (sense.examples ?? [])
|
||||
.map((e) => e.text?.trim())
|
||||
.filter((t): t is string => !!t);
|
||||
}
|
||||
|
||||
function processEntry(
|
||||
entry: KaikkiEntry,
|
||||
sourceLang: SupportedLanguageCode,
|
||||
): Omit<ExtractedSense, "sense_index">[] {
|
||||
const pos = mapPos(entry.pos ?? "");
|
||||
if (!pos) return [];
|
||||
if (!entry.word?.trim()) return [];
|
||||
|
||||
// For non-English files, only process entries in the target language
|
||||
const entryLang = (entry as Record<string, unknown>)["lang_code"] as
|
||||
| string
|
||||
| undefined;
|
||||
if (sourceLang !== "en" && entryLang !== sourceLang) return [];
|
||||
|
||||
const headword = entry.word.trim();
|
||||
const results: Omit<ExtractedSense, "sense_index">[] = [];
|
||||
|
||||
for (const sense of entry.senses ?? []) {
|
||||
const gloss = sense.glosses?.[0]?.trim() ?? null;
|
||||
|
||||
if (gloss && isAbbreviation(gloss)) continue;
|
||||
|
||||
if (sourceLang === "en") {
|
||||
// English: require translations in supported languages
|
||||
const translations = extractTranslations(sense, sourceLang);
|
||||
if (translations.length === 0) continue;
|
||||
results.push({
|
||||
headword,
|
||||
language: sourceLang,
|
||||
pos,
|
||||
gloss,
|
||||
examples: extractExamples(sense),
|
||||
translations,
|
||||
});
|
||||
} else {
|
||||
// Non-English: just extract the entry, no translations needed
|
||||
results.push({
|
||||
headword,
|
||||
language: sourceLang,
|
||||
pos,
|
||||
gloss,
|
||||
examples: extractExamples(sense),
|
||||
translations: [],
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return results;
|
||||
}
|
||||
|
||||
// ── Extract ───────────────────────────────────────────────────────────────────
|
||||
|
||||
export async function extract(
|
||||
lang: SupportedLanguageCode,
|
||||
sampleLimit?: number,
|
||||
): Promise<void> {
|
||||
const filename = LANG_TO_FILE[lang];
|
||||
const sourcePath = path.join(SOURCES_DIR, filename);
|
||||
const outputPath = path.join(OUTPUT_DIR, `${lang}.json`);
|
||||
|
||||
console.log(`\nExtracting ${lang}...`);
|
||||
console.log(` Source: ${sourcePath}`);
|
||||
if (sampleLimit) console.log(` Sample mode: ${sampleLimit} entries`);
|
||||
|
||||
await fs.promises.mkdir(OUTPUT_DIR, { recursive: true });
|
||||
|
||||
const fileStream = fs.createReadStream(sourcePath);
|
||||
const rl = readline.createInterface({
|
||||
input: fileStream,
|
||||
crlfDelay: Infinity,
|
||||
});
|
||||
|
||||
const senses: ExtractedSense[] = [];
|
||||
const senseIndexMap = new Map<string, number>();
|
||||
let linesRead = 0;
|
||||
let entriesProcessed = 0;
|
||||
let entriesSkipped = 0;
|
||||
|
||||
for await (const line of rl) {
|
||||
if (!line.trim()) continue;
|
||||
if (sampleLimit && entriesProcessed >= sampleLimit) break;
|
||||
|
||||
linesRead++;
|
||||
|
||||
let entry: KaikkiEntry;
|
||||
try {
|
||||
entry = JSON.parse(line) as KaikkiEntry;
|
||||
} catch {
|
||||
console.warn(` Warning: failed to parse line ${linesRead}, skipping`);
|
||||
continue;
|
||||
}
|
||||
|
||||
const extracted = processEntry(entry, lang);
|
||||
|
||||
if (extracted.length === 0) {
|
||||
entriesSkipped++;
|
||||
continue;
|
||||
}
|
||||
|
||||
for (const sense of extracted) {
|
||||
const key = `${sense.headword}|${sense.pos}`;
|
||||
const senseIndex = senseIndexMap.get(key) ?? 0;
|
||||
senseIndexMap.set(key, senseIndex + 1);
|
||||
senses.push({ ...sense, sense_index: senseIndex });
|
||||
}
|
||||
|
||||
entriesProcessed++;
|
||||
|
||||
if (entriesProcessed % 10_000 === 0) {
|
||||
console.log(
|
||||
` Processed ${entriesProcessed.toLocaleString()} entries...`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
await fs.promises.writeFile(
|
||||
outputPath,
|
||||
JSON.stringify(senses, null, 2),
|
||||
"utf-8",
|
||||
);
|
||||
|
||||
console.log(` Lines read: ${linesRead.toLocaleString()}`);
|
||||
console.log(` Entries processed: ${entriesProcessed.toLocaleString()}`);
|
||||
console.log(` Entries skipped: ${entriesSkipped.toLocaleString()}`);
|
||||
console.log(` Senses extracted: ${senses.length.toLocaleString()}`);
|
||||
console.log(` Output: ${outputPath}`);
|
||||
}
|
||||
|
||||
// ── Main ─────────────────────────────────────────────────────────────────────
|
||||
|
||||
async function main(): Promise<void> {
|
||||
// Hardcoded sample limit for development — remove for full extraction
|
||||
const SAMPLE = 500;
|
||||
|
||||
for (const lang of SUPPORTED_LANGUAGE_CODES) {
|
||||
await extract(lang, SAMPLE);
|
||||
}
|
||||
|
||||
console.log("\nExtraction complete.");
|
||||
}
|
||||
|
||||
if (import.meta.url === `file://${process.argv[1]}`) {
|
||||
main().catch((err) => {
|
||||
console.error(err);
|
||||
process.exit(1);
|
||||
});
|
||||
}
|
||||
|
|
@ -1,109 +0,0 @@
|
|||
import { openDb } from "../../db/index.js";
|
||||
|
||||
// ── Types ─────────────────────────────────────────────────────────────────────
|
||||
|
||||
type TranslationRow = {
|
||||
translation_id: number;
|
||||
entry_id: number;
|
||||
entry_language: string;
|
||||
entry_headword: string;
|
||||
target_lang: string;
|
||||
word: string;
|
||||
sense_hint: string | null;
|
||||
};
|
||||
|
||||
type EntryRow = { id: number };
|
||||
|
||||
// ── Sync ──────────────────────────────────────────────────────────────────────
|
||||
|
||||
export function reverseLink(): void {
|
||||
const db = openDb();
|
||||
|
||||
// Find all translations and their source entry details
|
||||
const translations = db
|
||||
.prepare(
|
||||
`SELECT
|
||||
t.id AS translation_id,
|
||||
t.entry_id,
|
||||
e.language AS entry_language,
|
||||
e.headword AS entry_headword,
|
||||
t.target_lang,
|
||||
t.word,
|
||||
t.sense_hint
|
||||
FROM translations t
|
||||
JOIN entries e ON e.id = t.entry_id`,
|
||||
)
|
||||
.all() as TranslationRow[];
|
||||
|
||||
console.log(
|
||||
` Found ${translations.length.toLocaleString()} translations to check`,
|
||||
);
|
||||
|
||||
const findEntry = db.prepare(
|
||||
`SELECT id FROM entries WHERE headword = ? AND language = ? LIMIT 1`,
|
||||
);
|
||||
|
||||
const insertReverseLink = db.prepare(
|
||||
`INSERT INTO translations (entry_id, target_lang, word, sense_hint, source)
|
||||
VALUES (?, ?, ?, ?, 'reverse_link')
|
||||
ON CONFLICT (entry_id, target_lang, word) DO NOTHING`,
|
||||
);
|
||||
|
||||
const sync = db.transaction(() => {
|
||||
let inserted = 0;
|
||||
let skipped = 0;
|
||||
let noEntry = 0;
|
||||
|
||||
for (const t of translations) {
|
||||
// Look for an entry in the target language with the translation word as headword
|
||||
const targetEntry = findEntry.get(t.word, t.target_lang) as
|
||||
| EntryRow
|
||||
| undefined;
|
||||
|
||||
if (!targetEntry) {
|
||||
noEntry++;
|
||||
continue;
|
||||
}
|
||||
|
||||
// Insert reverse link: target entry → source language → source headword
|
||||
const result = insertReverseLink.run(
|
||||
targetEntry.id,
|
||||
t.entry_language,
|
||||
t.entry_headword,
|
||||
t.sense_hint ?? null,
|
||||
);
|
||||
|
||||
if (result.changes > 0) {
|
||||
inserted++;
|
||||
} else {
|
||||
skipped++;
|
||||
}
|
||||
}
|
||||
|
||||
return { inserted, skipped, noEntry };
|
||||
});
|
||||
|
||||
const counts = sync();
|
||||
|
||||
db.close();
|
||||
|
||||
console.log(` Inserted: ${counts.inserted.toLocaleString()} reverse links`);
|
||||
console.log(
|
||||
` Skipped: ${counts.skipped.toLocaleString()} (already existed)`,
|
||||
);
|
||||
console.log(
|
||||
` No entry: ${counts.noEntry.toLocaleString()} (target word not in entries)`,
|
||||
);
|
||||
}
|
||||
|
||||
// ── Main ─────────────────────────────────────────────────────────────────────
|
||||
|
||||
function main(): void {
|
||||
console.log("Running reverse link sync...");
|
||||
reverseLink();
|
||||
console.log("\nReverse link sync complete.");
|
||||
}
|
||||
|
||||
if (import.meta.url === `file://${process.argv[1]}`) {
|
||||
main();
|
||||
}
|
||||
|
|
@ -1,123 +0,0 @@
|
|||
// ── Provider configuration ────────────────────────────────────────────────────
|
||||
//
|
||||
// Each provider + model combination counts as one vote in the final majority.
|
||||
// Running the same model twice is not supported — one model, one vote.
|
||||
// The `name` field is used as the model identifier in pipeline.db and must
|
||||
// be unique across all runs.
|
||||
//
|
||||
// The pipeline iterates through ALL_PROVIDERS in order, skipping models that
|
||||
// have already completed a full run and resuming models with partial progress.
|
||||
//
|
||||
// See llm-setup.md for full setup instructions and model recommendations.
|
||||
|
||||
export type ProviderConfig = {
|
||||
name: string; // unique model identifier — stored in pipeline.db
|
||||
baseURL: string;
|
||||
apiKey: string;
|
||||
model: string;
|
||||
maxTokens: number;
|
||||
};
|
||||
|
||||
// ── Local llama.cpp ───────────────────────────────────────────────────────────
|
||||
|
||||
export const LOCAL_QWEN35_4B: ProviderConfig = {
|
||||
name: "local-qwen3.5-4b",
|
||||
baseURL: "http://127.0.0.1:8080/v1",
|
||||
apiKey: "none",
|
||||
model: "qwen3.5-4b",
|
||||
maxTokens: 1024, // no reasoning overhead so 1024 is enough
|
||||
};
|
||||
|
||||
export const LOCAL_GEMMA4: ProviderConfig = {
|
||||
name: "local-gemma4-e4b",
|
||||
baseURL: "http://127.0.0.1:8080/v1",
|
||||
apiKey: "none", // llama.cpp ignores this
|
||||
model: "gemma4-e4b", // llama.cpp ignores model name, uses loaded model
|
||||
maxTokens: 2048,
|
||||
};
|
||||
|
||||
export const LOCAL_QWEN7B: ProviderConfig = {
|
||||
name: "local-qwen2.5-7b",
|
||||
baseURL: "http://127.0.0.1:8080/v1",
|
||||
apiKey: "none",
|
||||
model: "qwen2.5-7b",
|
||||
maxTokens: 512,
|
||||
};
|
||||
|
||||
// ── OpenRouter — free tier ────────────────────────────────────────────────────
|
||||
|
||||
export const OR_QWEN3_480B: ProviderConfig = {
|
||||
name: "or-qwen3-480b",
|
||||
baseURL: "https://openrouter.ai/api/v1",
|
||||
apiKey: process.env["OPENROUTER_API_KEY"] ?? "",
|
||||
model: "qwen/qwen3-coder:free",
|
||||
maxTokens: 512,
|
||||
};
|
||||
|
||||
export const OR_GEMMA4_31B: ProviderConfig = {
|
||||
name: "or-gemma4-31b",
|
||||
baseURL: "https://openrouter.ai/api/v1",
|
||||
apiKey: process.env["OPENROUTER_API_KEY"] ?? "",
|
||||
model: "google/gemma-4-31b-it:free",
|
||||
maxTokens: 512,
|
||||
};
|
||||
|
||||
export const OR_QWEN3_80B: ProviderConfig = {
|
||||
name: "or-qwen3-80b",
|
||||
baseURL: "https://openrouter.ai/api/v1",
|
||||
apiKey: process.env["OPENROUTER_API_KEY"] ?? "",
|
||||
model: "qwen/qwen3-next-80b-a3b-instruct:free",
|
||||
maxTokens: 512,
|
||||
};
|
||||
|
||||
export const OR_NEMOTRON: ProviderConfig = {
|
||||
name: "or-nemotron-120b",
|
||||
baseURL: "https://openrouter.ai/api/v1",
|
||||
apiKey: process.env["OPENROUTER_API_KEY"] ?? "",
|
||||
model: "nvidia/nemotron-3-super-120b-a12b:free",
|
||||
maxTokens: 512,
|
||||
};
|
||||
|
||||
// ── Anthropic — reference baseline ───────────────────────────────────────────
|
||||
// Note: Anthropic uses a different API format. An adapter is required.
|
||||
// See llm-setup.md for details.
|
||||
|
||||
export const ANTHROPIC_SONNET: ProviderConfig = {
|
||||
name: "anthropic-sonnet-4",
|
||||
baseURL: "https://api.anthropic.com/v1",
|
||||
apiKey: process.env["ANTHROPIC_API_KEY"] ?? "",
|
||||
model: "claude-sonnet-4-6",
|
||||
maxTokens: 512,
|
||||
};
|
||||
|
||||
// ── All configured providers ──────────────────────────────────────────────────
|
||||
// The pipeline runs through these in order — local models first, then cloud.
|
||||
// Add new providers here to include them in the voting pool.
|
||||
|
||||
export const ALL_PROVIDERS: ProviderConfig[] = [
|
||||
LOCAL_QWEN35_4B,
|
||||
// LOCAL_GEMMA4,
|
||||
// LOCAL_QWEN7B,
|
||||
// OR_QWEN3_480B,
|
||||
// OR_GEMMA4_31B,
|
||||
// OR_QWEN3_80B,
|
||||
// OR_NEMOTRON,
|
||||
// ANTHROPIC_SONNET,
|
||||
];
|
||||
|
||||
// ── Key validation ────────────────────────────────────────────────────────────
|
||||
|
||||
const LOCAL_PROVIDERS = new Set(["none"]);
|
||||
|
||||
export function validateProviderKey(provider: ProviderConfig): void {
|
||||
if (LOCAL_PROVIDERS.has(provider.apiKey)) return;
|
||||
|
||||
if (!provider.apiKey) {
|
||||
const keyName = provider.name.startsWith("anthropic")
|
||||
? "ANTHROPIC_API_KEY"
|
||||
: "OPENROUTER_API_KEY";
|
||||
console.error(`\n ERROR: ${keyName} is not set in .env`);
|
||||
console.error(` Provider "${provider.name}" requires this key to run.\n`);
|
||||
process.exit(1);
|
||||
}
|
||||
}
|
||||
|
|
@ -1,877 +0,0 @@
|
|||
import { openDb } from "../../db/index.js";
|
||||
import type { ProviderConfig } from "../config.js";
|
||||
import { CEFR_LEVELS, SUPPORTED_LANGUAGE_CODES } from "@lila/shared";
|
||||
import type { SupportedLanguageCode } from "@lila/shared";
|
||||
|
||||
// ── Types ─────────────────────────────────────────────────────────────────────
|
||||
|
||||
type EntryRow = {
|
||||
id: number;
|
||||
headword: string;
|
||||
language: SupportedLanguageCode;
|
||||
pos: string;
|
||||
gloss: string | null;
|
||||
examples: string; // JSON array string
|
||||
};
|
||||
|
||||
type TranslationRow = {
|
||||
id: number;
|
||||
target_lang: SupportedLanguageCode;
|
||||
word: string;
|
||||
};
|
||||
|
||||
type GlossResult = { status: "ok" } | { status: "improved"; gloss: string };
|
||||
|
||||
type ExampleResult = { status: "ok" } | { status: "improved"; example: string };
|
||||
|
||||
type TranslationResult = {
|
||||
translations: Partial<
|
||||
Record<SupportedLanguageCode, Record<string, "ok" | "reject">>
|
||||
>;
|
||||
generated?: Partial<Record<SupportedLanguageCode, string>>;
|
||||
};
|
||||
|
||||
type CefrResult = {
|
||||
headword_cefr: string;
|
||||
translation_cefr: Partial<
|
||||
Record<SupportedLanguageCode, Record<string, string>>
|
||||
>;
|
||||
};
|
||||
|
||||
type SubStage =
|
||||
| "round1_gloss"
|
||||
| "round1_example"
|
||||
| "round1_translations"
|
||||
| "round1_cefr";
|
||||
|
||||
// ── Constants ─────────────────────────────────────────────────────────────────
|
||||
|
||||
const SUPPORTED_LANG_SET = new Set<string>(SUPPORTED_LANGUAGE_CODES);
|
||||
const CEFR_SET = new Set<string>(CEFR_LEVELS);
|
||||
|
||||
// ── Shutdown ──────────────────────────────────────────────────────────────────
|
||||
|
||||
let shutdownRequested = false;
|
||||
let currentCallController: AbortController | null = null;
|
||||
|
||||
export function registerEnrichShutdown(): void {
|
||||
const handler = (): void => {
|
||||
if (shutdownRequested) return;
|
||||
shutdownRequested = true;
|
||||
console.log("\n\n Shutdown requested — aborting current LLM call...");
|
||||
currentCallController?.abort();
|
||||
};
|
||||
process.on("SIGINT", handler);
|
||||
process.on("SIGTERM", handler);
|
||||
}
|
||||
|
||||
// ── Prompt builders ───────────────────────────────────────────────────────────
|
||||
|
||||
function buildGlossPrompt(entry: EntryRow): string {
|
||||
const glossText = entry.gloss ?? "none";
|
||||
const examples: string[] = JSON.parse(entry.examples) as string[];
|
||||
const examplesText =
|
||||
examples.length > 0 ? examples.map((e) => ` - ${e}`).join("\n") : " none";
|
||||
|
||||
return `You are a language learning expert.
|
||||
|
||||
Review this gloss for the ${entry.pos} "${entry.headword}" (sense ${entry.sense_index}).
|
||||
Gloss: "${glossText}"
|
||||
Examples of this specific sense:
|
||||
${examplesText}
|
||||
|
||||
Is this gloss clear, accurate for this specific sense, and suitable for a language learner?
|
||||
- If yes, respond with: {"status": "ok"}
|
||||
- If no or if gloss is "none", respond with: {"status": "improved", "gloss": "your improved gloss here"}
|
||||
|
||||
IMPORTANT: Your improved gloss must describe THIS SPECIFIC SENSE shown by the examples above,
|
||||
not a more common or general meaning of the word.
|
||||
|
||||
Respond ONLY with valid JSON and nothing else.`;
|
||||
}
|
||||
|
||||
function buildTranslationsPrompt(
|
||||
entry: EntryRow,
|
||||
translations: TranslationRow[],
|
||||
verifiedGloss: string,
|
||||
): string {
|
||||
const byLang = new Map<SupportedLanguageCode, string[]>();
|
||||
for (const t of translations) {
|
||||
if (!byLang.has(t.target_lang)) byLang.set(t.target_lang, []);
|
||||
byLang.get(t.target_lang)!.push(t.word);
|
||||
}
|
||||
|
||||
const coveredLangs = new Set(byLang.keys());
|
||||
const missingLangs = SUPPORTED_LANGUAGE_CODES.filter(
|
||||
(l) => l !== entry.language && !coveredLangs.has(l),
|
||||
);
|
||||
|
||||
const translationsText =
|
||||
byLang.size > 0
|
||||
? [...byLang.entries()]
|
||||
.map(([lang, words]) => ` ${lang}: ${words.join(", ")}`)
|
||||
.join("\n")
|
||||
: " none";
|
||||
|
||||
const missingText =
|
||||
missingLangs.length > 0 ? missingLangs.join(", ") : "none";
|
||||
|
||||
const exampleResponse: Record<string, unknown> = {
|
||||
translations: {
|
||||
de: { frei: "ok", "-frei": "reject" },
|
||||
it: { libero: "ok", free: "reject" },
|
||||
},
|
||||
};
|
||||
if (missingLangs.length > 0) {
|
||||
exampleResponse["generated"] = { es: "libre", fr: "libre" };
|
||||
}
|
||||
|
||||
return `You are a language learning expert.
|
||||
|
||||
For the ${entry.language} ${entry.pos} "${entry.headword}" (meaning: "${verifiedGloss}"), review these translations:
|
||||
${translationsText}
|
||||
|
||||
For each translation:
|
||||
- Write "ok" if it is a valid translation for this specific meaning
|
||||
- Write "reject" if it is wrong, a suffix (starts with -), garbled text, or the wrong language
|
||||
|
||||
Examples of correct behaviour:
|
||||
- "free" listed as Italian → "reject" (it is English, not Italian)
|
||||
- "-frei" listed as German → "reject" (it is a suffix, not a standalone word)
|
||||
- "libre" listed as Spanish → "ok" (it is a valid Spanish word)
|
||||
|
||||
${missingLangs.length > 0 ? `Also generate the single best translation for these missing languages: ${missingText}` : ""}
|
||||
|
||||
Respond ONLY with valid JSON and nothing else:
|
||||
${JSON.stringify(exampleResponse, null, 2)}`;
|
||||
}
|
||||
|
||||
function buildCefrPrompt(
|
||||
entry: EntryRow,
|
||||
verifiedGloss: string,
|
||||
validatedTranslations: Map<SupportedLanguageCode, string[]>,
|
||||
): string {
|
||||
const translationsText =
|
||||
validatedTranslations.size > 0
|
||||
? [...validatedTranslations.entries()]
|
||||
.map(([lang, words]) => ` ${lang}: ${words.join(", ")}`)
|
||||
.join("\n")
|
||||
: " none";
|
||||
|
||||
return `You are a language learning expert.
|
||||
|
||||
Assign CEFR levels (A1, A2, B1, B2, C1, or C2) to this word and its validated translations.
|
||||
Base your levels on how commonly a language learner at that level would encounter this specific sense.
|
||||
Consider register — slang, technical, and archaic words should be rated higher.
|
||||
|
||||
WORD: ${entry.headword} (${entry.pos})
|
||||
MEANING: ${verifiedGloss}
|
||||
VALIDATED TRANSLATIONS:
|
||||
${translationsText}
|
||||
|
||||
Respond ONLY with valid JSON and nothing else:
|
||||
{
|
||||
"headword_cefr": "B1",
|
||||
"translation_cefr": {
|
||||
"de": { "frei": "A2" },
|
||||
"it": { "libero": "A2" }
|
||||
}
|
||||
}`;
|
||||
}
|
||||
|
||||
// ── Validation ────────────────────────────────────────────────────────────────
|
||||
|
||||
function validateGloss(raw: string): GlossResult | null {
|
||||
try {
|
||||
const obj = JSON.parse(raw) as Record<string, unknown>;
|
||||
if (obj["status"] === "ok") return { status: "ok" };
|
||||
if (
|
||||
obj["status"] === "improved" &&
|
||||
typeof obj["gloss"] === "string" &&
|
||||
obj["gloss"].trim()
|
||||
) {
|
||||
return { status: "improved", gloss: obj["gloss"].trim() };
|
||||
}
|
||||
return null;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
function validateExample(raw: string): ExampleResult | null {
|
||||
try {
|
||||
const obj = JSON.parse(raw) as Record<string, unknown>;
|
||||
if (obj["status"] === "ok") return { status: "ok" };
|
||||
if (
|
||||
obj["status"] === "improved" &&
|
||||
typeof obj["example"] === "string" &&
|
||||
obj["example"].trim()
|
||||
) {
|
||||
return { status: "improved", example: obj["example"].trim() };
|
||||
}
|
||||
return null;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
function validateTranslations(
|
||||
raw: string,
|
||||
translations: TranslationRow[],
|
||||
): TranslationResult | null {
|
||||
try {
|
||||
const obj = JSON.parse(raw) as Record<string, unknown>;
|
||||
if (typeof obj["translations"] !== "object" || obj["translations"] === null)
|
||||
return null;
|
||||
|
||||
const result: TranslationResult = { translations: {} };
|
||||
const translationsObj = obj["translations"] as Record<string, unknown>;
|
||||
|
||||
// Validate each language's votes
|
||||
for (const [lang, votes] of Object.entries(translationsObj)) {
|
||||
if (!SUPPORTED_LANG_SET.has(lang)) continue;
|
||||
if (typeof votes !== "object" || votes === null) continue;
|
||||
|
||||
result.translations[lang as SupportedLanguageCode] = {};
|
||||
for (const [word, status] of Object.entries(
|
||||
votes as Record<string, unknown>,
|
||||
)) {
|
||||
if (status === "ok" || status === "reject") {
|
||||
result.translations[lang as SupportedLanguageCode]![word] = status;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Validate generated translations
|
||||
if (obj["generated"] !== undefined && obj["generated"] !== null) {
|
||||
if (typeof obj["generated"] !== "object") return null;
|
||||
result.generated = {};
|
||||
for (const [lang, word] of Object.entries(
|
||||
obj["generated"] as Record<string, unknown>,
|
||||
)) {
|
||||
if (!SUPPORTED_LANG_SET.has(lang)) continue;
|
||||
if (typeof word === "string" && word.trim()) {
|
||||
result.generated[lang as SupportedLanguageCode] = word.trim();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Check all translations got a vote
|
||||
const byLang = new Map<string, Set<string>>();
|
||||
for (const t of translations) {
|
||||
if (!byLang.has(t.target_lang)) byLang.set(t.target_lang, new Set());
|
||||
byLang.get(t.target_lang)!.add(t.word);
|
||||
}
|
||||
|
||||
for (const [lang, words] of byLang.entries()) {
|
||||
const votes = result.translations[lang as SupportedLanguageCode];
|
||||
if (!votes) return null;
|
||||
for (const word of words) {
|
||||
if (!votes[word]) return null;
|
||||
}
|
||||
}
|
||||
|
||||
return result;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
function validateCefr(
|
||||
raw: string,
|
||||
validatedTranslations: Map<SupportedLanguageCode, string[]>,
|
||||
): CefrResult | null {
|
||||
try {
|
||||
const obj = JSON.parse(raw) as Record<string, unknown>;
|
||||
if (typeof obj["headword_cefr"] !== "string") return null;
|
||||
if (!CEFR_SET.has(obj["headword_cefr"])) return null;
|
||||
if (
|
||||
typeof obj["translation_cefr"] !== "object" ||
|
||||
obj["translation_cefr"] === null
|
||||
)
|
||||
return null;
|
||||
|
||||
const translationCefr = obj["translation_cefr"] as Record<string, unknown>;
|
||||
|
||||
// Verify all validated translations have a CEFR vote
|
||||
for (const [lang, words] of validatedTranslations.entries()) {
|
||||
const votes = translationCefr[lang] as Record<string, string> | undefined;
|
||||
if (!votes) return null;
|
||||
for (const word of words) {
|
||||
if (!votes[word] || !CEFR_SET.has(votes[word])) return null;
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
headword_cefr: obj["headword_cefr"],
|
||||
translation_cefr: translationCefr as Partial<
|
||||
Record<SupportedLanguageCode, Record<string, string>>
|
||||
>,
|
||||
};
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
// ── LLM call ──────────────────────────────────────────────────────────────────
|
||||
|
||||
async function callLlm(
|
||||
prompt: string,
|
||||
provider: ProviderConfig,
|
||||
): Promise<string> {
|
||||
currentCallController = new AbortController();
|
||||
const timeout = setTimeout(() => currentCallController?.abort(), 120_000);
|
||||
|
||||
let response: Response;
|
||||
try {
|
||||
response = await fetch(`${provider.baseURL}/chat/completions`, {
|
||||
method: "POST",
|
||||
signal: currentCallController.signal,
|
||||
headers: {
|
||||
"Content-Type": "application/json",
|
||||
Authorization: `Bearer ${provider.apiKey}`,
|
||||
},
|
||||
body: JSON.stringify({
|
||||
model: provider.model,
|
||||
max_tokens: provider.maxTokens,
|
||||
messages: [{ role: "user", content: prompt }],
|
||||
temperature: 0.1,
|
||||
}),
|
||||
});
|
||||
} finally {
|
||||
clearTimeout(timeout);
|
||||
currentCallController = null;
|
||||
}
|
||||
|
||||
if (!response.ok) {
|
||||
throw new Error(`LLM API error: ${response.status} ${response.statusText}`);
|
||||
}
|
||||
|
||||
const data = (await response.json()) as {
|
||||
choices?: { message?: { content?: string } }[];
|
||||
};
|
||||
|
||||
const content = data.choices?.[0]?.message?.content;
|
||||
if (!content) throw new Error("LLM returned empty response");
|
||||
|
||||
return content
|
||||
.replace(/```json\n?/g, "")
|
||||
.replace(/```\n?/g, "")
|
||||
.trim();
|
||||
}
|
||||
|
||||
// ── Status helpers ────────────────────────────────────────────────────────────
|
||||
|
||||
function getSubStageStatus(
|
||||
entryId: number,
|
||||
modelName: string,
|
||||
stage: SubStage,
|
||||
): "complete" | "needs_review" | "pending" {
|
||||
const db = openDb();
|
||||
const row = db
|
||||
.prepare(
|
||||
`SELECT status FROM run_status
|
||||
WHERE entry_id = ? AND model_name = ? AND stage = ?`,
|
||||
)
|
||||
.get(entryId, modelName, stage) as { status: string } | undefined;
|
||||
db.close();
|
||||
if (!row) return "pending";
|
||||
if (row.status === "complete") return "complete";
|
||||
if (row.status === "needs_review") return "needs_review";
|
||||
return "pending";
|
||||
}
|
||||
|
||||
function markSubStage(
|
||||
entryId: number,
|
||||
modelName: string,
|
||||
stage: SubStage,
|
||||
status: "complete" | "needs_review",
|
||||
): void {
|
||||
const db = openDb();
|
||||
db.prepare(
|
||||
`INSERT INTO run_status (entry_id, model_name, stage, status)
|
||||
VALUES (?, ?, ?, ?)
|
||||
ON CONFLICT (entry_id, model_name, stage)
|
||||
DO UPDATE SET status = ?, updated_at = datetime('now')`,
|
||||
).run(entryId, modelName, stage, status, status);
|
||||
db.close();
|
||||
}
|
||||
|
||||
// ── Write helpers ─────────────────────────────────────────────────────────────
|
||||
|
||||
function writeGloss(
|
||||
entryId: number,
|
||||
modelName: string,
|
||||
result: GlossResult,
|
||||
): void {
|
||||
if (result.status === "improved") {
|
||||
const db = openDb();
|
||||
db.prepare(
|
||||
`INSERT INTO generated_glosses (entry_id, model_name, text)
|
||||
VALUES (?, ?, ?)
|
||||
ON CONFLICT (entry_id, model_name) DO NOTHING`,
|
||||
).run(entryId, modelName, result.gloss);
|
||||
db.close();
|
||||
}
|
||||
}
|
||||
|
||||
function writeExample(
|
||||
entryId: number,
|
||||
modelName: string,
|
||||
result: ExampleResult,
|
||||
): void {
|
||||
if (result.status === "improved") {
|
||||
const db = openDb();
|
||||
db.prepare(
|
||||
`INSERT INTO generated_examples (entry_id, model_name, text)
|
||||
VALUES (?, ?, ?)
|
||||
ON CONFLICT (entry_id, model_name) DO NOTHING`,
|
||||
).run(entryId, modelName, result.example);
|
||||
db.close();
|
||||
}
|
||||
}
|
||||
|
||||
function writeTranslations(
|
||||
entryId: number,
|
||||
modelName: string,
|
||||
result: TranslationResult,
|
||||
translations: TranslationRow[],
|
||||
): void {
|
||||
const db = openDb();
|
||||
|
||||
db.transaction(() => {
|
||||
// Write rejections
|
||||
for (const t of translations) {
|
||||
const vote = result.translations[t.target_lang]?.[t.word];
|
||||
if (vote === "reject") {
|
||||
db.prepare(
|
||||
`INSERT INTO model_translation_rejections (translation_id, model_name)
|
||||
VALUES (?, ?)
|
||||
ON CONFLICT (translation_id, model_name) DO NOTHING`,
|
||||
).run(t.id, modelName);
|
||||
}
|
||||
}
|
||||
|
||||
// Write generated translations
|
||||
if (result.generated) {
|
||||
for (const [lang, word] of Object.entries(result.generated)) {
|
||||
db.prepare(
|
||||
`INSERT INTO generated_translations (entry_id, model_name, target_lang, word)
|
||||
VALUES (?, ?, ?, ?)
|
||||
ON CONFLICT (entry_id, model_name, target_lang) DO NOTHING`,
|
||||
).run(entryId, modelName, lang, word);
|
||||
}
|
||||
}
|
||||
})();
|
||||
|
||||
db.close();
|
||||
}
|
||||
|
||||
function writeCefr(
|
||||
entryId: number,
|
||||
modelName: string,
|
||||
result: CefrResult,
|
||||
translations: TranslationRow[],
|
||||
): void {
|
||||
const db = openDb();
|
||||
|
||||
db.transaction(() => {
|
||||
// Headword CEFR
|
||||
db.prepare(
|
||||
`INSERT INTO model_entry_cefr_votes (entry_id, model_name, cefr_level)
|
||||
VALUES (?, ?, ?)
|
||||
ON CONFLICT (entry_id, model_name) DO NOTHING`,
|
||||
).run(entryId, modelName, result.headword_cefr);
|
||||
|
||||
// Translation CEFR votes
|
||||
for (const t of translations) {
|
||||
const level = result.translation_cefr[t.target_lang]?.[t.word];
|
||||
if (level && CEFR_SET.has(level)) {
|
||||
db.prepare(
|
||||
`INSERT INTO model_translation_cefr_votes (translation_id, model_name, cefr_level)
|
||||
VALUES (?, ?, ?)
|
||||
ON CONFLICT (translation_id, model_name) DO NOTHING`,
|
||||
).run(t.id, modelName, level);
|
||||
}
|
||||
}
|
||||
})();
|
||||
|
||||
db.close();
|
||||
}
|
||||
|
||||
// ── Progress ──────────────────────────────────────────────────────────────────
|
||||
|
||||
function updateProgress(
|
||||
processed: number,
|
||||
needsReview: number,
|
||||
total: number,
|
||||
llmMs: number,
|
||||
startTime: number,
|
||||
): void {
|
||||
const totalProcessed = processed + needsReview;
|
||||
const pct = ((totalProcessed / total) * 100).toFixed(1);
|
||||
const elapsed = (Date.now() - startTime) / 1000;
|
||||
const rate = elapsed > 0 ? totalProcessed / elapsed : 0;
|
||||
const remaining = rate > 0 ? (total - totalProcessed) / rate : 0;
|
||||
const eta =
|
||||
remaining === 0
|
||||
? "calculating..."
|
||||
: remaining < 60
|
||||
? `${Math.round(remaining)}s`
|
||||
: `${Math.round(remaining / 60)}m`;
|
||||
const totalElapsedStr =
|
||||
elapsed < 60
|
||||
? `${Math.round(elapsed)}s`
|
||||
: `${Math.floor(elapsed / 60)}m ${Math.round(elapsed % 60)}s`;
|
||||
|
||||
process.stdout.write(
|
||||
`\r ${totalProcessed}/${total} (${pct}%) — entry: ${(llmMs / 1000).toFixed(1)}s — total: ${totalElapsedStr} — ETA: ${eta} `,
|
||||
);
|
||||
}
|
||||
|
||||
// ── Main enrich function ──────────────────────────────────────────────────────
|
||||
|
||||
export async function enrich(
|
||||
provider: ProviderConfig,
|
||||
): Promise<{ processed: number; skipped: number; needsReview: number }> {
|
||||
registerEnrichShutdown();
|
||||
const db = openDb();
|
||||
|
||||
const allEntries = db
|
||||
.prepare(`SELECT * FROM entries WHERE language = 'en'`)
|
||||
.all() as EntryRow[];
|
||||
|
||||
// An entry is fully complete when all 4 sub-stages are complete
|
||||
const completeEntries = db
|
||||
.prepare(
|
||||
`SELECT entry_id FROM run_status
|
||||
WHERE model_name = ? AND stage = 'round1_gloss'
|
||||
AND status = 'complete'`,
|
||||
)
|
||||
.all(provider.name) as { entry_id: number }[];
|
||||
|
||||
const completeIds = new Set(completeEntries.map((r) => r.entry_id));
|
||||
const pending = allEntries.filter((e) => !completeIds.has(e.id)).slice(0, 50);
|
||||
|
||||
db.close();
|
||||
|
||||
console.log(`\n Model: ${provider.name}`);
|
||||
console.log(` Total entries: ${allEntries.length.toLocaleString()}`);
|
||||
console.log(` Already complete: ${completeIds.size.toLocaleString()}`);
|
||||
console.log(` Pending: ${pending.length.toLocaleString()}`);
|
||||
|
||||
if (pending.length === 0) {
|
||||
console.log(" Nothing to process.");
|
||||
return { processed: 0, skipped: completeIds.size, needsReview: 0 };
|
||||
}
|
||||
|
||||
let processedCount = 0;
|
||||
let needsReviewCount = 0;
|
||||
let llmMs = 0;
|
||||
const startTime = Date.now();
|
||||
|
||||
for (const entry of pending) {
|
||||
if (shutdownRequested) break;
|
||||
|
||||
const db2 = openDb();
|
||||
const translations = db2
|
||||
.prepare(
|
||||
`SELECT id, target_lang, word FROM translations WHERE entry_id = ? AND source = 'kaikki'`,
|
||||
)
|
||||
.all(entry.id) as TranslationRow[];
|
||||
db2.close();
|
||||
|
||||
let entryFailed = false;
|
||||
|
||||
// ── Sub-stage 1: Gloss ────────────────────────────────────────────────────
|
||||
|
||||
let verifiedGloss = entry.gloss ?? "";
|
||||
|
||||
if (
|
||||
getSubStageStatus(entry.id, provider.name, "round1_gloss") !== "complete"
|
||||
) {
|
||||
try {
|
||||
const llmStart = Date.now();
|
||||
const raw = await callLlm(buildGlossPrompt(entry), provider);
|
||||
llmMs = Date.now() - llmStart;
|
||||
|
||||
const result = validateGloss(raw);
|
||||
if (!result) {
|
||||
markSubStage(entry.id, provider.name, "round1_gloss", "needs_review");
|
||||
console.warn(
|
||||
`\n needs_review: entry ${entry.id} round1_gloss — invalid response`,
|
||||
);
|
||||
entryFailed = true;
|
||||
} else {
|
||||
writeGloss(entry.id, provider.name, result);
|
||||
if (result.status === "improved") verifiedGloss = result.gloss;
|
||||
markSubStage(entry.id, provider.name, "round1_gloss", "complete");
|
||||
}
|
||||
} catch (err) {
|
||||
llmMs = 0;
|
||||
const message = err instanceof Error ? err.message : String(err);
|
||||
markSubStage(entry.id, provider.name, "round1_gloss", "needs_review");
|
||||
console.warn(
|
||||
`\n needs_review: entry ${entry.id} round1_gloss — ${message}`,
|
||||
);
|
||||
entryFailed = true;
|
||||
}
|
||||
}
|
||||
|
||||
if (entryFailed) {
|
||||
needsReviewCount++;
|
||||
updateProgress(
|
||||
processedCount,
|
||||
needsReviewCount,
|
||||
pending.length,
|
||||
llmMs,
|
||||
startTime,
|
||||
);
|
||||
continue;
|
||||
}
|
||||
|
||||
/*
|
||||
// ── Sub-stages 2, 3, 4 — not yet active ──────────────────────────────────
|
||||
// ── Sub-stage 2: Example ──────────────────────────────────────────────────
|
||||
|
||||
if (
|
||||
getSubStageStatus(entry.id, provider.name, "round1_example") !==
|
||||
"complete"
|
||||
) {
|
||||
try {
|
||||
const llmStart = Date.now();
|
||||
const raw = await callLlm(
|
||||
buildExamplePrompt(entry, verifiedGloss),
|
||||
provider,
|
||||
);
|
||||
llmMs = Date.now() - llmStart;
|
||||
|
||||
const result = validateExample(raw);
|
||||
if (!result) {
|
||||
markSubStage(
|
||||
entry.id,
|
||||
provider.name,
|
||||
"round1_example",
|
||||
"needs_review",
|
||||
);
|
||||
console.warn(
|
||||
`\n needs_review: entry ${entry.id} round1_example — invalid response`,
|
||||
);
|
||||
entryFailed = true;
|
||||
} else {
|
||||
writeExample(entry.id, provider.name, result);
|
||||
markSubStage(entry.id, provider.name, "round1_example", "complete");
|
||||
}
|
||||
} catch (err) {
|
||||
llmMs = 0;
|
||||
const message = err instanceof Error ? err.message : String(err);
|
||||
markSubStage(entry.id, provider.name, "round1_example", "needs_review");
|
||||
console.warn(
|
||||
`\n needs_review: entry ${entry.id} round1_example — ${message}`,
|
||||
);
|
||||
entryFailed = true;
|
||||
}
|
||||
}
|
||||
|
||||
if (entryFailed) {
|
||||
needsReviewCount++;
|
||||
updateProgress(
|
||||
processedCount,
|
||||
needsReviewCount,
|
||||
pending.length,
|
||||
llmMs,
|
||||
startTime,
|
||||
);
|
||||
continue;
|
||||
}
|
||||
|
||||
// ── Sub-stage 3: Translations ─────────────────────────────────────────────
|
||||
|
||||
const validatedTranslations = new Map<SupportedLanguageCode, string[]>();
|
||||
|
||||
if (
|
||||
getSubStageStatus(entry.id, provider.name, "round1_translations") !==
|
||||
"complete"
|
||||
) {
|
||||
try {
|
||||
const llmStart = Date.now();
|
||||
const raw = await callLlm(
|
||||
buildTranslationsPrompt(entry, translations, verifiedGloss),
|
||||
provider,
|
||||
);
|
||||
llmMs = Date.now() - llmStart;
|
||||
|
||||
const result = validateTranslations(raw, translations);
|
||||
if (!result) {
|
||||
markSubStage(
|
||||
entry.id,
|
||||
provider.name,
|
||||
"round1_translations",
|
||||
"needs_review",
|
||||
);
|
||||
console.warn(
|
||||
`\n needs_review: entry ${entry.id} round1_translations — invalid response`,
|
||||
);
|
||||
entryFailed = true;
|
||||
} else {
|
||||
writeTranslations(entry.id, provider.name, result, translations);
|
||||
markSubStage(
|
||||
entry.id,
|
||||
provider.name,
|
||||
"round1_translations",
|
||||
"complete",
|
||||
);
|
||||
|
||||
// Build validated translations map for CEFR sub-stage
|
||||
// Include kaikki translations that were ok'd + generated translations
|
||||
for (const t of translations) {
|
||||
const vote = result.translations[t.target_lang]?.[t.word];
|
||||
if (vote === "ok") {
|
||||
if (!validatedTranslations.has(t.target_lang)) {
|
||||
validatedTranslations.set(t.target_lang, []);
|
||||
}
|
||||
validatedTranslations.get(t.target_lang)!.push(t.word);
|
||||
}
|
||||
}
|
||||
if (result.generated) {
|
||||
for (const [lang, word] of Object.entries(result.generated)) {
|
||||
const l = lang as SupportedLanguageCode;
|
||||
if (!validatedTranslations.has(l))
|
||||
validatedTranslations.set(l, []);
|
||||
validatedTranslations.get(l)!.push(word);
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (err) {
|
||||
llmMs = 0;
|
||||
const message = err instanceof Error ? err.message : String(err);
|
||||
markSubStage(
|
||||
entry.id,
|
||||
provider.name,
|
||||
"round1_translations",
|
||||
"needs_review",
|
||||
);
|
||||
console.warn(
|
||||
`\n needs_review: entry ${entry.id} round1_translations — ${message}`,
|
||||
);
|
||||
entryFailed = true;
|
||||
}
|
||||
} else {
|
||||
// Already complete — rebuild validated translations from db
|
||||
const db3 = openDb();
|
||||
const rejections = new Set(
|
||||
(
|
||||
db3
|
||||
.prepare(
|
||||
`SELECT translation_id FROM model_translation_rejections WHERE model_name = ?`,
|
||||
)
|
||||
.all(provider.name) as { translation_id: number }[]
|
||||
).map((r) => r.translation_id),
|
||||
);
|
||||
for (const t of translations) {
|
||||
if (!rejections.has(t.id)) {
|
||||
if (!validatedTranslations.has(t.target_lang)) {
|
||||
validatedTranslations.set(t.target_lang, []);
|
||||
}
|
||||
validatedTranslations.get(t.target_lang)!.push(t.word);
|
||||
}
|
||||
}
|
||||
const generated = db3
|
||||
.prepare(
|
||||
`SELECT target_lang, word FROM generated_translations WHERE entry_id = ? AND model_name = ?`,
|
||||
)
|
||||
.all(entry.id, provider.name) as {
|
||||
target_lang: SupportedLanguageCode;
|
||||
word: string;
|
||||
}[];
|
||||
for (const g of generated) {
|
||||
if (!validatedTranslations.has(g.target_lang))
|
||||
validatedTranslations.set(g.target_lang, []);
|
||||
validatedTranslations.get(g.target_lang)!.push(g.word);
|
||||
}
|
||||
db3.close();
|
||||
}
|
||||
|
||||
if (entryFailed) {
|
||||
needsReviewCount++;
|
||||
updateProgress(
|
||||
processedCount,
|
||||
needsReviewCount,
|
||||
pending.length,
|
||||
llmMs,
|
||||
startTime,
|
||||
);
|
||||
continue;
|
||||
}
|
||||
|
||||
// ── Sub-stage 4: CEFR ─────────────────────────────────────────────────────
|
||||
|
||||
if (
|
||||
getSubStageStatus(entry.id, provider.name, "round1_cefr") !== "complete"
|
||||
) {
|
||||
try {
|
||||
const llmStart = Date.now();
|
||||
const raw = await callLlm(
|
||||
buildCefrPrompt(entry, verifiedGloss, validatedTranslations),
|
||||
provider,
|
||||
);
|
||||
llmMs = Date.now() - llmStart;
|
||||
|
||||
const result = validateCefr(raw, validatedTranslations);
|
||||
if (!result) {
|
||||
markSubStage(entry.id, provider.name, "round1_cefr", "needs_review");
|
||||
console.warn(
|
||||
`\n needs_review: entry ${entry.id} round1_cefr — invalid response`,
|
||||
);
|
||||
needsReviewCount++;
|
||||
} else {
|
||||
// Get translation rows for validated words only
|
||||
const validatedRows = translations.filter((t) => {
|
||||
return validatedTranslations.get(t.target_lang)?.includes(t.word);
|
||||
});
|
||||
writeCefr(entry.id, provider.name, result, validatedRows);
|
||||
markSubStage(entry.id, provider.name, "round1_cefr", "complete");
|
||||
processedCount++;
|
||||
}
|
||||
} catch (err) {
|
||||
llmMs = 0;
|
||||
const message = err instanceof Error ? err.message : String(err);
|
||||
markSubStage(entry.id, provider.name, "round1_cefr", "needs_review");
|
||||
console.warn(
|
||||
`\n needs_review: entry ${entry.id} round1_cefr — ${message}`,
|
||||
);
|
||||
needsReviewCount++;
|
||||
}
|
||||
} else {
|
||||
processedCount++;
|
||||
}
|
||||
|
||||
*/
|
||||
|
||||
processedCount++;
|
||||
updateProgress(
|
||||
processedCount,
|
||||
needsReviewCount,
|
||||
pending.length,
|
||||
llmMs,
|
||||
startTime,
|
||||
);
|
||||
}
|
||||
|
||||
process.stdout.write("\n");
|
||||
const totalMs = Date.now() - startTime;
|
||||
const totalMin = Math.floor(totalMs / 60_000);
|
||||
const totalSec = Math.round((totalMs % 60_000) / 1000);
|
||||
console.log(` Total time: ${totalMin}m ${totalSec}s`);
|
||||
console.log(
|
||||
` Avg per entry: ${(totalMs / Math.max(processedCount + needsReviewCount, 1) / 1000).toFixed(1)}s`,
|
||||
);
|
||||
console.log(` Processed: ${processedCount.toLocaleString()}`);
|
||||
console.log(` Needs review: ${needsReviewCount.toLocaleString()}`);
|
||||
|
||||
return {
|
||||
processed: processedCount,
|
||||
skipped: completeIds.size,
|
||||
needsReview: needsReviewCount,
|
||||
};
|
||||
}
|
||||
|
|
@ -1,230 +0,0 @@
|
|||
import fs from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import { describe, it, expect, beforeAll } from "vitest";
|
||||
import { SUPPORTED_LANGUAGE_CODES } from "@lila/shared";
|
||||
import type { SupportedLanguageCode, SupportedPos } from "@lila/shared";
|
||||
|
||||
// ── Types ─────────────────────────────────────────────────────────────────────
|
||||
|
||||
type ExtractedSense = {
|
||||
headword: string;
|
||||
language: SupportedLanguageCode;
|
||||
pos: SupportedPos;
|
||||
sense_index: number;
|
||||
gloss: string | null;
|
||||
examples: string[];
|
||||
translations: {
|
||||
target_lang: SupportedLanguageCode;
|
||||
word: string;
|
||||
sense_hint: string | null;
|
||||
}[];
|
||||
};
|
||||
|
||||
// ── Paths ─────────────────────────────────────────────────────────────────────
|
||||
|
||||
const DB_PATH = path.resolve("db/pipeline.db");
|
||||
const OUTPUT_DIR = path.resolve("stage-1-extract/output");
|
||||
|
||||
// ── Helpers ───────────────────────────────────────────────────────────────────
|
||||
|
||||
async function dbExists(): Promise<boolean> {
|
||||
try {
|
||||
await fs.access(DB_PATH);
|
||||
return true;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
// ── Tests ─────────────────────────────────────────────────────────────────────
|
||||
|
||||
describe("pipeline.db — import validation", () => {
|
||||
let db: import("better-sqlite3").Database;
|
||||
let expectedEntriesByLang: Map<SupportedLanguageCode, number>;
|
||||
let expectedTotalTranslations: number;
|
||||
|
||||
beforeAll(async () => {
|
||||
if (!(await dbExists())) return;
|
||||
|
||||
const Database = (await import("better-sqlite3")).default;
|
||||
db = new Database(DB_PATH, { readonly: true });
|
||||
db.pragma("foreign_keys = ON");
|
||||
|
||||
expectedEntriesByLang = new Map();
|
||||
expectedTotalTranslations = 0;
|
||||
|
||||
for (const lang of SUPPORTED_LANGUAGE_CODES) {
|
||||
try {
|
||||
const raw = await fs.readFile(
|
||||
path.join(OUTPUT_DIR, `${lang}.json`),
|
||||
"utf-8",
|
||||
);
|
||||
const senses = JSON.parse(raw) as ExtractedSense[];
|
||||
expectedEntriesByLang.set(lang, senses.length);
|
||||
if (lang === "en") {
|
||||
for (const sense of senses) {
|
||||
expectedTotalTranslations += sense.translations.length;
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
expectedEntriesByLang.set(lang, 0);
|
||||
}
|
||||
}
|
||||
}, 30_000);
|
||||
|
||||
it("pipeline.db exists — skipping all tests if not", async () => {
|
||||
const exists = await dbExists();
|
||||
if (!exists) {
|
||||
console.warn(
|
||||
"\n pipeline.db not found — run pnpm db:init and pnpm db:import first\n",
|
||||
);
|
||||
}
|
||||
expect(exists).toBe(true);
|
||||
});
|
||||
|
||||
it("entry count per language matches source files", () => {
|
||||
if (!db) return;
|
||||
const errors: string[] = [];
|
||||
|
||||
for (const lang of SUPPORTED_LANGUAGE_CODES) {
|
||||
const expected = expectedEntriesByLang.get(lang) ?? 0;
|
||||
const row = db
|
||||
.prepare("SELECT COUNT(*) as count FROM entries WHERE language = ?")
|
||||
.get(lang) as { count: number };
|
||||
|
||||
if (row.count !== expected) {
|
||||
errors.push(`${lang}: expected ${expected} entries, got ${row.count}`);
|
||||
}
|
||||
}
|
||||
|
||||
expect(errors, `\n${errors.join("\n")}`).toHaveLength(0);
|
||||
});
|
||||
|
||||
it("translation count matches source files plus reverse links", () => {
|
||||
if (!db) return;
|
||||
const row = db
|
||||
.prepare("SELECT COUNT(*) as count FROM translations")
|
||||
.get() as { count: number };
|
||||
const reverseLinks = db
|
||||
.prepare(
|
||||
"SELECT COUNT(*) as count FROM translations WHERE source = 'reverse_link'",
|
||||
)
|
||||
.get() as { count: number };
|
||||
expect(row.count).toBe(expectedTotalTranslations + reverseLinks.count);
|
||||
});
|
||||
|
||||
it("every translation references a valid entry", () => {
|
||||
if (!db) return;
|
||||
const rows = db
|
||||
.prepare(
|
||||
`SELECT t.id, t.entry_id
|
||||
FROM translations t
|
||||
LEFT JOIN entries e ON e.id = t.entry_id
|
||||
WHERE e.id IS NULL`,
|
||||
)
|
||||
.all() as { id: number; entry_id: number }[];
|
||||
|
||||
const errors = rows.map(
|
||||
(r) => `translation ${r.id}: references missing entry ${r.entry_id}`,
|
||||
);
|
||||
expect(errors, `\n${errors.join("\n")}`).toHaveLength(0);
|
||||
});
|
||||
|
||||
it("every entry has a valid language code", () => {
|
||||
if (!db) return;
|
||||
const validLangs = SUPPORTED_LANGUAGE_CODES.map((l) => `'${l}'`).join(", ");
|
||||
const rows = db
|
||||
.prepare(
|
||||
`SELECT id, headword, language FROM entries
|
||||
WHERE language NOT IN (${validLangs})`,
|
||||
)
|
||||
.all() as { id: number; headword: string; language: string }[];
|
||||
|
||||
const errors = rows.map(
|
||||
(r) => `entry ${r.id} "${r.headword}": invalid language "${r.language}"`,
|
||||
);
|
||||
expect(errors, `\n${errors.join("\n")}`).toHaveLength(0);
|
||||
});
|
||||
|
||||
it("every entry has a valid pos", () => {
|
||||
if (!db) return;
|
||||
const rows = db
|
||||
.prepare(
|
||||
`SELECT id, headword, pos FROM entries
|
||||
WHERE pos NOT IN ('noun', 'verb', 'adjective', 'adverb')`,
|
||||
)
|
||||
.all() as { id: number; headword: string; pos: string }[];
|
||||
|
||||
const errors = rows.map(
|
||||
(r) => `entry ${r.id} "${r.headword}": invalid pos "${r.pos}"`,
|
||||
);
|
||||
expect(errors, `\n${errors.join("\n")}`).toHaveLength(0);
|
||||
});
|
||||
|
||||
it("sense_index is unique per headword, language, pos", () => {
|
||||
if (!db) return;
|
||||
const rows = db
|
||||
.prepare(
|
||||
`SELECT headword, language, pos, sense_index, COUNT(*) as c
|
||||
FROM entries
|
||||
GROUP BY headword, language, pos, sense_index
|
||||
HAVING c > 1`,
|
||||
)
|
||||
.all() as {
|
||||
headword: string;
|
||||
language: string;
|
||||
pos: string;
|
||||
sense_index: number;
|
||||
c: number;
|
||||
}[];
|
||||
|
||||
const errors = rows.map(
|
||||
(r) =>
|
||||
`"${r.headword}" (${r.language} ${r.pos}): duplicate sense_index ${r.sense_index} (${r.c} rows)`,
|
||||
);
|
||||
expect(errors, `\n${errors.join("\n")}`).toHaveLength(0);
|
||||
});
|
||||
|
||||
it("non-English entries have no Kaikki translations", () => {
|
||||
if (!db) return;
|
||||
const nonEnLangs = SUPPORTED_LANGUAGE_CODES.filter((l) => l !== "en")
|
||||
.map((l) => `'${l}'`)
|
||||
.join(", ");
|
||||
|
||||
const rows = db
|
||||
.prepare(
|
||||
`SELECT e.headword, e.language, COUNT(t.id) as c
|
||||
FROM entries e
|
||||
JOIN translations t ON t.entry_id = e.id
|
||||
WHERE e.language IN (${nonEnLangs})
|
||||
AND t.source = 'kaikki'
|
||||
GROUP BY e.id`,
|
||||
)
|
||||
.all() as { headword: string; language: string; c: number }[];
|
||||
|
||||
const errors = rows.map(
|
||||
(r) =>
|
||||
`"${r.headword}" (${r.language}): unexpected ${r.c} Kaikki translations`,
|
||||
);
|
||||
expect(errors, `\n${errors.join("\n")}`).toHaveLength(0);
|
||||
});
|
||||
|
||||
it("all Kaikki translation target languages are supported and not English", () => {
|
||||
if (!db) return;
|
||||
const validLangs = SUPPORTED_LANGUAGE_CODES.map((l) => `'${l}'`).join(", ");
|
||||
|
||||
const rows = db
|
||||
.prepare(
|
||||
`SELECT t.id, t.target_lang
|
||||
FROM translations t
|
||||
WHERE t.source = 'kaikki'
|
||||
AND (t.target_lang NOT IN (${validLangs}) OR t.target_lang = 'en')`,
|
||||
)
|
||||
.all() as { id: number; target_lang: string }[];
|
||||
|
||||
const errors = rows.map(
|
||||
(r) => `translation ${r.id}: invalid target_lang "${r.target_lang}"`,
|
||||
);
|
||||
expect(errors, `\n${errors.join("\n")}`).toHaveLength(0);
|
||||
});
|
||||
});
|
||||
|
|
@ -1,192 +0,0 @@
|
|||
import fs from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import { describe, it, expect, beforeAll } from "vitest";
|
||||
import { SUPPORTED_LANGUAGE_CODES, SUPPORTED_POS } from "@lila/shared";
|
||||
import type { SupportedLanguageCode, SupportedPos } from "@lila/shared";
|
||||
|
||||
// ── Types ─────────────────────────────────────────────────────────────────────
|
||||
|
||||
type ExtractedSense = {
|
||||
headword: string;
|
||||
language: SupportedLanguageCode;
|
||||
pos: SupportedPos;
|
||||
sense_index: number;
|
||||
gloss: string | null;
|
||||
examples: string[];
|
||||
translations: {
|
||||
target_lang: SupportedLanguageCode;
|
||||
word: string;
|
||||
sense_hint: string | null;
|
||||
}[];
|
||||
};
|
||||
|
||||
// ── Paths ─────────────────────────────────────────────────────────────────────
|
||||
|
||||
const OUTPUT_DIR = path.resolve("stage-1-extract/output");
|
||||
|
||||
// ── Tests ─────────────────────────────────────────────────────────────────────
|
||||
|
||||
describe("stage 1 — Kaikki extraction output validation", () => {
|
||||
const sensesByLang = new Map<SupportedLanguageCode, ExtractedSense[]>();
|
||||
|
||||
beforeAll(async () => {
|
||||
for (const lang of SUPPORTED_LANGUAGE_CODES) {
|
||||
const filePath = path.join(OUTPUT_DIR, `${lang}.json`);
|
||||
const raw = await fs.readFile(filePath, "utf-8");
|
||||
sensesByLang.set(lang, JSON.parse(raw) as ExtractedSense[]);
|
||||
}
|
||||
}, 30_000);
|
||||
|
||||
it("all five language output files exist", async () => {
|
||||
const errors: string[] = [];
|
||||
for (const lang of SUPPORTED_LANGUAGE_CODES) {
|
||||
try {
|
||||
await fs.access(path.join(OUTPUT_DIR, `${lang}.json`));
|
||||
} catch {
|
||||
errors.push(`missing: ${lang}.json`);
|
||||
}
|
||||
}
|
||||
expect(errors, `\n${errors.join("\n")}`).toHaveLength(0);
|
||||
});
|
||||
|
||||
it("every language file is a non-empty array", () => {
|
||||
const errors: string[] = [];
|
||||
for (const lang of SUPPORTED_LANGUAGE_CODES) {
|
||||
const senses = sensesByLang.get(lang)!;
|
||||
if (!Array.isArray(senses)) errors.push(`${lang}: not an array`);
|
||||
else if (senses.length === 0) errors.push(`${lang}: empty array`);
|
||||
}
|
||||
expect(errors, `\n${errors.join("\n")}`).toHaveLength(0);
|
||||
});
|
||||
|
||||
it("every sense has required fields", () => {
|
||||
const errors: string[] = [];
|
||||
for (const lang of SUPPORTED_LANGUAGE_CODES) {
|
||||
for (const sense of sensesByLang.get(lang)!) {
|
||||
if (!sense.headword) errors.push(`${lang}: sense missing headword`);
|
||||
if (!sense.language)
|
||||
errors.push(`${lang} ${sense.headword}: missing language`);
|
||||
if (!sense.pos) errors.push(`${lang} ${sense.headword}: missing pos`);
|
||||
if (sense.sense_index === undefined)
|
||||
errors.push(`${lang} ${sense.headword}: missing sense_index`);
|
||||
if (!Array.isArray(sense.examples))
|
||||
errors.push(`${lang} ${sense.headword}: examples not an array`);
|
||||
if (!Array.isArray(sense.translations))
|
||||
errors.push(`${lang} ${sense.headword}: translations not an array`);
|
||||
}
|
||||
}
|
||||
expect(errors, `\n${errors.join("\n")}`).toHaveLength(0);
|
||||
});
|
||||
|
||||
it("every sense has a valid pos", () => {
|
||||
const errors: string[] = [];
|
||||
const validPos = new Set(SUPPORTED_POS);
|
||||
for (const lang of SUPPORTED_LANGUAGE_CODES) {
|
||||
for (const sense of sensesByLang.get(lang)!) {
|
||||
if (!validPos.has(sense.pos)) {
|
||||
errors.push(`${lang} ${sense.headword}: invalid pos "${sense.pos}"`);
|
||||
}
|
||||
}
|
||||
}
|
||||
expect(errors, `\n${errors.join("\n")}`).toHaveLength(0);
|
||||
});
|
||||
|
||||
it("every sense language code matches its file", () => {
|
||||
const errors: string[] = [];
|
||||
for (const lang of SUPPORTED_LANGUAGE_CODES) {
|
||||
for (const sense of sensesByLang.get(lang)!) {
|
||||
if (sense.language !== lang) {
|
||||
errors.push(
|
||||
`${lang} ${sense.headword}: language field "${sense.language}" does not match file`,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
expect(errors, `\n${errors.join("\n")}`).toHaveLength(0);
|
||||
});
|
||||
|
||||
it("no abbreviation senses in output", () => {
|
||||
const errors: string[] = [];
|
||||
for (const lang of SUPPORTED_LANGUAGE_CODES) {
|
||||
for (const sense of sensesByLang.get(lang)!) {
|
||||
if (sense.gloss?.toLowerCase().startsWith("abbreviation of")) {
|
||||
errors.push(
|
||||
`${lang} ${sense.headword}: abbreviation sense not filtered`,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
expect(errors, `\n${errors.join("\n")}`).toHaveLength(0);
|
||||
});
|
||||
|
||||
it("English senses all have at least one translation", () => {
|
||||
const errors: string[] = [];
|
||||
for (const sense of sensesByLang.get("en")!) {
|
||||
if (sense.translations.length === 0) {
|
||||
errors.push(
|
||||
`en ${sense.headword} (sense ${sense.sense_index}): no translations`,
|
||||
);
|
||||
}
|
||||
}
|
||||
expect(errors, `\n${errors.join("\n")}`).toHaveLength(0);
|
||||
});
|
||||
|
||||
it("non-English senses have no translations", () => {
|
||||
const errors: string[] = [];
|
||||
for (const lang of SUPPORTED_LANGUAGE_CODES) {
|
||||
if (lang === "en") continue;
|
||||
for (const sense of sensesByLang.get(lang)!) {
|
||||
if (sense.translations.length > 0) {
|
||||
errors.push(
|
||||
`${lang} ${sense.headword}: unexpected translations in non-English file`,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
expect(errors, `\n${errors.join("\n")}`).toHaveLength(0);
|
||||
});
|
||||
|
||||
it("all translation target languages are supported and not English", () => {
|
||||
const errors: string[] = [];
|
||||
const validLangs = new Set(SUPPORTED_LANGUAGE_CODES);
|
||||
for (const sense of sensesByLang.get("en")!) {
|
||||
for (const t of sense.translations) {
|
||||
if (!validLangs.has(t.target_lang)) {
|
||||
errors.push(
|
||||
`en ${sense.headword}: unsupported translation language "${t.target_lang}"`,
|
||||
);
|
||||
}
|
||||
if (t.target_lang === "en") {
|
||||
errors.push(
|
||||
`en ${sense.headword}: translation to same language "en"`,
|
||||
);
|
||||
}
|
||||
if (!t.word?.trim()) {
|
||||
errors.push(
|
||||
`en ${sense.headword}: empty translation word for ${t.target_lang}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
expect(errors, `\n${errors.join("\n")}`).toHaveLength(0);
|
||||
});
|
||||
|
||||
it("sense_index is unique per headword and pos within each language", () => {
|
||||
const errors: string[] = [];
|
||||
for (const lang of SUPPORTED_LANGUAGE_CODES) {
|
||||
const seen = new Map<string, Set<number>>();
|
||||
for (const sense of sensesByLang.get(lang)!) {
|
||||
const key = `${sense.headword}|${sense.pos}`;
|
||||
if (!seen.has(key)) seen.set(key, new Set());
|
||||
const indexes = seen.get(key)!;
|
||||
if (indexes.has(sense.sense_index)) {
|
||||
errors.push(
|
||||
`${lang} ${sense.headword} (${sense.pos}): duplicate sense_index ${sense.sense_index}`,
|
||||
);
|
||||
}
|
||||
indexes.add(sense.sense_index);
|
||||
}
|
||||
}
|
||||
expect(errors, `\n${errors.join("\n")}`).toHaveLength(0);
|
||||
});
|
||||
});
|
||||
342
documentation/pipeline/ENGLISH_NOUNS.md
Normal file
342
documentation/pipeline/ENGLISH_NOUNS.md
Normal file
|
|
@ -0,0 +1,342 @@
|
|||
# english nouns
|
||||
|
||||
## step 1a
|
||||
|
||||
get freqency source list + verify lemmatization + confirm language/POS tagging
|
||||
|
||||
example:
|
||||
|
||||
house
|
||||
bank
|
||||
asdf
|
||||
|
||||
## step 1b
|
||||
|
||||
transform list into JSON (headword, language, POS)
|
||||
|
||||
example:
|
||||
|
||||
```json
|
||||
{
|
||||
"headword": "house",
|
||||
"language": "en",
|
||||
"pos": "noun"
|
||||
},
|
||||
{
|
||||
"headword": "bank",
|
||||
"language": "en",
|
||||
"pos": "noun"
|
||||
},
|
||||
{
|
||||
"headword": "asdf",
|
||||
"language": "en",
|
||||
"pos": "noun"
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## step 2a
|
||||
|
||||
check existence in kaikki (eg. is there a english noun "asdf" in kaikki, if not put it separate list)
|
||||
|
||||
example output:
|
||||
|
||||
```json
|
||||
{
|
||||
"headword": "house",
|
||||
"language": "en",
|
||||
"pos": "noun"
|
||||
},
|
||||
{
|
||||
"headword": "bank",
|
||||
"language": "en",
|
||||
"pos": "noun"
|
||||
}
|
||||
```
|
||||
|
||||
miss list (triage queue, not trash! contains junk, slipped inflections, and real Kaikki gaps, gets handled separately, never auto-drop!)
|
||||
|
||||
```json
|
||||
{ "headword": "asdf", "language": "en", "pos": "noun" }
|
||||
```
|
||||
|
||||
## step 2b
|
||||
|
||||
for each sense, extract: glosses, translations (with gender), examples
|
||||
id will be generated via: `headword:lang:pos:sense_index`
|
||||
|
||||
words without glosses will be dropped! (put in special list)
|
||||
|
||||
example output:
|
||||
|
||||
```json
|
||||
{
|
||||
"id": "house:en:noun:0",
|
||||
"headword": "house",
|
||||
"language": "en",
|
||||
"pos": "noun",
|
||||
"glosses": ["A building for human habitation."],
|
||||
"examples": ["They bought a house in the city."],
|
||||
"translations": {
|
||||
"de": [{ "word": "Haus", "gender": "neuter" }],
|
||||
"it": [{ "word": "casa", "gender": "feminine" }],
|
||||
"es": [{ "word": "casa", "gender": "feminine" }],
|
||||
"fr": [{ "word": "maison", "gender": "feminine" }]
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "house:en:noun:1",
|
||||
"headword": "house",
|
||||
"language": "en",
|
||||
"pos": "noun",
|
||||
"glosses": ["A noble family or lineage."],
|
||||
"examples": ["The House of Tudor ruled England."],
|
||||
"translations": {
|
||||
"de": [
|
||||
{ "word": "Adelsgeschlecht", "gender": "neuter" },
|
||||
{ "word": "Haus", "gender": "neuter" }
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "bank:en:noun:0",
|
||||
"headword": "bank",
|
||||
"language": "en",
|
||||
"pos": "noun",
|
||||
"glosses": ["An institution where one can place and borrow money."],
|
||||
"examples": ["She deposited her paycheck at the bank."],
|
||||
"translations": {
|
||||
"de": [{ "word": "Bank", "gender": "feminine" }],
|
||||
"it": [{ "word": "banca", "gender": "feminine" }],
|
||||
"es": [{ "word": "banco", "gender": "masculine" }],
|
||||
"fr": [{ "word": "banque", "gender": "feminine" }]
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "bank:en:noun:1",
|
||||
"headword": "bank",
|
||||
"language": "en",
|
||||
"pos": "noun",
|
||||
"glosses": ["The land alongside a river or lake."],
|
||||
"examples": ["They picnicked on the bank of the river."],
|
||||
"translations": {
|
||||
"de": [{ "word": "Ufer", "gender": "neuter" }],
|
||||
"it": [{ "word": "riva", "gender": "feminine" }],
|
||||
"es": [{ "word": "orilla", "gender": "feminine" }],
|
||||
"fr": [{ "word": "rive", "gender": "feminine" }]
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "bank:en:noun:2",
|
||||
"headword": "bank",
|
||||
"language": "en",
|
||||
"pos": "noun",
|
||||
"glosses": ["A collection or store of something held in reserve."],
|
||||
"examples": ["The hospital keeps a blood bank."],
|
||||
"translations": {
|
||||
"de": [{ "word": "Bank", "gender": "feminine" }]
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## step 2c
|
||||
|
||||
fill gaps Kaikki left (LLM, 3 models, generate-then-vote)
|
||||
|
||||
takes 2b's partial cards. 3 different-family models (qwen, llama, gemma) generate, then vote.
|
||||
|
||||
separate focused sub-passes, one field at a time — never combined:
|
||||
|
||||
- **translations** → per missing language, generated with the gloss as context. generate → vote. no agreement → gap stays (cloud audit later)
|
||||
- **examples** → for senses with no Kaikki example. generate → verify-vote ("is this a valid example of the gloss?"), since freeform sentences never exact-match. no agreement → no example (card still valid) no tiebreak: no agreement leaves the gap, never escalates to more models. vote records stored in pipeline.db for the cloud audit.
|
||||
- **invariant**: the gloss is never LLM-generated — translations are filled, examples generated, difficulty graded, but the gloss must always be Kaikki's (no gloss → sense dropped in 2b)
|
||||
|
||||
example input:
|
||||
|
||||
```json
|
||||
{
|
||||
"id": "harbor:en:noun:0",
|
||||
"headword": "harbor",
|
||||
"language": "en",
|
||||
"pos": "noun",
|
||||
"glosses": ["A sheltered area of water where ships can dock safely."],
|
||||
"examples": [],
|
||||
"translations": { "de": [{ "word": "Hafen", "gender": "masculine" }] }
|
||||
}
|
||||
```
|
||||
|
||||
next substage is the translation generation:
|
||||
|
||||
```json
|
||||
{
|
||||
"id": "harbor:en:noun:0",
|
||||
"headword": "harbor",
|
||||
"language": "en",
|
||||
"pos": "noun",
|
||||
"glosses": ["A sheltered area of water where ships can dock safely."],
|
||||
"examples": [],
|
||||
"translations": {
|
||||
"de": [{ "word": "Hafen", "gender": "masculine" }],
|
||||
"it": [{ "word": "porto", "gender": null }],
|
||||
"es": [{ "word": "puerto", "gender": null }],
|
||||
"fr": [{ "word": "port", "gender": null }]
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
next substage is the gender modification (by looking up kaikki for it/es/fr or corresponding wiktionary, fill only nulls, not touching existing genders):
|
||||
**runs only after all genders are present**
|
||||
|
||||
```json
|
||||
{
|
||||
"id": "harbor:en:noun:0",
|
||||
"headword": "harbor",
|
||||
"language": "en",
|
||||
"pos": "noun",
|
||||
"glosses": ["A sheltered area of water where ships can dock safely."],
|
||||
"examples": [],
|
||||
"translations": {
|
||||
"de": [{ "word": "Hafen", "gender": "masculine" }],
|
||||
"it": [{ "word": "porto", "gender": "masculine" }],
|
||||
"es": [{ "word": "puerto", "gender": "masculine" }],
|
||||
"fr": [{ "word": "port", "gender": "masculine" }]
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
next substage is the example generation:
|
||||
|
||||
```json
|
||||
{
|
||||
"id": "harbor:en:noun:0",
|
||||
"headword": "harbor",
|
||||
"language": "en",
|
||||
"pos": "noun",
|
||||
"glosses": ["A sheltered area of water where ships can dock safely."],
|
||||
"examples": ["The fishing boats returned to the harbor at dusk."],
|
||||
"translations": {
|
||||
"de": [{ "word": "Hafen", "gender": "masculine" }],
|
||||
"it": [{ "word": "porto", "gender": "masculine" }],
|
||||
"es": [{ "word": "puerto", "gender": "masculine" }],
|
||||
"fr": [{ "word": "port", "gender": "masculine" }]
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## step 3
|
||||
|
||||
adding difficulty level
|
||||
|
||||
CEFR is mapped to three buckets:
|
||||
A1/A2 → easy
|
||||
B1/B2 → intermediate
|
||||
C1/C2 → hard
|
||||
|
||||
depending on number of senses per headword:
|
||||
|
||||
- One sense → derive difficulty from the CEFRLex distribution (first CEFR level crossing a frequency threshold, mapped to a bucket, not the peak), for the four covered languages (en/de/es/fr). Deterministic, no LLM
|
||||
- Multiple senses → the three local LLMs grade each sense (easy/intermediate/hard) from the gloss. CEFRLex not involved.
|
||||
- No CEFRLex entry at all (Italian, or single-sense word that's missing) → LLMs grade from gloss
|
||||
|
||||
to get coverage (not quality), 3 local llms are going to be used: gemma, qwen and llama
|
||||
|
||||
on the llm votes:
|
||||
|
||||
- **Majority agrees (3-0 or 2-1)** → ship the majority value.
|
||||
- **3-way split (all three differ)** → no consensus → exclude the card to the review queue. Not shipped.
|
||||
|
||||
example output:
|
||||
|
||||
```json
|
||||
{
|
||||
"id": "house:en:noun:0",
|
||||
"headword": "house",
|
||||
"language": "en",
|
||||
"pos": "noun",
|
||||
"difficulty_level": "easy",
|
||||
"glosses": ["A building for human habitation."],
|
||||
"examples": ["They bought a house in the city."],
|
||||
"translations": {
|
||||
"de": [{ "word": "Haus", "gender": "neuter" }],
|
||||
"it": [{ "word": "casa", "gender": "feminine" }],
|
||||
"es": [{ "word": "casa", "gender": "feminine" }],
|
||||
"fr": [{ "word": "maison", "gender": "feminine" }]
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "house:en:noun:1",
|
||||
"headword": "house",
|
||||
"language": "en",
|
||||
"pos": "noun",
|
||||
"difficulty_level": "hard",
|
||||
"glosses": ["A noble family or lineage."],
|
||||
"examples": ["The House of Tudor ruled England."],
|
||||
"translations": {
|
||||
"de": [
|
||||
{ "word": "Adelsgeschlecht", "gender": "neuter" },
|
||||
{ "word": "Haus", "gender": "neuter" }
|
||||
]
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
**important**: `difficulty_level` is mandatory on every shipped card (the user sets difficulty before a game)
|
||||
|
||||
This is coverage, not final quality: three local models make every shipped card's difficulty present and plausible, but not guaranteed correct
|
||||
|
||||
no cefr list: needs to be graded by online llms (see notes)
|
||||
|
||||
---
|
||||
|
||||
## coverage report (trial deliverable)
|
||||
|
||||
- not a data stage => reads the finished cards and summarises them
|
||||
- the trial (english + italian nouns, local models) runs steps 1a→3 then emits a report
|
||||
|
||||
### pipeline health — did it run?
|
||||
|
||||
- input lemmas → output cards (ratio; multi-sense makes cards > lemmas)
|
||||
- dropped per stage + why: 2a misses (junk/inflection/gap), 2b no-gloss drops, 3 no-consensus exclusions
|
||||
- parse failures / errors
|
||||
|
||||
### coverage — how complete?
|
||||
|
||||
- translation completeness: all 4 langs vs gaps, per language
|
||||
- examples: from Kaikki vs LLM-generated vs none
|
||||
- gender: translations still null after Wiktionary fill
|
||||
- difficulty distribution per bucket (lopsided = broken CEFRLex threshold or bad grading)
|
||||
|
||||
### local good enough? — model agreement
|
||||
|
||||
- difficulty votes: 3-0 / 2-1 / 3-way split rates (high 3-way = locals can't do it → need cloud)
|
||||
- translation gap-fill: agreement rate, gaps left unfilled
|
||||
- → go/no-go on local-for-launch
|
||||
|
||||
### rent vs API? — workload volume
|
||||
|
||||
- total LLM calls across 2c + 3 (× per-token rate = API cost)
|
||||
- cards needing LLM vs handled deterministically (english: high deterministic; italian: ~0, no CEFRLex)
|
||||
- per-language call counts → scale english (low) and italian (high) to estimate the middle three
|
||||
|
||||
### review backlog — what's deferred
|
||||
|
||||
- miss-list size + composition, per language
|
||||
- no-consensus exclusions (cloud-audit queue size)
|
||||
- unfilled translation gaps (also cloud-audit work)
|
||||
|
||||
**framing**: english = optimistic floor (rich Kaikki, CEFRLex exists, models strongest).
|
||||
italian = pessimistic ceiling (no CEFRLex, thinner Kaikki). all five languages sit between.
|
||||
read to decide, not to archive.
|
||||
|
||||
---
|
||||
|
||||
Notes:
|
||||
|
||||
- the miss list needs to get verified/re-worked later on
|
||||
- the kaikki data contains ipas and links to audio files, add them later if needed, they are not needed now"
|
||||
- use online llms to set the difficulty(cefr) of the not shipped words
|
||||
- add plurals from kaikki/wiktionary
|
||||
- postgres sync to prod db
|
||||
41
documentation/pipeline/TRIAL_IMPLEMENTATION_ROADMAP.md
Normal file
41
documentation/pipeline/TRIAL_IMPLEMENTATION_ROADMAP.md
Normal file
|
|
@ -0,0 +1,41 @@
|
|||
# trial run — implementation roadmap (pure vertical)
|
||||
|
||||
goal: english + italian nouns through the full pipeline on local models, emit
|
||||
coverage report. build a thin end-to-end slice first, then widen each pass.
|
||||
|
||||
## phase 0 — foundation
|
||||
|
||||
- confirm @lila/shared types (lang codes, POS) exist and match
|
||||
- package.json deps for what phase 1 needs
|
||||
- one hardcoded test word to carry through the slice (e.g. "house")
|
||||
|
||||
## phase 1 — thin vertical slice (ONE word, 1a→3, crudest possible)
|
||||
|
||||
goal: prove a single card can travel the whole pipeline and come out the end.
|
||||
allowed to be ugly — hardcode, skip voting, one model, fake CEFRLex.
|
||||
|
||||
- 1a: one lemma record, by hand or trivial read
|
||||
- 2a: look it up in Kaikki, confirm it exists
|
||||
- 2b: extract its senses → card(s) with gloss
|
||||
- 2c: fill one missing translation with ONE local model, no voting
|
||||
- 3: assign a difficulty crudely (even hardcoded "easy")
|
||||
- OUT: one finished card. the pipeline has a shape.
|
||||
|
||||
## phase 2 — widen: real deterministic front (1a–2b, all words)
|
||||
|
||||
- 1a: real frequency list, verified
|
||||
- 2a: real existence gate + miss-list triage
|
||||
- 2b: real sense extraction, gender from tags, gloss-required drop
|
||||
- still JSON output. inspect cards by hand.
|
||||
|
||||
## phase 3 — widen: real LLM back (2c + 3)
|
||||
|
||||
- storage seam: JSON → SQLite, schema, resumability
|
||||
- 2c: real generate→vote, three families, all sub-passes
|
||||
- 3: real CEFRLex single-sense + 3-model vote multi-sense + exclude-on-split
|
||||
|
||||
## phase 4 — coverage report + runs
|
||||
|
||||
- emit report
|
||||
- run english, then italian
|
||||
- decide: local good enough? rent vs API?
|
||||
Loading…
Add table
Add a link
Reference in a new issue