From cc89f0c75cae20f931c984898ef50adb953cdc46 Mon Sep 17 00:00:00 2001 From: lila Date: Wed, 3 Jun 2026 00:02:19 +0200 Subject: [PATCH] wip --- .gitignore | 8 +- data-pipeline/audit.ts | 87 - data-pipeline/db/import.ts | 154 - data-pipeline/db/index.ts | 24 - data-pipeline/db/init.ts | 42 - data-pipeline/db/pipeline.db-shm | Bin 32768 -> 0 bytes data-pipeline/db/pipeline.db-wal | 0 data-pipeline/db/reset.ts | 41 - data-pipeline/db/schema.sql | 164 -- data-pipeline/pipeline.ts | 616 ---- data-pipeline/sample/output/sample.json | 2472 ----------------- data-pipeline/sample/scripts/sample.ts | 205 -- .../stage-1-extract/scripts/extract.ts | 257 -- .../scripts/reverse-link.ts | 109 - data-pipeline/stage-3-enrich/config.ts | 123 - .../stage-3-enrich/scripts/enrich.ts | 877 ------ .../validation/db-import.validation.test.ts | 230 -- .../validation/stage-1.validation.test.ts | 192 -- documentation/pipeline/ENGLISH_NOUNS.md | 342 +++ .../pipeline/TRIAL_IMPLEMENTATION_ROADMAP.md | 41 + 20 files changed, 384 insertions(+), 5600 deletions(-) delete mode 100644 data-pipeline/audit.ts delete mode 100644 data-pipeline/db/import.ts delete mode 100644 data-pipeline/db/index.ts delete mode 100644 data-pipeline/db/init.ts delete mode 100644 data-pipeline/db/pipeline.db-shm delete mode 100644 data-pipeline/db/pipeline.db-wal delete mode 100644 data-pipeline/db/reset.ts delete mode 100644 data-pipeline/db/schema.sql delete mode 100644 data-pipeline/pipeline.ts delete mode 100644 data-pipeline/sample/output/sample.json delete mode 100644 data-pipeline/sample/scripts/sample.ts delete mode 100644 data-pipeline/stage-1-extract/scripts/extract.ts delete mode 100644 data-pipeline/stage-2-reverse-link/scripts/reverse-link.ts delete mode 100644 data-pipeline/stage-3-enrich/config.ts delete mode 100644 data-pipeline/stage-3-enrich/scripts/enrich.ts delete mode 100644 data-pipeline/tests/validation/db-import.validation.test.ts delete mode 100644 data-pipeline/tests/validation/stage-1.validation.test.ts create mode 100644 documentation/pipeline/ENGLISH_NOUNS.md create mode 100644 documentation/pipeline/TRIAL_IMPLEMENTATION_ROADMAP.md diff --git a/.gitignore b/.gitignore index f8dbdb9..f533cdd 100644 --- a/.gitignore +++ b/.gitignore @@ -10,13 +10,7 @@ venv/ __pycache__/ *.pyc -data-pipeline/archive/ -data-pipeline/stage-1-extract/output/ -data-pipeline/stage-1-extract/sources/ -data-pipeline/stage-2-annotate/output/ -data-pipeline/stage-3-enrich/output/ -data-pipeline/stage-4-merge/output/ +data-pipeline/kaikki-source-files/ data-pipeline/db/pipeline.db -data-pipeline/reports/ data-pipeline/.env .aider* diff --git a/data-pipeline/audit.ts b/data-pipeline/audit.ts deleted file mode 100644 index fed3f3b..0000000 --- a/data-pipeline/audit.ts +++ /dev/null @@ -1,87 +0,0 @@ -import Database from "better-sqlite3"; -import path from "node:path"; -import fs from "node:fs"; -import { fileURLToPath } from "node:url"; - -const __dirname = path.dirname(fileURLToPath(import.meta.url)); -const DB_PATH = path.join(__dirname, "db/pipeline.db"); - -const db = new Database(DB_PATH, { readonly: true }); - -// Pull 50 synsets: ~12 per POS, all must have German translations -const synsets = db - .prepare( - ` - SELECT DISTINCT s.source_id, s.pos - FROM synsets s - JOIN translations t ON t.source_id = s.source_id - WHERE t.language = 'de' - ORDER BY RANDOM() - LIMIT 50 - `, - ) - .all() as { source_id: string; pos: string }[]; - -const results: string[] = []; -let index = 0; - -for (const synset of synsets) { - index++; - - const glosses = db - .prepare("SELECT language, text FROM glosses WHERE source_id = ?") - .all(synset.source_id) as { language: string; text: string }[]; - - const enGloss = glosses.find((g) => g.language === "en")?.text ?? "—"; - const deGloss = glosses.find((g) => g.language === "de")?.text ?? "—"; - - const deTranslations = db - .prepare( - "SELECT word FROM translations WHERE source_id = ? AND language = 'de'", - ) - .all(synset.source_id) as { word: string }[]; - - const enTranslations = db - .prepare( - "SELECT word FROM translations WHERE source_id = ? AND language = 'en'", - ) - .all(synset.source_id) as { word: string }[]; - - const deWords = deTranslations.map((t) => t.word); - const enWords = enTranslations.map((t) => t.word); - - results.push( - [ - `${String(index).padStart(2, " ")}. [${synset.pos}] ${synset.source_id}`, - ` EN gloss: ${enGloss}`, - ` DE gloss: ${deGloss}`, - ` EN words: ${enWords.join(", ")}`, - ` DE words: ${deWords.join(", ")}`, - ` QUALITY: ___`, - ``, - ].join("\n"), - ); -} - -const output = [ - "# OMW German Translation Quality Audit", - "", - "Instructions: for each entry, check if the German translations", - "match the meaning described by the English gloss.", - "", - "Mark QUALITY as:", - " OK — all German translations fit the meaning", - " PARTIAL — some fit, some don't", - " BAD — none of the German translations fit", - " USELESS — translations are correct but useless for learners", - "", - "---", - "", - ...results, -].join("\n"); - -const outPath = path.join(__dirname, "audit.md"); -fs.writeFileSync(outPath, output, "utf-8"); -console.log(`Wrote ${synsets.length} entries → ${outPath}`); - -db.close(); diff --git a/data-pipeline/db/import.ts b/data-pipeline/db/import.ts deleted file mode 100644 index 3733e81..0000000 --- a/data-pipeline/db/import.ts +++ /dev/null @@ -1,154 +0,0 @@ -import fs from "node:fs/promises"; -import path from "node:path"; -import { fileURLToPath } from "node:url"; -import { SUPPORTED_LANGUAGE_CODES } from "@lila/shared"; -import { openDb } from "./index.js"; -import type { ExtractedSense } from "../stage-1-extract/scripts/extract.js"; - -// ── Paths ───────────────────────────────────────────────────────────────────── - -const __dirname = path.dirname(fileURLToPath(import.meta.url)); - -const OUTPUT_DIR = path.resolve(__dirname, "../stage-1-extract/output"); - -// ── Import ──────────────────────────────────────────────────────────────────── - -export async function importKaikki(): Promise { - const db = openDb(); - - const insertEntry = db.prepare(` - INSERT INTO entries (headword, language, pos, sense_index, gloss, examples) - VALUES (?, ?, ?, ?, ?, ?) - ON CONFLICT (headword, language, pos, sense_index) - DO UPDATE SET - gloss = excluded.gloss, - examples = excluded.examples - RETURNING id - `); - - const insertTranslation = db.prepare(` - INSERT INTO translations (entry_id, target_lang, word, sense_hint) - VALUES (?, ?, ?, ?) - ON CONFLICT (entry_id, target_lang, word) DO NOTHING - `); - - let totalEntries = 0; - let totalTranslations = 0; - let totalSkipped = 0; - - for (const lang of SUPPORTED_LANGUAGE_CODES) { - const filePath = path.join(OUTPUT_DIR, `${lang}.json`); - - let senses: ExtractedSense[]; - try { - const raw = await fs.readFile(filePath, "utf-8"); - senses = JSON.parse(raw) as ExtractedSense[]; - } catch { - console.warn(` Warning: no output file found for ${lang}, skipping`); - continue; - } - - console.log( - ` Importing ${lang}: ${senses.length.toLocaleString()} senses...`, - ); - - // Track next available sense_index per (headword, pos) to handle - // the same word appearing in multiple JSONL entries with the same POS. - const senseIndexMap = new Map(); - - const importLang = db.transaction(() => { - let entries = 0; - let translations = 0; - let skipped = 0; - - for (const sense of senses) { - const key = `${sense.headword}|${sense.pos}`; - const nextIndex = senseIndexMap.get(key) ?? 0; - senseIndexMap.set(key, nextIndex + 1); - - const row = insertEntry.get( - sense.headword, - sense.language, - sense.pos, - nextIndex, - sense.gloss ?? null, - JSON.stringify(sense.examples), - ) as { id: number } | undefined; - - if (!row) { - skipped++; - continue; - } - - entries++; - - for (const t of sense.translations) { - insertTranslation.run( - row.id, - t.target_lang, - t.word, - t.sense_hint ?? null, - ); - translations++; - } - } - - return { entries, translations, skipped }; - }); - - const counts = importLang(); - totalEntries += counts.entries; - totalTranslations += counts.translations; - totalSkipped += counts.skipped; - - console.log( - ` entries: ${counts.entries.toLocaleString()}, translations: ${counts.translations.toLocaleString()}, skipped: ${counts.skipped.toLocaleString()}`, - ); - } - - db.close(); - - console.log(`\nImport complete:`); - console.log(` Total entries: ${totalEntries.toLocaleString()}`); - console.log(` Total translations: ${totalTranslations.toLocaleString()}`); - console.log(` Total skipped: ${totalSkipped.toLocaleString()}`); -} - -// ── Check if already imported ───────────────────────────────────────────────── - -export function isImported(): boolean { - const db = openDb(); - const row = db.prepare("SELECT COUNT(*) as count FROM entries").get() as { - count: number; - }; - db.close(); - return row.count > 0; -} - -// ── Main ───────────────────────────────────────────────────────────────────── - -async function main(): Promise { - const db = openDb(); - const row = db.prepare("SELECT COUNT(*) as count FROM entries").get() as { - count: number; - }; - db.close(); - - if (row.count > 0) { - console.log( - `pipeline.db already contains ${row.count.toLocaleString()} entries — skipping import.`, - ); - console.log("Delete pipeline.db and re-run db:init to start fresh."); - process.exit(0); - } - - console.log("Importing Kaikki data into pipeline.db..."); - await importKaikki(); -} - -if (import.meta.url === `file://${process.argv[1]}`) { - main().catch((err) => { - console.error(err); - process.exit(1); - }); -} diff --git a/data-pipeline/db/index.ts b/data-pipeline/db/index.ts deleted file mode 100644 index f0ce57d..0000000 --- a/data-pipeline/db/index.ts +++ /dev/null @@ -1,24 +0,0 @@ -import path from "node:path"; -import { fileURLToPath } from "node:url"; -import Database from "better-sqlite3"; - -// ── Paths ───────────────────────────────────────────────────────────────────── - -const __dirname = path.dirname(fileURLToPath(import.meta.url)); - -const DB_PATH = path.join(__dirname, "pipeline.db"); - -// ── Types ───────────────────────────────────────────────────────────────────── - -export type Db = InstanceType; - -// ── Open ────────────────────────────────────────────────────────────────────── - -export function openDb(): Db { - const db = new Database(DB_PATH); - - db.pragma("journal_mode = WAL"); - db.pragma("foreign_keys = ON"); - - return db; -} diff --git a/data-pipeline/db/init.ts b/data-pipeline/db/init.ts deleted file mode 100644 index 3ba0558..0000000 --- a/data-pipeline/db/init.ts +++ /dev/null @@ -1,42 +0,0 @@ -import fs from "node:fs/promises"; -import path from "node:path"; -import { fileURLToPath } from "node:url"; -import Database from "better-sqlite3"; - -// ── Paths ───────────────────────────────────────────────────────────────────── - -const __dirname = path.dirname(fileURLToPath(import.meta.url)); - -const PATHS = { - schema: path.join(__dirname, "schema.sql"), - db: path.join(__dirname, "pipeline.db"), -}; - -// ── Init ────────────────────────────────────────────────────────────────────── - -export async function initDb(): Promise { - const schema = await fs.readFile(PATHS.schema, "utf-8"); - const db = new Database(PATHS.db); - - db.pragma("journal_mode = WAL"); - db.pragma("foreign_keys = ON"); - db.exec(schema); - db.close(); - - console.log(` pipeline.db initialised → ${PATHS.db}`); -} - -// ── Main ───────────────────────────────────────────────────────────────────── - -async function main(): Promise { - console.log("Initialising pipeline.db..."); - await initDb(); -} - -// after -if (import.meta.url === `file://${process.argv[1]}`) { - main().catch((err) => { - console.error(err); - process.exit(1); - }); -} diff --git a/data-pipeline/db/pipeline.db-shm b/data-pipeline/db/pipeline.db-shm deleted file mode 100644 index fe9ac2845eca6fe6da8a63cd096d9cf9e24ece10..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 32768 zcmeIuAr62r3 { - try { - await fs.access(PATHS.extractedEn); - } catch { - console.error("\n ERROR: stage-1-extract/output/en.json not found."); - console.error(" Run the stage 1 extraction script first:"); - console.error(" pnpm extract\n"); - process.exit(1); - } -} - -async function checkAndInitDb(): Promise { - try { - await fs.access(PATHS.db); - } catch { - console.log(" pipeline.db not found — initialising..."); - await initDb(); - } -} - -async function checkAndImportDb(): Promise { - if (!isImported()) { - console.log(" Base tables empty — importing Kaikki data..."); - await importKaikki(); - } -} - -async function checkLlamaServer(): Promise { - try { - const res = await fetch(PATHS.llamaHealth); - return res.ok; - } catch { - return false; - } -} - -function isLocalProvider(provider: ProviderConfig): boolean { - return provider.apiKey === "none"; -} - -async function checkProviderReady(provider: ProviderConfig): Promise { - if (isLocalProvider(provider)) { - const running = await checkLlamaServer(); - if (!running) { - console.error("\n ERROR: llama.cpp server is not running."); - console.error(" Start the server before running the pipeline:"); - console.error( - " ./build/bin/llama-server --model models/.gguf \\", - ); - console.error(" --port 8080 --host 127.0.0.1"); - console.error(" See llm-setup.md for full instructions.\n"); - process.exit(1); - } - } else { - validateProviderKey(provider); - } -} - -// ── Run name generation ─────────────────────────────────────────────────────── - -async function generateRunName(): Promise { - await fs.mkdir(PATHS.reports, { recursive: true }); - - const date = new Date().toISOString().exi(0, 10); - const files = await fs.readdir(PATHS.reports); - const todaysRuns = files.filter( - (f) => f.startsWith(date) && f.endsWith(".json"), - ).length; - - return `${date}_run-${todaysRuns + 1}`; -} - -// ── Shutdown handler ────────────────────────────────────────────────────────── - -let shutdownRequested = false; - -function registerShutdownHandler(stats: RunStats): void { - const handler = (): void => { - if (shutdownRequested) return; - shutdownRequested = true; - stats.stoppedAt = new Date(); - console.log("\n\n Shutdown requested — finishing current record..."); - }; - - process.on("SIGINT", handler); - process.on("SIGTERM", handler); -} - -// ── Stage status helpers ────────────────────────────────────────────────────── - -function getSentinelStatus(stage: RunStage): StageStatus { - const db = openDb(); - const row = db - .prepare( - `SELECT status FROM run_status - WHERE entry_id = ? AND model_name = ? AND stage = ?`, - ) - .get(SENTINEL.entryId, SENTINEL.modelName, stage) as - | { status: string } - | undefined; - db.close(); - return row?.status === "complete" ? "complete" : "pending"; -} - -function markSentinelComplete(stage: RunStage): void { - const db = openDb(); - db.prepare( - `INSERT INTO run_status (entry_id, model_name, stage, status) - VALUES (?, ?, ?, 'complete') - ON CONFLICT (entry_id, model_name, stage) - DO UPDATE SET status = 'complete', updated_at = datetime('now')`, - ).run(SENTINEL.entryId, SENTINEL.modelName, stage); - db.close(); -} - -function getModelRound1Status(modelName: string): StageStatus { - const db = openDb(); - - const total = ( - db - .prepare("SELECT COUNT(*) as count FROM entries WHERE language = 'en'") - .get() as { count: number } - ).count; - - const complete = ( - db - .prepare( - `SELECT COUNT(*) as count FROM run_status - WHERE model_name = ? AND stage = 'round1_gloss' - AND status = 'complete'`, - ) - .get(modelName) as { count: number } - ).count; - - db.close(); - - if (complete === 0) return "pending"; - if (complete >= total) return "complete"; - return "in_progress"; -} - -function getModelRound2Status(modelName: string): StageStatus { - const db = openDb(); - - const total = ( - db - .prepare("SELECT COUNT(*) as count FROM entries WHERE language = 'en'") - .get() as { count: number } - ).count; - - const complete = ( - db - .prepare( - `SELECT COUNT(*) as count FROM run_status - WHERE model_name = ? AND stage = 'round2' AND status = 'complete'`, - ) - .get(modelName) as { count: number } - ).count; - - db.close(); - - if (complete === 0) return "pending"; - if (complete >= total) return "complete"; - return "in_progress"; -} - -function isReverseLinkDone(): boolean { - const db = openDb(); - const row = db - .prepare( - `SELECT status FROM run_status - WHERE entry_id = ? AND model_name = ? AND stage = 'reverse_link'`, - ) - .get(SENTINEL.entryId, SENTINEL.modelName) as - | { status: string } - | undefined; - db.close(); - return row?.status === "complete"; -} - -function markReverseLinkComplete(): void { - const db = openDb(); - db.prepare( - `INSERT INTO run_status (entry_id, model_name, stage, status) - VALUES (?, ?, 'reverse_link', 'complete') - ON CONFLICT (entry_id, model_name, stage) - DO UPDATE SET status = 'complete', updated_at = datetime('now')`, - ).run(SENTINEL.entryId, SENTINEL.modelName); - db.close(); -} - -// ── Stage runners ───────────────────────────────────────────────────────────── - -function runReverseLinkStage(): void { - if (isReverseLinkDone()) { - console.log("\n [reverse link] Already complete, skipping"); - return; - } - console.log("\n [reverse link] Syncing reverse translation links..."); - reverseLink(); - markReverseLinkComplete(); -} - -async function runRound1( - provider: ProviderConfig, - stats: RunStats, -): Promise { - console.log(`\n [round 1] Running ${provider.name}...`); - const counts = await enrich(provider); - stats.recordsProcessed += counts.processed; - stats.recordsSkipped += counts.skipped; - stats.needsReview += counts.needsReview; - stats.modelsRun.push(provider.name); -} - -function compileCandidates(): void { - console.log("\n [compile candidates] Compiling round 1 output..."); - // TODO: implement compile candidates script - console.log(" [compile candidates] not yet implemented"); - markSentinelComplete("compile_candidates"); -} - -function runRound2(provider: ProviderConfig, stats: RunStats): void { - console.log(`\n [round 2] Running ${provider.name}...`); - // TODO: implement round 2 enrich script - console.log(` [round 2] ${provider.name} — not yet implemented`); - stats.modelsRun.push(provider.name); -} - -function compileVotes(): void { - console.log("\n [compile votes] Compiling round 2 votes..."); - // TODO: implement compile votes script - console.log(" [compile votes] not yet implemented"); - markSentinelComplete("compile_votes"); -} - -function runMerge(): void { - console.log("\n [merge] Resolving votes..."); - // TODO: implement merge script - console.log(" [merge] not yet implemented"); - markSentinelComplete("merge"); -} - -function runTiebreak(stats: RunStats): void { - console.log("\n [tiebreak] Resolving flagged entries..."); - // TODO: implement tiebreak logic - console.log(" [tiebreak] not yet implemented"); - stats.currentStage = "tiebreak"; -} - -function runCompare(): void { - console.log("\n [compare] Generating COVERAGE.md..."); - // TODO: implement compare script - console.log(" [compare] not yet implemented"); - markSentinelComplete("compare"); -} - -// ── Report generation ───────────────────────────────────────────────────────── - -async function generateReport(runName: string, stats: RunStats): Promise { - const db = openDb(); - - const totalEntries = ( - db.prepare("SELECT COUNT(*) as count FROM entries").get() as { - count: number; - } - ).count; - - const resolvedEntries = ( - db.prepare("SELECT COUNT(*) as count FROM resolved_entry_cefr").get() as { - count: number; - } - ).count; - - const flaggedEntries = ( - db - .prepare( - `SELECT COUNT(*) as count FROM run_status - WHERE stage = 'merge' AND status = 'flagged'`, - ) - .get() as { count: number } - ).count; - - const needsReview = ( - db - .prepare( - `SELECT COUNT(*) as count FROM run_status - WHERE status = 'needs_review'`, - ) - .get() as { count: number } - ).count; - - db.close(); - - const stoppedAt = stats.stoppedAt ?? new Date(); - const durationMs = stoppedAt.getTime() - stats.startedAt.getTime(); - const durationMin = Math.round(durationMs / 60_000); - - const isFinal = - getSentinelStatus("compare") === "complete" && flaggedEntries === 0; - - const report = { - runName, - generatedAt: stoppedAt.toISOString(), - durationMinutes: durationMin, - isFinal, - progress: { - totalEntries, - resolvedEntries, - flaggedEntries, - needsReview, - recordsProcessedThisRun: stats.recordsProcessed, - recordsSkippedThisRun: stats.recordsSkipped, - }, - modelsRun: stats.modelsRun, - stages: { - reverseLink: isReverseLinkDone() ? "complete" : "pending", - round1: ALL_PROVIDERS.map((p) => ({ - model: p.name, - status: getModelRound1Status(p.name), - })), - compileCandidates: getSentinelStatus("compile_candidates"), - round2: ALL_PROVIDERS.map((p) => ({ - model: p.name, - status: getModelRound2Status(p.name), - })), - compileVotes: getSentinelStatus("compile_votes"), - merge: getSentinelStatus("merge"), - compare: getSentinelStatus("compare"), - }, - }; - - await fs.mkdir(PATHS.reports, { recursive: true }); - - const jsonPath = path.join(PATHS.reports, `${runName}.json`); - const mdPath = path.join(PATHS.reports, `${runName}.md`); - - await fs.writeFile(jsonPath, JSON.stringify(report, null, 2), "utf-8"); - - const md = [ - `# Pipeline run: ${runName}`, - ``, - `Generated: ${stoppedAt.toISOString()}`, - `Duration: ${durationMin} minutes`, - isFinal - ? `**Status: FINAL — pipeline complete**` - : `**Status: In progress**`, - ``, - `## Progress`, - ``, - `| Metric | Value |`, - `| ------ | ----- |`, - `| Total entries | ${totalEntries.toLocaleString()} |`, - `| Resolved entries | ${resolvedEntries.toLocaleString()} |`, - `| Flagged entries | ${flaggedEntries.toLocaleString()} |`, - `| Needs review | ${needsReview.toLocaleString()} |`, - `| Records processed this run | ${stats.recordsProcessed.toLocaleString()} |`, - `| Records skipped this run | ${stats.recordsSkipped.toLocaleString()} |`, - ``, - `## Stage status`, - ``, - `### Reverse link: ${report.stages.reverseLink}`, - ``, - `### Round 1`, - ``, - ...report.stages.round1.map( - (s) => - `- ${s.status === "complete" ? "✅" : s.status === "in_progress" ? "🔄" : "🔲"} ${s.model}`, - ), - ``, - `### Compile candidates: ${report.stages.compileCandidates}`, - ``, - `### Round 2`, - ``, - ...report.stages.round2.map( - (s) => - `- ${s.status === "complete" ? "✅" : s.status === "in_progress" ? "🔄" : "🔲"} ${s.model}`, - ), - ``, - `### Compile votes: ${report.stages.compileVotes}`, - `### Merge: ${report.stages.merge}`, - `### Compare: ${report.stages.compare}`, - ``, - `## Models run this session`, - ``, - stats.modelsRun.length > 0 - ? stats.modelsRun.map((m) => `- ${m}`).join("\n") - : "_none_", - ].join("\n"); - - await fs.writeFile(mdPath, md, "utf-8"); - - console.log(`\n Report written → ${jsonPath}`); - console.log(` Report written → ${mdPath}`); -} - -// ── Main ────────────────────────────────────────────────────────────────────── - -async function main(): Promise { - console.log("lila data pipeline\n"); - - // ── Startup checks - console.log("Checking prerequisites..."); - await checkExtractedFilesExist(); - await checkAndInitDb(); - await checkAndImportDb(); - console.log(" Prerequisites OK"); - - // ── Run name - const runName = await generateRunName(); - console.log(`\n Run: ${runName}`); - - // ── Stats - const stats: RunStats = { - startedAt: new Date(), - stoppedAt: null, - recordsProcessed: 0, - recordsSkipped: 0, - needsReview: 0, - modelsRun: [], - currentStage: null, - }; - - registerShutdownHandler(stats); - - // ── Stage 2 — Reverse link - runReverseLinkStage(); - - if (shutdownRequested) { - await generateReport(runName, stats); - process.exit(0); - } - - // ── Round 1 - console.log("\nRound 1 — generation"); - for (const provider of ALL_PROVIDERS) { - if (shutdownRequested) break; - - const status = getModelRound1Status(provider.name); - - if (status === "complete") { - console.log(` [round 1] ${provider.name} — already complete, skipping`); - continue; - } - - await checkProviderReady(provider); - stats.currentStage = "round1"; - - if (status === "in_progress") { - console.log(` [round 1] ${provider.name} — resuming...`); - } - - await runRound1(provider, stats); - } - - if (shutdownRequested) { - await generateReport(runName, stats); - process.exit(0); - } - - // ── Compile candidates - if (getSentinelStatus("compile_candidates") === "complete") { - console.log("\n [compile candidates] Already complete, skipping"); - } else { - stats.currentStage = "compile_candidates"; - compileCandidates(); - } - - if (shutdownRequested) { - await generateReport(runName, stats); - process.exit(0); - } - - // ── Round 2 - console.log("\nRound 2 — voting"); - for (const provider of ALL_PROVIDERS) { - if (shutdownRequested) break; - - const status = getModelRound2Status(provider.name); - - if (status === "complete") { - console.log(` [round 2] ${provider.name} — already complete, skipping`); - continue; - } - - await checkProviderReady(provider); - stats.currentStage = "round2"; - - if (status === "in_progress") { - console.log(` [round 2] ${provider.name} — resuming...`); - } - - runRound2(provider, stats); - } - - if (shutdownRequested) { - await generateReport(runName, stats); - process.exit(0); - } - - // ── Compile votes - if (getSentinelStatus("compile_votes") === "complete") { - console.log("\n [compile votes] Already complete, skipping"); - } else { - stats.currentStage = "compile_votes"; - compileVotes(); - } - - if (shutdownRequested) { - await generateReport(runName, stats); - process.exit(0); - } - - // ── Merge - if (getSentinelStatus("merge") === "complete") { - console.log("\n [merge] Already complete, skipping"); - } else { - stats.currentStage = "merge"; - runMerge(); - } - - if (shutdownRequested) { - await generateReport(runName, stats); - process.exit(0); - } - - // ── Tiebreak - const db = openDb(); - const flagged = ( - db - .prepare( - `SELECT COUNT(*) as count FROM run_status - WHERE stage = 'merge' AND status = 'flagged'`, - ) - .get() as { count: number } - ).count; - db.close(); - - if (flagged > 0) { - stats.currentStage = "tiebreak"; - runTiebreak(stats); - } - - if (shutdownRequested) { - await generateReport(runName, stats); - process.exit(0); - } - - // ── Compare - if (getSentinelStatus("compare") === "complete") { - console.log("\n [compare] Already complete, skipping"); - } else { - stats.currentStage = "compare"; - runCompare(); - } - - // ── Report (disabled until full pipeline is implemented) - // stats.stoppedAt = new Date(); - // await generateReport(runName, stats); - - console.log("\nPipeline complete."); -} - -main().catch((err) => { - console.error(err); - process.exit(1); -}); diff --git a/data-pipeline/sample/output/sample.json b/data-pipeline/sample/output/sample.json deleted file mode 100644 index 3177e22..0000000 --- a/data-pipeline/sample/output/sample.json +++ /dev/null @@ -1,2472 +0,0 @@ -[ - { - "source_id": "ili:i90862", - "pos": "noun", - "translations": { - "en": ["kinsman"], - "es": ["pariente"], - "de": [ - "Gevatter", - "Anverwandter", - "Familienmitglied", - "Verwandter", - "Familienangehöriger", - "Angehöriger", - "Verwandte" - ], - "fr": ["parent"] - }, - "glosses": { - "en": ["a male relative"], - "de": ["ein männlicher Verwandter"] - }, - "examples": { - "de": [ - { - "text": "Jedes Familienmitglied hat seine Aufgaben.", - "source": "cefr" - }, - { - "text": "Er ist ein entfernter Verwandter von mir.", - "source": "cefr" - }, - { - "text": "Alle Familienangehörigen kamen zum Treffen.", - "source": "cefr" - }, - { "text": "Er ist ein Angehöriger der Familie.", "source": "cefr" } - ], - "fr": [ - { "text": "Ses parents sont très fiers de lui.", "source": "cefr" } - ], - "es": [ - { - "text": "Tengo muchos parientes viviendo en esta ciudad.", - "source": "cefr" - } - ] - }, - "votes": { - "en": { "kinsman": { "cefr_source": "C1" } }, - "de": { - "Familienmitglied": { "cefr_source": "A2" }, - "Verwandter": { "cefr_source": "B1" }, - "Familienangehöriger": { "cefr_source": "B1" }, - "Angehöriger": { "cefr_source": "B2" } - }, - "fr": { "parent": { "cefr_source": "A1" } }, - "es": { "pariente": { "cefr_source": "A2" } } - }, - "_sample_bucket": "has_cefr_vote" - }, - { - "source_id": "ili:i23087", - "pos": "verb", - "translations": { - "en": ["teach"], - "it": ["addestrare", "ammaestrare", "insegnare"], - "es": ["enseñar"], - "fr": ["enseigner", "apprendre", "guider"] - }, - "glosses": { "en": ["accustom gradually to some action or attitude"] }, - "examples": { - "en": [ - { "text": "The child is taught to obey her parents", "source": "omw" } - ], - "it": [ - { "text": "Stiamo addestrando il nostro cane.", "source": "cefr" }, - { "text": "Lei insegna italiano ai bambini.", "source": "cefr" } - ], - "fr": [ - { "text": "Elle enseigne le français au lycée.", "source": "cefr" }, - { "text": "J'apprends le français.", "source": "cefr" }, - { "text": "Il va nous guider à travers la forêt.", "source": "cefr" } - ], - "es": [ - { "text": "Ella enseña español en la universidad.", "source": "cefr" } - ] - }, - "votes": { - "en": { "teach": { "cefr_source": "A1" } }, - "it": { - "addestrare": { "cefr_source": "B1" }, - "insegnare": { "cefr_source": "A1" } - }, - "fr": { - "enseigner": { "cefr_source": "A2" }, - "apprendre": { "cefr_source": "A1" }, - "guider": { "cefr_source": "A2" } - }, - "es": { "enseñar": { "cefr_source": "A1" } } - }, - "_sample_bucket": "has_cefr_vote" - }, - { - "source_id": "ili:i26718", - "pos": "verb", - "translations": { - "en": ["dub", "nickname"], - "it": ["battezzare", "cognominare", "doppiare", "soprannominare"], - "es": ["apodar"], - "fr": ["surnom", "baptiser"] - }, - "glosses": { "en": ["give a nickname to"] }, - "examples": { - "it": [ - { - "text": "Hanno deciso di battezzare il loro figlio la prossima primavera.", - "source": "cefr" - }, - { "text": "Lo hanno soprannominato 'il Professore'.", "source": "cefr" } - ], - "fr": [ - { - "text": "Ils ont décidé de baptiser leur enfant Marie.", - "source": "cefr" - } - ] - }, - "votes": { - "en": { "dub": { "cefr_source": "B2" } }, - "it": { - "battezzare": { "cefr_source": "B1" }, - "soprannominare": { "cefr_source": "B2" } - }, - "fr": { "baptiser": { "cefr_source": "B1" } } - }, - "_sample_bucket": "has_cefr_vote" - }, - { - "source_id": "ili:i4448", - "pos": "adjective", - "translations": { - "en": ["drab", "dreary"], - "es": ["igual", "rutinario"], - "fr": ["morne", "maussade", "sombre"] - }, - "glosses": { "en": ["lacking in liveliness or charm or surprise"] }, - "examples": { - "en": [ - { "text": "her drab personality", "source": "omw" }, - { - "text": "life was drab compared with the more exciting life style overseas", - "source": "omw" - }, - { "text": "a series of dreary dinner parties", "source": "omw" } - ], - "fr": [ - { "text": "Le temps était morne et pluvieux.", "source": "cefr" }, - { - "text": "Le temps était maussade toute la journée.", - "source": "cefr" - }, - { "text": "La pièce était sombre sans lumière.", "source": "cefr" } - ], - "es": [ - { "text": "Todos somos iguales.", "source": "cefr" }, - { "text": "Su trabajo se ha vuelto muy rutinario.", "source": "cefr" } - ] - }, - "votes": { - "en": { - "drab": { "cefr_source": "B2" }, - "dreary": { "cefr_source": "B2" } - }, - "fr": { - "morne": { "cefr_source": "B2" }, - "maussade": { "cefr_source": "B2" }, - "sombre": { "cefr_source": "B1" } - }, - "es": { - "igual": { "cefr_source": "A2" }, - "rutinario": { "cefr_source": "B1" } - } - }, - "_sample_bucket": "has_cefr_vote" - }, - { - "source_id": "ili:i85845", - "pos": "noun", - "translations": { - "en": ["natural depression", "depression"], - "it": ["avvallamento"], - "es": ["depresión", "depresión natural"], - "fr": ["dépression"] - }, - "glosses": { "en": ["a sunken or depressed geological formation"] }, - "examples": { - "fr": [{ "text": "Elle souffre de dépression.", "source": "cefr" }], - "es": [ - { "text": "La depresión es una enfermedad grave.", "source": "cefr" } - ] - }, - "votes": { - "en": { "depression": { "cefr_source": "B2" } }, - "fr": { "dépression": { "cefr_source": "B2" } }, - "es": { "depresión": { "cefr_source": "B1" } } - }, - "_sample_bucket": "has_cefr_vote" - }, - { - "source_id": "ili:i27202", - "pos": "verb", - "translations": { "en": ["jump"], "fr": ["sauter"] }, - "glosses": { "en": ["make a sudden physical attack on"] }, - "examples": { - "en": [ - { - "text": "The muggers jumped the woman in the fur coat", - "source": "omw" - } - ], - "fr": [ - { - "text": "Le chien aime sauter par-dessus la clôture.", - "source": "cefr" - } - ] - }, - "votes": { - "en": { "jump": { "cefr_source": "A1" } }, - "fr": { "sauter": { "cefr_source": "A2" } } - }, - "_sample_bucket": "has_cefr_vote" - }, - { - "source_id": "ili:i27830", - "pos": "verb", - "translations": { - "en": [ - "run into", - "bump into", - "jar against", - "butt against", - "knock against" - ], - "it": ["urtare"], - "es": ["chocar", "colisionar", "golpearse contra", "topar"], - "de": [ - "anraunzen", - "anfahren", - "anschnauzen", - "ankläffen", - "anschreien", - "anblaffen", - "anblaffen", - "anbelfern", - "anbrüllen", - "anbellen" - ] - }, - "glosses": { - "en": ["collide violently with an obstacle"], - "de": ["heftig mit einem Hindernis zusammenstoßen"] - }, - "examples": { - "en": [{ "text": "I ran into the telephone pole", "source": "omw" }], - "it": [ - { "text": "Ho urtato il tavolo con il gomito.", "source": "cefr" } - ], - "de": [ - { "text": "Der Bus fuhr an die Haltestelle an.", "source": "cefr" }, - { "text": "Er hat mich ohne Grund angeschrien.", "source": "cefr" } - ], - "es": [ - { "text": "El coche chocó contra un árbol.", "source": "cefr" }, - { "text": "Me topé con un viejo amigo en la calle.", "source": "cefr" } - ] - }, - "votes": { - "it": { "urtare": { "cefr_source": "B1" } }, - "de": { - "anfahren": { "cefr_source": "B1" }, - "anschreien": { "cefr_source": "B1" } - }, - "es": { - "chocar": { "cefr_source": "A2" }, - "topar": { "cefr_source": "B1" } - } - }, - "_sample_bucket": "has_cefr_vote" - }, - { - "source_id": "ili:i27676", - "pos": "verb", - "translations": { "en": ["fumble"] }, - "glosses": { "en": ["handle clumsily"] }, - "examples": {}, - "votes": { "en": { "fumble": { "cefr_source": "B2" } } }, - "_sample_bucket": "has_cefr_vote" - }, - { - "source_id": "ili:i30768", - "pos": "verb", - "translations": { - "en": ["attract", "appeal"], - "it": ["allettare", "attirare", "attrarre"], - "es": ["atraer"], - "de": [ - "anziehen", - "etwas überziehen", - "einkleiden", - "etwas überstreifen", - "bekleiden", - "hineinschlüpfen", - "überstülpen", - "ankleiden", - "Kleidung anlegen" - ], - "fr": ["allécher", "attirer"] - }, - "glosses": { - "en": ["be attractive to"], - "de": [ - "ein Kleidungsstück in der dafür vorgesehenen Weise auf den Körper bringen" - ] - }, - "examples": { - "en": [ - { "text": "The idea of a vacation appeals to me", "source": "omw" }, - { - "text": "The beautiful garden attracted many people", - "source": "omw" - } - ], - "de": [{ "text": "Sie zog sich das Kleid an.", "source": "omw" }], - "it": [ - { "text": "Il nuovo negozio attira molti clienti.", "source": "cefr" }, - { "text": "Il magnete attrae il metallo.", "source": "cefr" } - ], - "fr": [ - { - "text": "La promesse d'un salaire élevé a alléché de nombreux candidats.", - "source": "cefr" - }, - { "text": "Cette publicité attire l'attention.", "source": "cefr" } - ], - "es": [{ "text": "El imán atrae el metal.", "source": "cefr" }] - }, - "votes": { - "en": { "attract": { "cefr_source": "B1" } }, - "it": { - "attirare": { "cefr_source": "B2" }, - "attrarre": { "cefr_source": "B1" } - }, - "de": { - "anziehen": { "cefr_source": "A2" }, - "bekleiden": { "cefr_source": "B2" } - }, - "fr": { - "allécher": { "cefr_source": "C1" }, - "attirer": { "cefr_source": "B1" } - }, - "es": { "atraer": { "cefr_source": "B2" } } - }, - "_sample_bucket": "has_cefr_vote" - }, - { - "source_id": "ili:i112909", - "pos": "noun", - "translations": { - "en": ["regulation"], - "es": ["reglamento"], - "fr": ["réglementation", "gouvernement", "tenue"] - }, - "glosses": { "en": ["the state of being controlled or governed"] }, - "examples": { - "fr": [ - { - "text": "La nouvelle réglementation est très stricte.", - "source": "cefr" - }, - { - "text": "Le gouvernement a annoncé de nouvelles mesures.", - "source": "cefr" - }, - { - "text": "Elle a choisi une tenue élégante pour la soirée.", - "source": "cefr" - } - ], - "es": [{ "text": "Debemos seguir el reglamento.", "source": "cefr" }] - }, - "votes": { - "en": { "regulation": { "cefr_source": "B2" } }, - "fr": { - "réglementation": { "cefr_source": "B2" }, - "gouvernement": { "cefr_source": "B1" }, - "tenue": { "cefr_source": "B1" } - }, - "es": { "reglamento": { "cefr_source": "B2" } } - }, - "_sample_bucket": "has_cefr_vote" - }, - { - "source_id": "ili:i46846", - "pos": "noun", - "translations": { - "en": [ - "ladybug", - "ladybeetle", - "lady beetle", - "ladybird", - "ladybird beetle" - ], - "it": ["coccinella"], - "fr": ["coccinelle"] - }, - "glosses": { - "en": [ - "small round bright-colored and spotted beetle that usually feeds on aphids and other insect pests" - ] - }, - "examples": { - "fr": [ - { "text": "Une coccinelle s'est posée sur ma main.", "source": "cefr" } - ] - }, - "votes": { - "en": { "ladybug": { "cefr_source": "A2" } }, - "fr": { "coccinelle": { "cefr_source": "A2" } } - }, - "_sample_bucket": "has_cefr_vote" - }, - { - "source_id": "ili:i15517", - "pos": "adjective", - "translations": { - "en": ["judicial"], - "it": ["giudiziale", "giudiziario"], - "es": ["judicial"], - "de": [ - "durch einen Richter", - "durch ein Gericht", - "durch den Richter", - "richterlich" - ], - "fr": ["judiciaire"] - }, - "glosses": { - "en": ["belonging or appropriate to the office of a judge"], - "de": ["zum Amt eines Richters gehörend oder diesem zugehörig"] - }, - "examples": { - "en": [{ "text": "judicial robes", "source": "omw" }], - "it": [ - { "text": "Hanno avviato un'azione giudiziale.", "source": "cefr" }, - { - "text": "Il sistema giudiziario italiano è complesso.", - "source": "cefr" - } - ], - "de": [ - { "text": "Es bedarf einer richterlichen Anordnung.", "source": "cefr" } - ], - "fr": [ - { - "text": "L'affaire est en cours de procédure judiciaire.", - "source": "cefr" - } - ], - "es": [{ "text": "El proceso judicial fue largo.", "source": "cefr" }] - }, - "votes": { - "en": { "judicial": { "cefr_source": "C1" } }, - "it": { - "giudiziale": { "cefr_source": "C1" }, - "giudiziario": { "cefr_source": "C1" } - }, - "de": { "richterlich": { "cefr_source": "C1" } }, - "fr": { "judiciaire": { "cefr_source": "B2" } }, - "es": { "judicial": { "cefr_source": "C1" } } - }, - "_sample_bucket": "has_cefr_vote" - }, - { - "source_id": "ili:i11095", - "pos": "adjective", - "translations": { "en": ["poor"], "es": ["pobre"], "fr": ["pauvre"] }, - "glosses": { "en": ["characterized by or indicating poverty"] }, - "examples": { - "en": [ - { "text": "the country had a poor economy", "source": "omw" }, - { "text": "they lived in the poor section of town", "source": "omw" } - ], - "fr": [{ "text": "Il est très pauvre.", "source": "cefr" }], - "es": [{ "text": "Es un hombre muy pobre.", "source": "cefr" }] - }, - "votes": { - "en": { "poor": { "cefr_source": "A2" } }, - "fr": { "pauvre": { "cefr_source": "A1" } }, - "es": { "pobre": { "cefr_source": "A1" } } - }, - "_sample_bucket": "has_cefr_vote" - }, - { - "source_id": "ili:i62321", - "pos": "noun", - "translations": { - "en": [ - "flashiness", - "garishness", - "gaudiness", - "loudness", - "brashness", - "meretriciousness", - "tawdriness", - "glitz" - ], - "it": ["pacchianeria", "vistosità"], - "es": [ - "astracanada", - "chabacanería", - "garrulería", - "horterada", - "mal gusto", - "ordinariez", - "zafiedad" - ], - "de": ["Aufdringlichkeit", "Zudringlichkeit", "Penetranz"], - "fr": ["culot"] - }, - "glosses": { - "en": ["tasteless showiness"], - "de": ["geschmacklose Aufdringlichkeit"] - }, - "examples": { - "fr": [ - { - "text": "Il a eu le culot de me demander de l'argent après tout ça.", - "source": "cefr" - } - ] - }, - "votes": { - "en": { - "loudness": { "cefr_source": "B2" }, - "glitz": { "cefr_source": "B2" } - }, - "fr": { "culot": { "cefr_source": "B2" } } - }, - "_sample_bucket": "has_cefr_vote" - }, - { - "source_id": "ili:i22613", - "pos": "verb", - "translations": { - "en": ["scavenge", "clean"], - "es": ["limpiar"], - "fr": ["nettoyer"] - }, - "glosses": { "en": ["remove unwanted substances from"] }, - "examples": { - "fr": [{ "text": "Je dois nettoyer ma chambre.", "source": "cefr" }], - "es": [{ "text": "Necesito limpiar mi habitación.", "source": "cefr" }] - }, - "votes": { - "en": { "scavenge": { "cefr_source": "B2" } }, - "fr": { "nettoyer": { "cefr_source": "A1" } }, - "es": { "limpiar": { "cefr_source": "A1" } } - }, - "_sample_bucket": "has_cefr_vote" - }, - { - "source_id": "ili:i4857", - "pos": "adjective", - "translations": { - "en": ["enthusiastic"], - "it": ["caloroso", "entusiastico", "fervido", "entusiasta"], - "fr": ["courageux", "enthousiaste"] - }, - "glosses": { "en": ["having or showing great excitement and interest"] }, - "examples": { - "en": [ - { "text": "enthusiastic crowds filled the streets", "source": "omw" }, - { "text": "an enthusiastic response", "source": "omw" }, - { - "text": "was enthusiastic about taking ballet lessons", - "source": "omw" - } - ], - "it": [ - { - "text": "Abbiamo ricevuto un'accoglienza molto calorosa.", - "source": "cefr" - }, - { - "text": "Ha espresso un fervido desiderio di pace.", - "source": "cefr" - }, - { "text": "Era molto entusiasta del nuovo progetto.", "source": "cefr" } - ], - "fr": [ - { "text": "C'est une personne très courageuse.", "source": "cefr" }, - { - "text": "Elle est très enthousiaste à l'idée de ce voyage.", - "source": "cefr" - } - ] - }, - "votes": { - "en": { "enthusiastic": { "cefr_source": "B1" } }, - "it": { - "caloroso": { "cefr_source": "B1" }, - "fervido": { "cefr_source": "C1" }, - "entusiasta": { "cefr_source": "B1" } - }, - "fr": { - "courageux": { "cefr_source": "A2" }, - "enthousiaste": { "cefr_source": "B1" } - } - }, - "_sample_bucket": "has_cefr_vote" - }, - { - "source_id": "ili:i104521", - "pos": "noun", - "translations": { - "en": ["veronica", "speedwell"], - "it": ["veronica"], - "de": [ - "Allerweltsheil", - "Grundheil", - "Ehrenpreis", - "Männertreu", - "Köhlerkraut", - "Schlangenkraut" - ], - "fr": ["veronica", "véronique"] - }, - "glosses": { - "en": ["any plant of the genus Veronica"], - "de": ["jede Pflanze der Gattung Veronica"] - }, - "examples": { - "de": [ - { - "text": "Er erhielt den Ehrenpreis für sein Lebenswerk.", - "source": "cefr" - } - ] - }, - "votes": { "de": { "Ehrenpreis": { "cefr_source": "C1" } } }, - "_sample_bucket": "has_cefr_vote" - }, - { - "source_id": "ili:i958", - "pos": "adjective", - "translations": { "en": ["gracious"], "es": ["amable"] }, - "glosses": { "en": ["disposed to bestow favors"] }, - "examples": { - "en": [{ "text": "thanks to the gracious gods", "source": "omw" }], - "es": [{ "text": "Siempre es muy amable con todos.", "source": "cefr" }] - }, - "votes": { - "en": { "gracious": { "cefr_source": "B2" } }, - "es": { "amable": { "cefr_source": "A2" } } - }, - "_sample_bucket": "has_cefr_vote" - }, - { - "source_id": "ili:i109447", - "pos": "noun", - "translations": { - "en": ["declension"], - "it": ["declinazione"], - "es": ["declinación"], - "de": ["Deklination", "Ortsmissweisung", "Missweisung"], - "fr": ["déclinaison"] - }, - "glosses": { - "en": [ - "the inflection of nouns and pronouns and adjectives in Indo-European languages" - ], - "de": [ - "die Beugung von Substantiven, Pronomen und Adjektiven in den indogermanischen Sprachen" - ] - }, - "examples": { - "it": [ - { - "text": "La declinazione dei nomi latini può essere complessa.", - "source": "cefr" - } - ], - "fr": [ - { "text": "En latin, les noms ont des déclinaisons.", "source": "cefr" } - ] - }, - "votes": { - "it": { "declinazione": { "cefr_source": "B2" } }, - "fr": { "déclinaison": { "cefr_source": "C1" } } - }, - "_sample_bucket": "has_cefr_vote" - }, - { - "source_id": "ili:i18812", - "pos": "adverb", - "translations": { - "en": ["fairly", "fair", "evenhandedly"], - "es": ["con justicia", "imparcialmente", "justamente"] - }, - "glosses": { - "en": ["without favoring one party, in a fair evenhanded manner"] - }, - "examples": { - "en": [{ "text": "deal fairly with one another", "source": "omw" }], - "es": [ - { - "text": "Llegó justamente a tiempo para la reunión.", - "source": "cefr" - } - ] - }, - "votes": { - "en": { "fairly": { "cefr_source": "B1" } }, - "es": { "justamente": { "cefr_source": "B2" } } - }, - "_sample_bucket": "has_cefr_vote" - }, - { - "source_id": "ili:i44747", - "pos": "noun", - "translations": { - "en": ["Centrocercus", "genus Centrocercus"], - "es": ["Centrocercus", "género Centrocercus"], - "fr": ["centrocercus"] - }, - "glosses": { "en": ["sage grouse"] }, - "examples": {}, - "votes": {}, - "_sample_bucket": "no_cefr_vote" - }, - { - "source_id": "ili:i20736", - "pos": "adverb", - "translations": { "en": ["insinuatingly"] }, - "glosses": { "en": ["in an insinuating manner"] }, - "examples": { - "en": [ - { - "text": "the art book has art to sell, insinuatingly, and for a purpose, like the American muse, which has in fact a tradition to sell, and one which doesn't exist, in painting", - "source": "omw" - } - ] - }, - "votes": {}, - "_sample_bucket": "no_cefr_vote" - }, - { - "source_id": "ili:i25017", - "pos": "verb", - "translations": { "en": ["superordinate"] }, - "glosses": { "en": ["place in a superior order or rank"] }, - "examples": { - "en": [ - { - "text": "These two notions are superordinated to a third", - "source": "omw" - } - ] - }, - "votes": {}, - "_sample_bucket": "no_cefr_vote" - }, - { - "source_id": "ili:i46616", - "pos": "noun", - "translations": { - "en": ["sand cat"], - "fr": [ - "chat de marguerite", - "chat du désert", - "chat du général marguerite", - "chat des sables" - ] - }, - "glosses": { "en": ["a desert wildcat"] }, - "examples": {}, - "votes": {}, - "_sample_bucket": "no_cefr_vote" - }, - { - "source_id": "ili:i83491", - "pos": "noun", - "translations": { "en": ["Bangor"] }, - "glosses": { - "en": ["a university town in northwestern Wales on the Menai Strait"] - }, - "examples": {}, - "votes": {}, - "_sample_bucket": "no_cefr_vote" - }, - { - "source_id": "ili:i72819", - "pos": "noun", - "translations": { - "en": ["Missouri"], - "fr": ["Saint Peters", "Joplin", "Missouri"] - }, - "glosses": { - "en": ["a dialect of the Chiwere language spoken by the Missouri"] - }, - "examples": {}, - "votes": {}, - "_sample_bucket": "no_cefr_vote" - }, - { - "source_id": "ili:i99797", - "pos": "noun", - "translations": { - "en": ["prickly poppy", "argemone", "white thistle", "devil's fig"], - "es": ["argemone"], - "fr": ["argemone"] - }, - "glosses": { - "en": [ - "any plant of the genus Argemone having large white or yellow flowers and prickly leaves and stems and pods; chiefly of tropical America" - ] - }, - "examples": {}, - "votes": {}, - "_sample_bucket": "no_cefr_vote" - }, - { - "source_id": "ili:i90317", - "pos": "noun", - "translations": { - "en": ["great-uncle", "granduncle"], - "it": ["protio", "prozio"], - "es": ["tío abuelo"], - "fr": ["grand-oncle"] - }, - "glosses": { "en": ["an uncle of your father or mother"] }, - "examples": {}, - "votes": {}, - "_sample_bucket": "no_cefr_vote" - }, - { - "source_id": "ili:i53881", - "pos": "noun", - "translations": { - "en": ["flour bin"], - "es": ["frasco de harina", "tarro de harina"] - }, - "glosses": { "en": ["a bin for holding flour"] }, - "examples": {}, - "votes": {}, - "_sample_bucket": "no_cefr_vote" - }, - { - "source_id": "ili:i58210", - "pos": "noun", - "translations": { "en": ["road map"], "fr": ["carte routière"] }, - "glosses": { "en": ["a map showing roads (for automobile travel)"] }, - "examples": {}, - "votes": {}, - "_sample_bucket": "no_cefr_vote" - }, - { - "source_id": "ili:i82638", - "pos": "noun", - "translations": { - "en": ["South American country", "South American nation"] - }, - "glosses": { - "en": ["any one of the countries occupying the South American continent"] - }, - "examples": {}, - "votes": {}, - "_sample_bucket": "no_cefr_vote" - }, - { - "source_id": "ili:i71111", - "pos": "noun", - "translations": { - "en": ["weekly"], - "it": ["ebdomadario", "eddomadario", "settimanale"], - "fr": ["hebdomadaire"] - }, - "glosses": { - "en": [ - "a periodical that is published every week (or 52 issues per year)" - ] - }, - "examples": {}, - "votes": {}, - "_sample_bucket": "no_cefr_vote" - }, - { - "source_id": "ili:i10131", - "pos": "adjective", - "translations": { "en": ["embattled"], "it": ["GAP!", "in difficoltà"] }, - "glosses": { "en": ["prepared for battle"] }, - "examples": { "en": [{ "text": "an embattled city", "source": "omw" }] }, - "votes": {}, - "_sample_bucket": "no_cefr_vote" - }, - { - "source_id": "ili:i108195", - "pos": "noun", - "translations": { "en": ["mass unit"], "es": ["unidad de masa"] }, - "glosses": { "en": ["a unit of measurement for mass"] }, - "examples": {}, - "votes": {}, - "_sample_bucket": "no_cefr_vote" - }, - { - "source_id": "ili:i82225", - "pos": "noun", - "translations": { "en": ["Wrangell-St. Elias National Park"] }, - "glosses": { - "en": [ - "the largest national park of the United States; located in Alaska" - ] - }, - "examples": {}, - "votes": {}, - "_sample_bucket": "no_cefr_vote" - }, - { - "source_id": "ili:i47159", - "pos": "noun", - "translations": { - "en": ["Fenusa", "genus-Fenusa"], - "es": ["Fenusa", "género Fenusa"] - }, - "glosses": { "en": ["birch leaf miner"] }, - "examples": {}, - "votes": {}, - "_sample_bucket": "no_cefr_vote" - }, - { - "source_id": "ili:i106504", - "pos": "noun", - "translations": { "en": ["entail"] }, - "glosses": { "en": ["land received by fee tail"] }, - "examples": {}, - "votes": {}, - "_sample_bucket": "no_cefr_vote" - }, - { - "source_id": "ili:i46047", - "pos": "noun", - "translations": { - "en": ["Polynesian tattler", "Heteroscelus incanus"], - "fr": ["heteroscelus incanus"] - }, - "glosses": { "en": ["tattler of Pacific coastal regions"] }, - "examples": {}, - "votes": {}, - "_sample_bucket": "no_cefr_vote" - }, - { - "source_id": "ili:i71598", - "pos": "noun", - "translations": { "en": ["market letter"] }, - "glosses": { - "en": [ - "a newsletter written by an analyst of the stock market and sold to subscribers" - ] - }, - "examples": {}, - "votes": {}, - "_sample_bucket": "no_cefr_vote" - }, - { - "source_id": "ili:i115719", - "pos": "noun", - "translations": { - "en": ["monosaccharide", "monosaccharose", "simple sugar"], - "it": ["manosio", "monosaccaride", "monosio", "monoso"], - "es": ["monosacárido"], - "de": ["Monosaccharid", "Einfachzucker"], - "fr": ["ose", "Ose", "monosaccharide"] - }, - "glosses": { - "en": [ - "a sugar (like sucrose or fructose) that does not hydrolyse to give other sugars; the simplest group of carbohydrates" - ], - "de": [ - "ein Zucker (wie Saccharose oder Fruktose), der nicht zu anderen Zuckern hydrolysiert wird; die einfachste Gruppe der Kohlenhydrate" - ] - }, - "examples": {}, - "votes": {}, - "_sample_bucket": "no_cefr_vote" - }, - { - "source_id": "ili:i74228", - "pos": "noun", - "translations": { - "en": ["negotiation", "dialogue", "talks"], - "it": [ - "contrattazione", - "deal", - "dialogo", - "negoziato", - "negoziazione", - "trattativa" - ], - "es": ["gestión", "negociación", "tramitación"], - "de": ["Besprechung", "Verhandlung"], - "fr": ["dialogue", "négociation"] - }, - "glosses": { - "en": ["a discussion intended to produce an agreement"], - "de": ["Diskussion zur Ausarbeitung eines Abkommens"] - }, - "examples": { - "en": [ - { - "text": "the buyout negotiation lasted several days", - "source": "omw" - }, - { "text": "they disagreed but kept an open dialogue", "source": "omw" }, - { "text": "talks between Israelis and Palestinians", "source": "omw" } - ], - "it": [ - { - "text": "La contrattazione collettiva è importante per i lavoratori.", - "source": "cefr" - }, - { "text": "Abbiamo chiuso un buon deal.", "source": "cefr" }, - { - "text": "È importante mantenere un dialogo aperto.", - "source": "cefr" - }, - { - "text": "Il negoziato per la pace è stato lungo e difficile.", - "source": "cefr" - }, - { - "text": "Le negoziazioni per il nuovo contratto sono state lunghe e complesse.", - "source": "cefr" - }, - { "text": "Le trattative sono in corso.", "source": "cefr" } - ], - "de": [ - { - "text": "Wir haben morgen eine wichtige Besprechung.", - "source": "cefr" - }, - { - "text": "Die Verhandlungen dauerten den ganzen Tag.", - "source": "cefr" - } - ], - "fr": [ - { - "text": "Le dialogue est essentiel pour résoudre les conflits.", - "source": "cefr" - }, - { - "text": "Les négociations ont été longues et difficiles.", - "source": "cefr" - } - ], - "es": [ - { "text": "La gestión del proyecto fue excelente.", "source": "cefr" }, - { "text": "Las negociaciones fueron difíciles.", "source": "cefr" }, - { - "text": "La tramitación de los documentos puede llevar tiempo.", - "source": "cefr" - } - ] - }, - "votes": { - "en": { - "negotiation": { "cefr_source": "B2" }, - "dialogue": { "cefr_source": "B2" } - }, - "it": { - "contrattazione": { "cefr_source": "B2" }, - "deal": { "cefr_source": "B1" }, - "dialogo": { "cefr_source": "B1" }, - "negoziato": { "cefr_source": "B2" }, - "negoziazione": { "cefr_source": "B2" }, - "trattativa": { "cefr_source": "B2" } - }, - "de": { - "Besprechung": { "cefr_source": "B1" }, - "Verhandlung": { "cefr_source": "B2" } - }, - "fr": { - "dialogue": { "cefr_source": "B1" }, - "négociation": { "cefr_source": "B2" } - }, - "es": { - "gestión": { "cefr_source": "B2" }, - "negociación": { "cefr_source": "B2" }, - "tramitación": { "cefr_source": "B2" } - } - }, - "_sample_bucket": "has_glosses_and_examples" - }, - { - "source_id": "ili:i408", - "pos": "adjective", - "translations": { - "en": ["aground"], - "es": ["encallado", "varado"], - "de": [ - "aufgrund", - "dank", - "aufgrund von", - "auf Grund von", - "vermöge", - "infolge", - "auf Grund" - ] - }, - "glosses": { - "en": ["stuck in a place where a ship can no longer float"], - "de": [ - "an einer Stelle feststecken, an der ein Schiff nicht mehr schwimmen kann" - ] - }, - "examples": { - "en": [ - { "text": "a ship aground offshore", "source": "omw" }, - { - "text": "a boat aground on the beach waiting for the tide to lift it", - "source": "omw" - } - ], - "es": [{ "text": "El barco quedó varado en la arena.", "source": "cefr" }] - }, - "votes": { "es": { "varado": { "cefr_source": "B2" } } }, - "_sample_bucket": "has_glosses_and_examples" - }, - { - "source_id": "ili:i41575", - "pos": "noun", - "translations": { "en": ["walkout"] }, - "glosses": { - "en": [ - "the act of walking out (of a meeting or organization) as a sign of protest" - ] - }, - "examples": { - "en": [ - { - "text": "there was a walkout by the Black members as the chairman rose to speak", - "source": "omw" - } - ] - }, - "votes": { "en": { "walkout": { "cefr_source": "B2" } } }, - "_sample_bucket": "has_glosses_and_examples" - }, - { - "source_id": "ili:i67480", - "pos": "noun", - "translations": { "en": ["tasting"], "fr": ["dégustation"] }, - "glosses": { "en": ["a small amount (especially of food or wine)"] }, - "examples": { - "fr": [ - { - "text": "Nous avons participé à une dégustation de vins.", - "source": "cefr" - } - ] - }, - "votes": { - "en": { "tasting": { "cefr_source": "B1" } }, - "fr": { "dégustation": { "cefr_source": "B1" } } - }, - "_sample_bucket": "has_glosses_and_examples" - }, - { - "source_id": "ili:i11256", - "pos": "adjective", - "translations": { "en": ["hobnailed"] }, - "glosses": { - "en": ["marked by the wearing of heavy boots studded with hobnails"] - }, - "examples": { "en": [{ "text": "hobnailed laborers", "source": "omw" }] }, - "votes": {}, - "_sample_bucket": "has_glosses_and_examples" - }, - { - "source_id": "ili:i86151", - "pos": "noun", - "translations": { - "en": ["sediment", "deposit"], - "it": ["deposito", "posatura", "sedimento"], - "es": ["depósito", "sedimento"], - "de": [ - "Ablagerung", - "Sedimentation", - "Sedimentierung", - "Sedimentbildung" - ], - "fr": ["sédiment", "dépôt"] - }, - "glosses": { - "en": ["matter that has been deposited by some natural process"], - "de": ["Materie, die durch einen natürlichen Prozess abgelagert wurde"] - }, - "examples": { - "it": [ - { "text": "Ho lasciato i bagagli al deposito.", "source": "cefr" }, - { - "text": "C'era un sedimento sul fondo della bottiglia.", - "source": "cefr" - } - ], - "de": [ - { "text": "Es gab Ablagerungen in den Rohren.", "source": "cefr" } - ], - "fr": [ - { - "text": "Le sédiment au fond du lac est très fin.", - "source": "cefr" - }, - { "text": "J'ai fait un dépôt à la banque.", "source": "cefr" } - ], - "es": [{ "text": "Hice un depósito en el banco.", "source": "cefr" }] - }, - "votes": { - "en": { - "sediment": { "cefr_source": "C1" }, - "deposit": { "cefr_source": "B1" } - }, - "it": { - "deposito": { "cefr_source": "B1" }, - "sedimento": { "cefr_source": "B2" } - }, - "de": { "Ablagerung": { "cefr_source": "B2" } }, - "fr": { - "sédiment": { "cefr_source": "B2" }, - "dépôt": { "cefr_source": "B1" } - }, - "es": { "depósito": { "cefr_source": "B1" } } - }, - "_sample_bucket": "has_glosses_and_examples" - }, - { - "source_id": "ili:i45550", - "pos": "noun", - "translations": { "en": ["conch"], "fr": ["conque"] }, - "glosses": { - "en": [ - "any of various edible tropical marine gastropods of the genus Strombus having a brightly-colored spiral shell with large outer lip" - ] - }, - "examples": { - "fr": [{ "text": "On entend la mer dans une conque.", "source": "cefr" }] - }, - "votes": { - "en": { "conch": { "cefr_source": "B1" } }, - "fr": { "conque": { "cefr_source": "B2" } } - }, - "_sample_bucket": "has_glosses_and_examples" - }, - { - "source_id": "ili:i117521", - "pos": "noun", - "translations": { - "en": ["moratorium"], - "it": ["moratoria"], - "fr": ["moratoire"] - }, - "glosses": { - "en": [ - "a legally authorized postponement before some obligation must be discharged" - ] - }, - "examples": { - "it": [ - { - "text": "Il governo ha imposto una moratoria sui nuovi progetti edilizi.", - "source": "cefr" - } - ], - "fr": [ - { - "text": "Le gouvernement a décrété un moratoire sur la pêche.", - "source": "cefr" - } - ] - }, - "votes": { - "en": { "moratorium": { "cefr_source": "C1" } }, - "it": { "moratoria": { "cefr_source": "C1" } }, - "fr": { "moratoire": { "cefr_source": "C1" } } - }, - "_sample_bucket": "has_glosses_and_examples" - }, - { - "source_id": "ili:i31764", - "pos": "verb", - "translations": { - "en": ["return"], - "fr": ["rendre", "retourner", "revenir"] - }, - "glosses": { "en": ["return to a previous position; in mathematics"] }, - "examples": { - "en": [ - { - "text": "The point returned to the interior of the figure", - "source": "omw" - } - ], - "fr": [ - { - "text": "Il doit rendre les livres à la bibliothèque.", - "source": "cefr" - }, - { - "text": "Je dois retourner ce livre à la bibliothèque.", - "source": "cefr" - }, - { "text": "Je dois revenir demain.", "source": "cefr" } - ] - }, - "votes": { - "fr": { - "rendre": { "cefr_source": "A2" }, - "retourner": { "cefr_source": "A2" }, - "revenir": { "cefr_source": "A1" } - } - }, - "_sample_bucket": "has_glosses_and_examples" - }, - { - "source_id": "ili:i48149", - "pos": "noun", - "translations": { - "en": ["post horse", "post-horse", "poster"], - "it": ["cavallo di posta"], - "fr": ["affiche"] - }, - "glosses": { - "en": [ - "a horse kept at an inn or post house for use by mail carriers or for rent to travelers" - ] - }, - "examples": { - "fr": [ - { "text": "L'affiche du concert est très colorée.", "source": "cefr" } - ] - }, - "votes": { - "en": { "poster": { "cefr_source": "A2" } }, - "fr": { "affiche": { "cefr_source": "A2" } } - }, - "_sample_bucket": "has_glosses_and_examples" - }, - { - "source_id": "ili:i51126", - "pos": "noun", - "translations": { - "en": ["brickwork"], - "it": ["GAP!", "muratura in mattoni"], - "es": ["aparejo", "calicanto", "enladrillado", "mampostería"], - "fr": ["appareil"] - }, - "glosses": { "en": ["masonry done with bricks and mortar"] }, - "examples": { - "fr": [ - { "text": "J'ai acheté un nouvel appareil photo.", "source": "cefr" } - ] - }, - "votes": { - "en": { "brickwork": { "cefr_source": "B2" } }, - "fr": { "appareil": { "cefr_source": "B1" } } - }, - "_sample_bucket": "has_glosses_and_examples" - }, - { - "source_id": "ili:i17542", - "pos": "adjective", - "translations": { - "en": ["interdisciplinary"], - "it": ["interdisciplinare", "multidisciplinare"], - "de": [ - "multidisziplinär", - "fachübergreifend", - "interdisziplinär", - "fächerübergreifend" - ], - "fr": ["interdisciplinaire"] - }, - "glosses": { - "en": [ - "drawing from or characterized by participation of two or more fields of study" - ], - "de": ["die Zusammenarbeit mehrerer Disziplinen betreffend\">"] - }, - "examples": { - "en": [ - { "text": "interdisciplinary studies", "source": "omw" }, - { "text": "an interdisciplinary conference", "source": "omw" } - ], - "it": [ - { - "text": "Il progetto richiede un approccio interdisciplinare.", - "source": "cefr" - } - ], - "de": [ - { - "text": "Das Projekt ist interdisziplinär angelegt.", - "source": "cefr" - } - ], - "fr": [ - { - "text": "Ce projet de recherche est résolument interdisciplinaire.", - "source": "cefr" - } - ] - }, - "votes": { - "en": { "interdisciplinary": { "cefr_source": "C1" } }, - "it": { "interdisciplinare": { "cefr_source": "C1" } }, - "de": { "interdisziplinär": { "cefr_source": "C1" } }, - "fr": { "interdisciplinaire": { "cefr_source": "C1" } } - }, - "_sample_bucket": "has_glosses_and_examples" - }, - { - "source_id": "ili:i69459", - "pos": "noun", - "translations": { - "en": ["new edition"], - "it": ["riedizione"], - "fr": ["new edition"] - }, - "glosses": { - "en": [ - "a publication (such as a book) that has been modified or updated and offered again for sale" - ] - }, - "examples": { - "it": [ - { - "text": "Il libro è stato pubblicato in una nuova riedizione.", - "source": "cefr" - } - ] - }, - "votes": { "it": { "riedizione": { "cefr_source": "C1" } } }, - "_sample_bucket": "has_glosses_and_examples" - }, - { - "source_id": "ili:i75841", - "pos": "noun", - "translations": { - "en": ["stampede"], - "de": ["Stampede", "Herdenpanik"], - "fr": ["débandade"] - }, - "glosses": { - "en": ["a wild headlong rush of frightened animals (horses or cattle)"], - "de": [ - "eine wilde, kopfüber laufende Flucht von verängstigten Tieren (Pferden oder Rindern)" - ] - }, - "examples": { - "fr": [ - { - "text": "Après l'explosion, ce fut la débandade générale.", - "source": "cefr" - } - ] - }, - "votes": { - "en": { "stampede": { "cefr_source": "B2" } }, - "fr": { "débandade": { "cefr_source": "C1" } } - }, - "_sample_bucket": "has_glosses_and_examples" - }, - { - "source_id": "ili:i67108", - "pos": "noun", - "translations": { - "en": ["stocktaking", "stock-taking"], - "it": ["inventario"], - "es": ["balance"] - }, - "glosses": { "en": ["reappraisal of a situation or position or outlook"] }, - "examples": { - "it": [ - { - "text": "Dobbiamo fare l'inventario del magazzino.", - "source": "cefr" - } - ], - "es": [ - { - "text": "Es importante mantener un balance entre trabajo y vida personal.", - "source": "cefr" - } - ] - }, - "votes": { - "it": { "inventario": { "cefr_source": "B2" } }, - "es": { "balance": { "cefr_source": "B1" } } - }, - "_sample_bucket": "has_glosses_and_examples" - }, - { - "source_id": "ili:i14270", - "pos": "adjective", - "translations": { - "en": [ - "cockamamie", - "cockamamy", - "goofy", - "sappy", - "silly", - "wacky", - "whacky", - "zany" - ], - "es": ["tonto"], - "de": [ - "albern", - "naiv", - "unreif", - "kindsköpfig", - "kindlich", - "kindisch", - "unentwickelt", - "kindhaft", - "pueril", - "infantil", - "puerilistisch" - ], - "fr": ["déraisonnable", "fou", "drôle", "aberrant"] - }, - "glosses": { "en": ["ludicrous, foolish"], "de": ["lächerlich, töricht"] }, - "examples": { - "en": [ - { - "text": "gave me a cockamamie reason for not going", - "source": "omw" - }, - { "text": "wore a goofy hat", "source": "omw" }, - { "text": "a silly idea", "source": "omw" }, - { "text": "some wacky plan for selling more books", "source": "omw" } - ], - "de": [ - { "text": "Hör auf, so albern zu sein!", "source": "cefr" }, - { "text": "Sie ist manchmal etwas naiv.", "source": "cefr" }, - { "text": "Die Früchte sind noch unreif.", "source": "cefr" }, - { "text": "Sie hat eine sehr kindliche Freude.", "source": "cefr" }, - { "text": "Sein Verhalten war ziemlich kindisch.", "source": "cefr" } - ], - "fr": [ - { "text": "Ses exigences sont déraisonnables.", "source": "cefr" }, - { "text": "C'est une idée folle.", "source": "cefr" }, - { "text": "C'est une histoire drôle.", "source": "cefr" }, - { - "text": "Son comportement était aberrant et choquant.", - "source": "cefr" - } - ], - "es": [{ "text": "No seas tonto, eso no es verdad.", "source": "cefr" }] - }, - "votes": { - "en": { - "goofy": { "cefr_source": "B1" }, - "sappy": { "cefr_source": "B2" }, - "silly": { "cefr_source": "A2" }, - "wacky": { "cefr_source": "B2" }, - "zany": { "cefr_source": "B2" } - }, - "de": { - "albern": { "cefr_source": "B1" }, - "naiv": { "cefr_source": "B1" }, - "unreif": { "cefr_source": "B1" }, - "kindlich": { "cefr_source": "B1" }, - "kindisch": { "cefr_source": "B1" } - }, - "fr": { - "déraisonnable": { "cefr_source": "B2" }, - "fou": { "cefr_source": "B1" }, - "drôle": { "cefr_source": "A2" }, - "aberrant": { "cefr_source": "C1" } - }, - "es": { "tonto": { "cefr_source": "A2" } } - }, - "_sample_bucket": "has_glosses_and_examples" - }, - { - "source_id": "ili:i1291", - "pos": "adjective", - "translations": { "en": ["unifacial"] }, - "glosses": { "en": ["having but one principal or specialized surface"] }, - "examples": { - "en": [{ "text": "a primitive unifacial flint tool", "source": "omw" }] - }, - "votes": {}, - "_sample_bucket": "has_glosses_and_examples" - }, - { - "source_id": "ili:i73668", - "pos": "noun", - "translations": { - "en": ["cantata", "oratorio"], - "it": ["cantata", "oratorio"], - "es": ["oratorio"], - "de": ["Andachtsraum", "Oratorium", "Gebetsraum"], - "fr": ["oratorio", "cantate"] - }, - "glosses": { - "en": [ - "a musical composition for voices and orchestra based on a religious text" - ], - "de": [ - "eine musikalische Komposition für Stimmen und Orchester auf der Grundlage eines religiösen Textes" - ] - }, - "examples": { - "it": [ - { - "text": "I bambini giocano nell'oratorio della chiesa.", - "source": "cefr" - } - ], - "de": [ - { - "text": "Händels \"Messiah\" ist ein berühmtes Oratorium.", - "source": "cefr" - } - ], - "es": [ - { - "text": "El oratorio de la iglesia es un lugar de paz y reflexión.", - "source": "cefr" - } - ] - }, - "votes": { - "it": { "oratorio": { "cefr_source": "B1" } }, - "de": { "Oratorium": { "cefr_source": "C1" } }, - "es": { "oratorio": { "cefr_source": "C1" } } - }, - "_sample_bucket": "has_glosses_and_examples" - }, - { - "source_id": "ili:i39774", - "pos": "noun", - "translations": { "en": ["respiration"], "es": ["respiración"] }, - "glosses": { "en": ["a single complete act of breathing in and out"] }, - "examples": { - "en": [{ "text": "thirty respirations per minute", "source": "omw" }], - "es": [ - { "text": "Su respiración era lenta y profunda.", "source": "cefr" } - ] - }, - "votes": { - "en": { "respiration": { "cefr_source": "B2" } }, - "es": { "respiración": { "cefr_source": "B1" } } - }, - "_sample_bucket": "has_glosses_and_examples" - }, - { - "source_id": "ili:i28838", - "pos": "verb", - "translations": { "en": ["unplug", "disconnect"], "fr": ["débrancher"] }, - "glosses": { - "en": ["pull the plug of (electrical appliances) and render inoperable"] - }, - "examples": { - "en": [ - { "text": "unplug the hair dryer after using it", "source": "omw" } - ], - "fr": [ - { - "text": "N'oubliez pas de débrancher l'appareil après utilisation.", - "source": "cefr" - } - ] - }, - "votes": { - "en": { "unplug": { "cefr_source": "A2" } }, - "fr": { "débrancher": { "cefr_source": "A2" } } - }, - "_sample_bucket": "has_glosses_and_examples" - }, - { - "source_id": "ili:i85884", - "pos": "noun", - "translations": { - "en": ["North Sea"], - "es": ["Mar del Norte"], - "de": ["Nordsee", "Deutsches Meer"], - "fr": ["mer du Nord", "Mer du Nord"] - }, - "glosses": { - "en": [ - "an arm of the North Atlantic between the British Isles and Scandinavia; oil was discovered under the North Sea in 1970" - ], - "de": [ - "ein Arm des Nordatlantiks zwischen den Britischen Inseln und Skandinavien; 1970 wurde unter der Nordsee Öl entdeckt" - ] - }, - "examples": { - "de": [ - { "text": "Wir fahren im Sommer an die Nordsee.", "source": "cefr" } - ] - }, - "votes": { "de": { "Nordsee": { "cefr_source": "A2" } } }, - "_sample_bucket": "no_glosses_no_examples" - }, - { - "source_id": "ili:i57058", - "pos": "noun", - "translations": { - "en": ["patriarchal cross"], - "es": ["cruz patriarcal"], - "de": [ - "Erzbischofskreuz", - "Spanisches Kreuz", - "Ungarisches Kreuz", - "Patriarchenkreuz", - "Patriarchenhochkreuz" - ] - }, - "glosses": { - "en": ["a cross with two crossbars"], - "de": ["ein Kreuz mit zwei Querbalken"] - }, - "examples": {}, - "votes": {}, - "_sample_bucket": "no_glosses_no_examples" - }, - { - "source_id": "ili:i14067", - "pos": "adjective", - "translations": { - "en": ["maximizing", "maximising"], - "fr": ["maximaliste"] - }, - "glosses": { "en": ["making as great as possible"] }, - "examples": {}, - "votes": {}, - "_sample_bucket": "no_glosses_no_examples" - }, - { - "source_id": "ili:i57206", - "pos": "noun", - "translations": { - "en": ["photocathode"], - "es": ["fotocátodo"], - "de": ["Photokathode", "Fotokathode"], - "fr": ["photocathode"] - }, - "glosses": { - "en": ["a cathode that emits electrons when illuminated"], - "de": ["eine Kathode, die bei Beleuchtung Elektronen abgibt"] - }, - "examples": {}, - "votes": {}, - "_sample_bucket": "no_glosses_no_examples" - }, - { - "source_id": "ili:i97025", - "pos": "noun", - "translations": { - "en": ["Stockton", "Frank Stockton", "Francis Richard Stockton"], - "es": ["Francis Richard Stockton", "Frank Stockton", "Stockton"], - "fr": ["Stockton"] - }, - "glosses": { "en": ["United States writer (1834-1902)"] }, - "examples": {}, - "votes": {}, - "_sample_bucket": "no_glosses_no_examples" - }, - { - "source_id": "ili:i101248", - "pos": "noun", - "translations": { "en": ["obeche"] }, - "glosses": { - "en": [ - "the wood of an African obeche tree; used especially for veneering" - ] - }, - "examples": {}, - "votes": {}, - "_sample_bucket": "no_glosses_no_examples" - }, - { - "source_id": "ili:i94985", - "pos": "noun", - "translations": { - "en": ["Eames", "Charles Eames"], - "es": ["Charles Eames"] - }, - "glosses": { - "en": [ - "United States designer noted for an innovative series of chairs (1907-1978)" - ] - }, - "examples": {}, - "votes": {}, - "_sample_bucket": "no_glosses_no_examples" - }, - { - "source_id": "ili:i16699", - "pos": "adjective", - "translations": { - "en": ["mensural", "measured", "mensurable"], - "es": ["mensural"] - }, - "glosses": { "en": ["having notes of fixed rhythmic value"] }, - "examples": {}, - "votes": {}, - "_sample_bucket": "no_glosses_no_examples" - }, - { - "source_id": "ili:i99999", - "pos": "noun", - "translations": { - "en": ["China aster", "Callistephus chinensis"], - "fr": ["callistephus chinensis"] - }, - "glosses": { - "en": [ - "valued for their beautiful flowers in a wide range of clear bright colors; grown primarily for cutting" - ] - }, - "examples": {}, - "votes": {}, - "_sample_bucket": "no_glosses_no_examples" - }, - { - "source_id": "ili:i75135", - "pos": "noun", - "translations": { "en": ["kiss of death"], "fr": ["baiser de la mort"] }, - "glosses": { "en": ["something that is ruinous"] }, - "examples": { - "en": [ - { - "text": "if this were known it would be the kiss of death for my political career", - "source": "omw" - } - ] - }, - "votes": {}, - "_sample_bucket": "no_glosses_no_examples" - }, - { - "source_id": "ili:i36428", - "pos": "noun", - "translations": { "en": ["dark adaptation"] }, - "glosses": { - "en": [ - "the process of adjusting the eyes to low levels of illumination; cones adapt first; rods continue to adapt for up to four hours" - ] - }, - "examples": {}, - "votes": {}, - "_sample_bucket": "no_glosses_no_examples" - }, - { - "source_id": "ili:i103092", - "pos": "noun", - "translations": { - "en": ["saw palmetto", "scrub palmetto", "Serenoa repens"] - }, - "glosses": { - "en": ["small hardy clump-forming spiny palm of southern United States"] - }, - "examples": {}, - "votes": {}, - "_sample_bucket": "no_glosses_no_examples" - }, - { - "source_id": "ili:i14834", - "pos": "adjective", - "translations": { "en": ["zoic"] }, - "glosses": { "en": ["pertaining to animals or animal life or action"] }, - "examples": {}, - "votes": {}, - "_sample_bucket": "no_glosses_no_examples" - }, - { - "source_id": "ili:i25953", - "pos": "verb", - "translations": { "en": ["blog"], "es": ["blogear"] }, - "glosses": { "en": ["read, write, or edit a shared on-line journal"] }, - "examples": {}, - "votes": {}, - "_sample_bucket": "no_glosses_no_examples" - }, - { - "source_id": "ili:i24441", - "pos": "verb", - "translations": { "en": ["ream"], "es": ["taladrar"] }, - "glosses": { "en": ["enlarge with a reamer"] }, - "examples": { "en": [{ "text": "ream a hole", "source": "omw" }] }, - "votes": {}, - "_sample_bucket": "no_glosses_no_examples" - }, - { - "source_id": "ili:i60874", - "pos": "noun", - "translations": { - "en": ["virtual memory", "virtual storage"], - "it": ["memoria virtuale"], - "es": ["memoria virtual"], - "fr": ["mémoire virtuelle"] - }, - "glosses": { - "en": [ - "(computer science) memory created by using the hard disk to simulate additional random-access memory; the addressable storage space available to the user of a computer system in which virtual addresses are mapped into real addresses" - ] - }, - "examples": {}, - "votes": {}, - "_sample_bucket": "no_glosses_no_examples" - }, - { - "source_id": "ili:i105979", - "pos": "noun", - "translations": { - "en": ["Dryopteris", "genus Dryopteris"], - "fr": ["Dryopteris", "dryopteris"] - }, - "glosses": { - "en": [ - "large widespread genus of medium-sized terrestrial ferns; in some classification systems placed in Polypodiaceae" - ] - }, - "examples": {}, - "votes": {}, - "_sample_bucket": "no_glosses_no_examples" - }, - { - "source_id": "ili:i44411", - "pos": "noun", - "translations": { - "en": ["blue racer", "Coluber constrictor flaviventris"], - "fr": ["coluber constrictor"] - }, - "glosses": { "en": ["bluish-green blacksnake found from Ohio to Texas"] }, - "examples": {}, - "votes": {}, - "_sample_bucket": "no_glosses_no_examples" - }, - { - "source_id": "ili:i14592", - "pos": "adjective", - "translations": { - "en": ["anagrammatic", "anagrammatical"], - "it": ["anagrammatico"] - }, - "glosses": { - "en": ["related to anagrams or containing or making an anagram"] - }, - "examples": {}, - "votes": {}, - "_sample_bucket": "no_glosses_no_examples" - }, - { - "source_id": "ili:i5174", - "pos": "adjective", - "translations": { - "en": ["protrusile", "protrusible"], - "fr": ["protrusible"] - }, - "glosses": { "en": ["capable of being thrust forward, as the tongue"] }, - "examples": {}, - "votes": {}, - "_sample_bucket": "no_glosses_no_examples" - }, - { - "source_id": "ili:i99278", - "pos": "noun", - "translations": { "en": ["pink calla", "Zantedeschia rehmanii"] }, - "glosses": { "en": ["calla having a rose-colored spathe"] }, - "examples": {}, - "votes": {}, - "_sample_bucket": "pos_spread" - }, - { - "source_id": "ili:i97983", - "pos": "noun", - "translations": { - "en": ["phosphorescence"], - "it": ["fosforescenza", "fotoluminescenza"], - "fr": ["phosphorescence"] - }, - "glosses": { - "en": [ - "a fluorescence that persists after the bombarding radiation has ceased" - ] - }, - "examples": {}, - "votes": {}, - "_sample_bucket": "pos_spread" - }, - { - "source_id": "ili:i54194", - "pos": "noun", - "translations": { "en": ["garrison cap", "overseas cap"] }, - "glosses": { - "en": ["a wedge-shaped wool or cotton cap; worn as part of a uniform"] - }, - "examples": {}, - "votes": {}, - "_sample_bucket": "pos_spread" - }, - { - "source_id": "ili:i102972", - "pos": "noun", - "translations": { "en": ["Tipuana", "genus Tipuana"], "fr": ["tipuana"] }, - "glosses": { "en": ["one species: South American tree: tipu tree"] }, - "examples": {}, - "votes": {}, - "_sample_bucket": "pos_spread" - }, - { - "source_id": "ili:i55386", - "pos": "noun", - "translations": { "en": ["king"], "fr": ["roi"] }, - "glosses": { - "en": [ - "a checker that has been moved to the opponent's first row where it is promoted to a piece that is free to move either forward or backward" - ] - }, - "examples": { - "fr": [{ "text": "Le roi a visité la ville.", "source": "cefr" }] - }, - "votes": { - "en": { "king": { "cefr_source": "A2" } }, - "fr": { "roi": { "cefr_source": "B1" } } - }, - "_sample_bucket": "pos_spread" - }, - { - "source_id": "ili:i26482", - "pos": "verb", - "translations": { - "en": ["articulate", "enunciate", "vocalize", "vocalise"], - "it": ["articolare", "enunciare", "enunziare", "scandire"], - "es": ["articular"], - "de": ["ausdrücken", "artikulieren"], - "fr": ["articuler", "exprimer", "énoncer", "formuler", "vocaliser"] - }, - "glosses": { - "en": ["express or state clearly"], - "de": ["klar ausdrücken oder erklären"] - }, - "examples": { - "it": [ - { "text": "È importante articolare bene le parole.", "source": "cefr" } - ], - "de": [ - { - "text": "Er konnte seine Gefühle nicht ausdrücken.", - "source": "cefr" - }, - { - "text": "Er konnte seine Gedanken nicht klar artikulieren.", - "source": "cefr" - } - ], - "fr": [ - { - "text": "Il faut bien articuler pour être compris.", - "source": "cefr" - }, - { - "text": "Il est difficile d'exprimer ses sentiments.", - "source": "cefr" - }, - { - "text": "Le professeur a énoncé les règles clairement.", - "source": "cefr" - }, - { - "text": "Il a formulé une question très pertinente.", - "source": "cefr" - } - ], - "es": [ - { - "text": "Es importante articular bien las palabras al hablar en público.", - "source": "cefr" - } - ] - }, - "votes": { - "en": { "articulate": { "cefr_source": "B2" } }, - "it": { "articolare": { "cefr_source": "B2" } }, - "de": { - "ausdrücken": { "cefr_source": "B1" }, - "artikulieren": { "cefr_source": "B2" } - }, - "fr": { - "articuler": { "cefr_source": "B1" }, - "exprimer": { "cefr_source": "B1" }, - "énoncer": { "cefr_source": "B2" }, - "formuler": { "cefr_source": "B2" } - }, - "es": { "articular": { "cefr_source": "B2" } } - }, - "_sample_bucket": "pos_spread" - }, - { - "source_id": "ili:i22492", - "pos": "verb", - "translations": { "en": ["spike"] }, - "glosses": { "en": ["manifest a sharp increase"] }, - "examples": { "en": [{ "text": "the voltage spiked", "source": "omw" }] }, - "votes": {}, - "_sample_bucket": "pos_spread" - }, - { - "source_id": "ili:i26383", - "pos": "verb", - "translations": { "en": ["redefine"], "fr": ["redéfinir"] }, - "glosses": { "en": ["give a new or different definition of (a word)"] }, - "examples": { - "fr": [ - { "text": "Il est temps de redéfinir nos objectifs.", "source": "cefr" } - ] - }, - "votes": { - "en": { "redefine": { "cefr_source": "B2" } }, - "fr": { "redéfinir": { "cefr_source": "B2" } } - }, - "_sample_bucket": "pos_spread" - }, - { - "source_id": "ili:i22943", - "pos": "verb", - "translations": { - "en": ["slake", "abate", "slack"], - "es": ["aflojar", "reducir"], - "fr": [ - "descendre", - "cesser", - "réduire", - "ralentir", - "amoindrir", - "diminuer", - "supprimer" - ] - }, - "glosses": { "en": ["make less active or intense"] }, - "examples": { - "fr": [ - { - "text": "Nous allons descendre au rez-de-chaussée.", - "source": "cefr" - }, - { "text": "La pluie a cessé de tomber.", "source": "cefr" }, - { "text": "Nous devons réduire nos dépenses.", "source": "cefr" }, - { "text": "Il faut ralentir avant le virage.", "source": "cefr" }, - { - "text": "Ces mesures visent à amoindrir l'impact de la crise.", - "source": "cefr" - }, - { "text": "Les prix ont commencé à diminuer.", "source": "cefr" }, - { "text": "Il faut supprimer les fichiers inutiles.", "source": "cefr" } - ], - "es": [ - { "text": "Tienes que aflojar el nudo.", "source": "cefr" }, - { - "text": "Necesitamos reducir el consumo de energía.", - "source": "cefr" - } - ] - }, - "votes": { - "en": { "abate": { "cefr_source": "C1" } }, - "fr": { - "descendre": { "cefr_source": "A2" }, - "cesser": { "cefr_source": "B1" }, - "réduire": { "cefr_source": "B1" }, - "ralentir": { "cefr_source": "B1" }, - "amoindrir": { "cefr_source": "C1" }, - "diminuer": { "cefr_source": "B1" }, - "supprimer": { "cefr_source": "B2" } - }, - "es": { - "aflojar": { "cefr_source": "B1" }, - "reducir": { "cefr_source": "B1" } - } - }, - "_sample_bucket": "pos_spread" - }, - { - "source_id": "ili:i31348", - "pos": "verb", - "translations": { "en": ["romp"] }, - "glosses": { "en": ["run easily and fairly fast"] }, - "examples": {}, - "votes": { "en": { "romp": { "cefr_source": "B2" } } }, - "_sample_bucket": "pos_spread" - }, - { - "source_id": "ili:i10413", - "pos": "adjective", - "translations": { - "en": ["imprudent"], - "it": ["imprudente", "incauto"], - "es": ["imprudente", "insensato"], - "fr": ["imprudent"] - }, - "glosses": { "en": ["not prudent or wise"] }, - "examples": { - "en": [ - { - "text": "very imprudent of her mother to encourage her in such silly romantic ideas", - "source": "omw" - }, - { - "text": "\"would be imprudent for a noneconomist to talk about the details of economic policy\"- A.M.Schlesinger", - "source": "omw" - } - ], - "it": [ - { - "text": "È stato imprudente guidare così velocemente.", - "source": "cefr" - } - ], - "fr": [ - { - "text": "C'était imprudent de traverser sans regarder.", - "source": "cefr" - } - ], - "es": [ - { - "text": "Fue una decisión imprudente conducir tan rápido.", - "source": "cefr" - }, - { "text": "Fue una decisión insensata.", "source": "cefr" } - ] - }, - "votes": { - "it": { "imprudente": { "cefr_source": "B2" } }, - "fr": { "imprudent": { "cefr_source": "B2" } }, - "es": { - "imprudente": { "cefr_source": "B2" }, - "insensato": { "cefr_source": "B2" } - } - }, - "_sample_bucket": "pos_spread" - }, - { - "source_id": "ili:i8645", - "pos": "adjective", - "translations": { - "en": ["metaphysical"], - "es": ["metafísico"], - "fr": ["métaphysique"] - }, - "glosses": { "en": ["without material form or substance"] }, - "examples": { "en": [{ "text": "metaphysical forces", "source": "omw" }] }, - "votes": { "en": { "metaphysical": { "cefr_source": "C1" } } }, - "_sample_bucket": "pos_spread" - }, - { - "source_id": "ili:i6969", - "pos": "adjective", - "translations": { - "en": [ - "all-important", - "all important", - "crucial", - "essential", - "of the essence" - ], - "it": ["essenziale"], - "es": ["crucial", "esencial"], - "de": [ - "bedeutsam", - "wesentlich", - "wichtig", - "prägnant", - "hauptsächlich", - "gehaltvoll", - "aussagekräftig", - "signifikant" - ], - "fr": ["essentiel"] - }, - "glosses": { - "en": ["of the greatest importance"], - "de": ["von allergrößter Bedeutung"] - }, - "examples": { - "en": [ - { "text": "the all-important subject of disarmament", "source": "omw" }, - { "text": "crucial information", "source": "omw" }, - { "text": "in chess cool nerves are of the essence", "source": "omw" } - ], - "it": [{ "text": "L'acqua è essenziale per la vita.", "source": "cefr" }], - "de": [ - { - "text": "Das war ein bedeutsamer Moment in der Geschichte.", - "source": "cefr" - }, - { "text": "Das ist ein wesentlicher Unterschied.", "source": "cefr" }, - { "text": "Das ist eine wichtige Information.", "source": "cefr" }, - { - "text": "Er formulierte seine Gedanken sehr prägnant.", - "source": "cefr" - }, - { - "text": "Die Studie lieferte aussagekräftige Ergebnisse.", - "source": "cefr" - }, - { "text": "Es gab eine signifikante Veränderung.", "source": "cefr" } - ], - "fr": [ - { - "text": "C'est essentiel de bien manger pour rester en forme.", - "source": "cefr" - } - ], - "es": [ - { "text": "Es crucial que lleguemos a tiempo.", "source": "cefr" }, - { "text": "El agua es esencial para la vida.", "source": "cefr" } - ] - }, - "votes": { - "en": { - "crucial": { "cefr_source": "B2" }, - "essential": { "cefr_source": "B1" } - }, - "it": { "essenziale": { "cefr_source": "B1" } }, - "de": { - "bedeutsam": { "cefr_source": "B2" }, - "wesentlich": { "cefr_source": "B1" }, - "wichtig": { "cefr_source": "A1" }, - "prägnant": { "cefr_source": "B2" }, - "aussagekräftig": { "cefr_source": "B2" }, - "signifikant": { "cefr_source": "C1" } - }, - "fr": { "essentiel": { "cefr_source": "B1" } }, - "es": { - "crucial": { "cefr_source": "B2" }, - "esencial": { "cefr_source": "B1" } - } - }, - "_sample_bucket": "pos_spread" - }, - { - "source_id": "ili:i13690", - "pos": "adjective", - "translations": { "en": ["round-arm"] }, - "glosses": { "en": ["with the arm swung round at shoulder height"] }, - "examples": { "en": [{ "text": "round-arm bowling", "source": "omw" }] }, - "votes": {}, - "_sample_bucket": "pos_spread" - }, - { - "source_id": "ili:i16993", - "pos": "adjective", - "translations": { - "en": ["Monacan", "Monegasque"], - "it": ["monegasco"], - "fr": ["monégasque"] - }, - "glosses": { - "en": ["of or relating to or characteristic of Monaco or its people"] - }, - "examples": { - "fr": [{ "text": "Il est de nationalité monégasque.", "source": "cefr" }] - }, - "votes": { "fr": { "monégasque": { "cefr_source": "B1" } } }, - "_sample_bucket": "pos_spread" - }, - { - "source_id": "ili:i18824", - "pos": "adverb", - "translations": { - "en": ["here", "hither"], - "it": ["qua", "qui"], - "fr": ["ici", "çà", "par ici"] - }, - "glosses": { "en": ["to this place (especially toward the speaker)"] }, - "examples": { - "en": [{ "text": "come here, please", "source": "omw" }], - "it": [ - { "text": "Vieni qua, per favore.", "source": "cefr" }, - { "text": "Vieni qui!", "source": "cefr" } - ], - "fr": [{ "text": "Venez ici !", "source": "cefr" }] - }, - "votes": { - "en": { - "here": { "cefr_source": "A1" }, - "hither": { "cefr_source": "C2" } - }, - "it": { "qua": { "cefr_source": "A1" }, "qui": { "cefr_source": "A1" } }, - "fr": { "ici": { "cefr_source": "A1" } } - }, - "_sample_bucket": "pos_spread" - }, - { - "source_id": "ili:i19641", - "pos": "adverb", - "translations": { "en": ["head-on"], "es": ["de frente"] }, - "glosses": { "en": ["with the front foremost"] }, - "examples": { - "en": [{ "text": "the cars collided head-on", "source": "omw" }] - }, - "votes": {}, - "_sample_bucket": "pos_spread" - }, - { - "source_id": "ili:i21417", - "pos": "adverb", - "translations": { "en": ["sweepingly"] }, - "glosses": { "en": ["in a sweeping manner"] }, - "examples": { - "en": [ - { - "text": "he sweepingly condemned the entire population of the country for the war crimes", - "source": "omw" - } - ] - }, - "votes": {}, - "_sample_bucket": "pos_spread" - }, - { - "source_id": "ili:i20131", - "pos": "adverb", - "translations": { - "en": ["gallantly", "chivalrously"], - "it": ["galantemente"], - "fr": ["chevaleresquement"] - }, - "glosses": { "en": ["in a gallant manner"] }, - "examples": { - "en": [ - { "text": "he gallantly offered to take her home", "source": "omw" } - ] - }, - "votes": {}, - "_sample_bucket": "pos_spread" - }, - { - "source_id": "ili:i20516", - "pos": "adverb", - "translations": { "en": ["fractiously"] }, - "glosses": { "en": ["in a fractious manner"] }, - "examples": { - "en": [ - { - "text": "the horse was behaving fractiously and refused to jump", - "source": "omw" - } - ] - }, - "votes": {}, - "_sample_bucket": "pos_spread" - } -] diff --git a/data-pipeline/sample/scripts/sample.ts b/data-pipeline/sample/scripts/sample.ts deleted file mode 100644 index 9aece55..0000000 --- a/data-pipeline/sample/scripts/sample.ts +++ /dev/null @@ -1,205 +0,0 @@ -import fs from "node:fs/promises"; -import path from "node:path"; -import { SUPPORTED_LANGUAGE_CODES } from "@lila/shared"; -import type { SupportedLanguageCode, SupportedPos } from "@lila/shared"; - -// ── Types ───────────────────────────────────────────────────────────────────── - -type Example = { text: string; source: "omw" | "cefr" }; - -type AnnotatedRecord = { - source_id: string; - pos: SupportedPos; - translations: Partial>; - glosses: Partial>; - examples: Partial>; - votes: Partial< - Record> - >; -}; - -type SampleRecord = AnnotatedRecord & { _sample_bucket: string }; - -// ── Constants ───────────────────────────────────────────────────────────────── - -const PATHS = { - annotatedDir: "stage-2-annotate/output", - output: "test/output/sample.json", -}; - -const BUCKET_SIZE = 20; - -// ── Bucket predicates ───────────────────────────────────────────────────────── - -type Bucket = { name: string; predicate: (record: AnnotatedRecord) => boolean }; - -const BUCKETS: Bucket[] = [ - { - name: "has_cefr_vote", - predicate: (r) => - Object.values(r.votes).some( - (langVotes) => Object.keys(langVotes ?? {}).length > 0, - ), - }, - { - name: "no_cefr_vote", - predicate: (r) => - Object.values(r.votes).every( - (langVotes) => Object.keys(langVotes ?? {}).length === 0, - ), - }, - { - name: "has_glosses_and_examples", - predicate: (r) => - Object.keys(r.glosses).length > 0 && Object.keys(r.examples).length > 0, - }, - { - name: "no_glosses_no_examples", - predicate: (r) => - !r.glosses["fr"] && - !r.examples["fr"] && - !r.votes["fr"] && - !r.glosses["es"] && - !r.examples["es"] && - !r.votes["es"], - }, - { - name: "pos_spread", - predicate: () => true, // sampled separately to ensure POS coverage - }, -]; - -// ── Sampling ────────────────────────────────────────────────────────────────── - -function sampleBucket( - records: AnnotatedRecord[], - predicate: (r: AnnotatedRecord) => boolean, - size: number, - exclude: Set, -): AnnotatedRecord[] { - const candidates = records.filter( - (r) => !exclude.has(r.source_id) && predicate(r), - ); - - // Shuffle for random sampling - for (let i = candidates.length - 1; i > 0; i--) { - const j = Math.floor(Math.random() * (i + 1)); - [candidates[i], candidates[j]] = [candidates[j]!, candidates[i]!]; - } - - return candidates.slice(0, size); -} - -function samplePosBucket( - records: AnnotatedRecord[], - exclude: Set, -): AnnotatedRecord[] { - const posList: SupportedPos[] = ["noun", "verb", "adjective", "adverb"]; - const perPos = Math.floor(BUCKET_SIZE / posList.length); - const result: AnnotatedRecord[] = []; - - for (const pos of posList) { - const sampled = sampleBucket( - records, - (r) => r.pos === pos, - perPos, - exclude, - ); - result.push(...sampled); - } - - return result; -} - -// ── Loading ─────────────────────────────────────────────────────────────────── - -async function loadAnnotated(): Promise { - // Load all language files and merge votes into a single record set. - // Use en.json as the base record structure since it has the most complete - // glosses and examples. Votes from all other languages are merged in. - const baseRaw = await fs.readFile( - path.join(PATHS.annotatedDir, "en.json"), - "utf-8", - ); - const base = JSON.parse(baseRaw) as AnnotatedRecord[]; - - // Build a map for fast lookup by source_id - const byId = new Map(); - for (const record of base) { - byId.set(record.source_id, record); - } - - // Merge votes from remaining language files - for (const lang of SUPPORTED_LANGUAGE_CODES) { - if (lang === "en") continue; - const raw = await fs.readFile( - path.join(PATHS.annotatedDir, `${lang}.json`), - "utf-8", - ); - const records = JSON.parse(raw) as AnnotatedRecord[]; - - for (const record of records) { - const base = byId.get(record.source_id); - if (!base) continue; - - // Merge votes - for (const [l, langVotes] of Object.entries(record.votes)) { - if (!base.votes[l as SupportedLanguageCode]) { - base.votes[l as SupportedLanguageCode] = {}; - } - Object.assign(base.votes[l as SupportedLanguageCode]!, langVotes); - } - - // Merge examples from CEFR source files not in base - for (const [l, examples] of Object.entries(record.examples)) { - const lang = l as SupportedLanguageCode; - if (!base.examples[lang]) { - base.examples[lang] = examples; - } - } - } - } - - return [...byId.values()]; -} - -// ── Main ───────────────────────────────────────────────────────────────────── - -async function main(): Promise { - console.log("Loading annotated files..."); - const records = await loadAnnotated(); - console.log(` Loaded ${records.length.toLocaleString()} synsets`); - - const sampled: SampleRecord[] = []; - const seen = new Set(); - - // Sample each bucket except pos_spread - for (const bucket of BUCKETS.filter((b) => b.name !== "pos_spread")) { - const results = sampleBucket(records, bucket.predicate, BUCKET_SIZE, seen); - for (const r of results) { - seen.add(r.source_id); - sampled.push({ ...r, _sample_bucket: bucket.name }); - } - console.log(` ${bucket.name}: ${results.length} records`); - } - - // Sample pos_spread bucket - const posResults = samplePosBucket(records, seen); - for (const r of posResults) { - seen.add(r.source_id); - sampled.push({ ...r, _sample_bucket: "pos_spread" }); - } - console.log(` pos_spread: ${posResults.length} records`); - - console.log(`\nTotal sampled: ${sampled.length} records`); - - // Write output - await fs.mkdir(path.dirname(PATHS.output), { recursive: true }); - await fs.writeFile(PATHS.output, JSON.stringify(sampled, null, 2), "utf-8"); - console.log(`Wrote sample → ${PATHS.output}`); -} - -main().catch((err) => { - console.error(err); - process.exit(1); -}); diff --git a/data-pipeline/stage-1-extract/scripts/extract.ts b/data-pipeline/stage-1-extract/scripts/extract.ts deleted file mode 100644 index 22defc2..0000000 --- a/data-pipeline/stage-1-extract/scripts/extract.ts +++ /dev/null @@ -1,257 +0,0 @@ -import fs from "node:fs"; -import path from "node:path"; -import readline from "node:readline"; -import { fileURLToPath } from "node:url"; -import { SUPPORTED_LANGUAGE_CODES } from "@lila/shared"; -import type { SupportedLanguageCode, SupportedPos } from "@lila/shared"; - -// ── Types ───────────────────────────────────────────────────────────────────── - -type KaikkiTranslation = { - code?: string; - lang_code?: string; - word?: string; - sense?: string; -}; - -type KaikkiSense = { - glosses?: string[]; - examples?: { text?: string }[]; - translations?: KaikkiTranslation[]; -}; - -type KaikkiEntry = { - word?: string; - pos?: string; - lang_code?: string; - senses?: KaikkiSense[]; -}; - -export type ExtractedSense = { - headword: string; - language: SupportedLanguageCode; - pos: SupportedPos; - sense_index: number; - gloss: string | null; - examples: string[]; - translations: { - target_lang: SupportedLanguageCode; - word: string; - sense_hint: string | null; - }[]; -}; - -// ── Constants ───────────────────────────────────────────────────────────────── - -const __dirname = path.dirname(fileURLToPath(import.meta.url)); - -const SOURCES_DIR = path.resolve(__dirname, "../sources"); -const OUTPUT_DIR = path.resolve(__dirname, "../output"); - -const LANG_TO_FILE: Record = { - en: "kaikki.org-dictionary-English.jsonl", - de: "kaikki.org-dictionary-German.jsonl", - it: "kaikki.org-dictionary-Italian.jsonl", - fr: "kaikki.org-dictionary-French.jsonl", - es: "kaikki.org-dictionary-Spanish.jsonl", -}; - -const POS_MAP: Record = { - noun: "noun", - verb: "verb", - adj: "adjective", - adv: "adverb", -}; - -const SUPPORTED_LANG_SET = new Set(SUPPORTED_LANGUAGE_CODES); - -// ── Helpers ─────────────────────────────────────────────────────────────────── - -function mapPos(kaikkiPos: string): SupportedPos | null { - return POS_MAP[kaikkiPos] ?? null; -} - -function isAbbreviation(gloss: string): boolean { - return gloss.toLowerCase().startsWith("abbreviation of"); -} - -function extractTranslations( - sense: KaikkiSense, - sourceLang: SupportedLanguageCode, -): ExtractedSense["translations"] { - const seen = new Set(); - const result: ExtractedSense["translations"] = []; - - for (const t of sense.translations ?? []) { - const code = t.code ?? t.lang_code; - if (!code || !SUPPORTED_LANG_SET.has(code)) continue; - if (code === sourceLang) continue; // skip same-language translations - if (!t.word?.trim()) continue; - - const key = `${code}:${t.word.trim()}`; - if (seen.has(key)) continue; - seen.add(key); - - result.push({ - target_lang: code as SupportedLanguageCode, - word: t.word.trim(), - sense_hint: t.sense?.trim() ?? null, - }); - } - - return result; -} - -function extractExamples(sense: KaikkiSense): string[] { - return (sense.examples ?? []) - .map((e) => e.text?.trim()) - .filter((t): t is string => !!t); -} - -function processEntry( - entry: KaikkiEntry, - sourceLang: SupportedLanguageCode, -): Omit[] { - const pos = mapPos(entry.pos ?? ""); - if (!pos) return []; - if (!entry.word?.trim()) return []; - - // For non-English files, only process entries in the target language - const entryLang = (entry as Record)["lang_code"] as - | string - | undefined; - if (sourceLang !== "en" && entryLang !== sourceLang) return []; - - const headword = entry.word.trim(); - const results: Omit[] = []; - - for (const sense of entry.senses ?? []) { - const gloss = sense.glosses?.[0]?.trim() ?? null; - - if (gloss && isAbbreviation(gloss)) continue; - - if (sourceLang === "en") { - // English: require translations in supported languages - const translations = extractTranslations(sense, sourceLang); - if (translations.length === 0) continue; - results.push({ - headword, - language: sourceLang, - pos, - gloss, - examples: extractExamples(sense), - translations, - }); - } else { - // Non-English: just extract the entry, no translations needed - results.push({ - headword, - language: sourceLang, - pos, - gloss, - examples: extractExamples(sense), - translations: [], - }); - } - } - - return results; -} - -// ── Extract ─────────────────────────────────────────────────────────────────── - -export async function extract( - lang: SupportedLanguageCode, - sampleLimit?: number, -): Promise { - const filename = LANG_TO_FILE[lang]; - const sourcePath = path.join(SOURCES_DIR, filename); - const outputPath = path.join(OUTPUT_DIR, `${lang}.json`); - - console.log(`\nExtracting ${lang}...`); - console.log(` Source: ${sourcePath}`); - if (sampleLimit) console.log(` Sample mode: ${sampleLimit} entries`); - - await fs.promises.mkdir(OUTPUT_DIR, { recursive: true }); - - const fileStream = fs.createReadStream(sourcePath); - const rl = readline.createInterface({ - input: fileStream, - crlfDelay: Infinity, - }); - - const senses: ExtractedSense[] = []; - const senseIndexMap = new Map(); - let linesRead = 0; - let entriesProcessed = 0; - let entriesSkipped = 0; - - for await (const line of rl) { - if (!line.trim()) continue; - if (sampleLimit && entriesProcessed >= sampleLimit) break; - - linesRead++; - - let entry: KaikkiEntry; - try { - entry = JSON.parse(line) as KaikkiEntry; - } catch { - console.warn(` Warning: failed to parse line ${linesRead}, skipping`); - continue; - } - - const extracted = processEntry(entry, lang); - - if (extracted.length === 0) { - entriesSkipped++; - continue; - } - - for (const sense of extracted) { - const key = `${sense.headword}|${sense.pos}`; - const senseIndex = senseIndexMap.get(key) ?? 0; - senseIndexMap.set(key, senseIndex + 1); - senses.push({ ...sense, sense_index: senseIndex }); - } - - entriesProcessed++; - - if (entriesProcessed % 10_000 === 0) { - console.log( - ` Processed ${entriesProcessed.toLocaleString()} entries...`, - ); - } - } - - await fs.promises.writeFile( - outputPath, - JSON.stringify(senses, null, 2), - "utf-8", - ); - - console.log(` Lines read: ${linesRead.toLocaleString()}`); - console.log(` Entries processed: ${entriesProcessed.toLocaleString()}`); - console.log(` Entries skipped: ${entriesSkipped.toLocaleString()}`); - console.log(` Senses extracted: ${senses.length.toLocaleString()}`); - console.log(` Output: ${outputPath}`); -} - -// ── Main ───────────────────────────────────────────────────────────────────── - -async function main(): Promise { - // Hardcoded sample limit for development — remove for full extraction - const SAMPLE = 500; - - for (const lang of SUPPORTED_LANGUAGE_CODES) { - await extract(lang, SAMPLE); - } - - console.log("\nExtraction complete."); -} - -if (import.meta.url === `file://${process.argv[1]}`) { - main().catch((err) => { - console.error(err); - process.exit(1); - }); -} diff --git a/data-pipeline/stage-2-reverse-link/scripts/reverse-link.ts b/data-pipeline/stage-2-reverse-link/scripts/reverse-link.ts deleted file mode 100644 index da8c9b6..0000000 --- a/data-pipeline/stage-2-reverse-link/scripts/reverse-link.ts +++ /dev/null @@ -1,109 +0,0 @@ -import { openDb } from "../../db/index.js"; - -// ── Types ───────────────────────────────────────────────────────────────────── - -type TranslationRow = { - translation_id: number; - entry_id: number; - entry_language: string; - entry_headword: string; - target_lang: string; - word: string; - sense_hint: string | null; -}; - -type EntryRow = { id: number }; - -// ── Sync ────────────────────────────────────────────────────────────────────── - -export function reverseLink(): void { - const db = openDb(); - - // Find all translations and their source entry details - const translations = db - .prepare( - `SELECT - t.id AS translation_id, - t.entry_id, - e.language AS entry_language, - e.headword AS entry_headword, - t.target_lang, - t.word, - t.sense_hint - FROM translations t - JOIN entries e ON e.id = t.entry_id`, - ) - .all() as TranslationRow[]; - - console.log( - ` Found ${translations.length.toLocaleString()} translations to check`, - ); - - const findEntry = db.prepare( - `SELECT id FROM entries WHERE headword = ? AND language = ? LIMIT 1`, - ); - - const insertReverseLink = db.prepare( - `INSERT INTO translations (entry_id, target_lang, word, sense_hint, source) - VALUES (?, ?, ?, ?, 'reverse_link') - ON CONFLICT (entry_id, target_lang, word) DO NOTHING`, - ); - - const sync = db.transaction(() => { - let inserted = 0; - let skipped = 0; - let noEntry = 0; - - for (const t of translations) { - // Look for an entry in the target language with the translation word as headword - const targetEntry = findEntry.get(t.word, t.target_lang) as - | EntryRow - | undefined; - - if (!targetEntry) { - noEntry++; - continue; - } - - // Insert reverse link: target entry → source language → source headword - const result = insertReverseLink.run( - targetEntry.id, - t.entry_language, - t.entry_headword, - t.sense_hint ?? null, - ); - - if (result.changes > 0) { - inserted++; - } else { - skipped++; - } - } - - return { inserted, skipped, noEntry }; - }); - - const counts = sync(); - - db.close(); - - console.log(` Inserted: ${counts.inserted.toLocaleString()} reverse links`); - console.log( - ` Skipped: ${counts.skipped.toLocaleString()} (already existed)`, - ); - console.log( - ` No entry: ${counts.noEntry.toLocaleString()} (target word not in entries)`, - ); -} - -// ── Main ───────────────────────────────────────────────────────────────────── - -function main(): void { - console.log("Running reverse link sync..."); - reverseLink(); - console.log("\nReverse link sync complete."); -} - -if (import.meta.url === `file://${process.argv[1]}`) { - main(); -} diff --git a/data-pipeline/stage-3-enrich/config.ts b/data-pipeline/stage-3-enrich/config.ts deleted file mode 100644 index 99a2f35..0000000 --- a/data-pipeline/stage-3-enrich/config.ts +++ /dev/null @@ -1,123 +0,0 @@ -// ── Provider configuration ──────────────────────────────────────────────────── -// -// Each provider + model combination counts as one vote in the final majority. -// Running the same model twice is not supported — one model, one vote. -// The `name` field is used as the model identifier in pipeline.db and must -// be unique across all runs. -// -// The pipeline iterates through ALL_PROVIDERS in order, skipping models that -// have already completed a full run and resuming models with partial progress. -// -// See llm-setup.md for full setup instructions and model recommendations. - -export type ProviderConfig = { - name: string; // unique model identifier — stored in pipeline.db - baseURL: string; - apiKey: string; - model: string; - maxTokens: number; -}; - -// ── Local llama.cpp ─────────────────────────────────────────────────────────── - -export const LOCAL_QWEN35_4B: ProviderConfig = { - name: "local-qwen3.5-4b", - baseURL: "http://127.0.0.1:8080/v1", - apiKey: "none", - model: "qwen3.5-4b", - maxTokens: 1024, // no reasoning overhead so 1024 is enough -}; - -export const LOCAL_GEMMA4: ProviderConfig = { - name: "local-gemma4-e4b", - baseURL: "http://127.0.0.1:8080/v1", - apiKey: "none", // llama.cpp ignores this - model: "gemma4-e4b", // llama.cpp ignores model name, uses loaded model - maxTokens: 2048, -}; - -export const LOCAL_QWEN7B: ProviderConfig = { - name: "local-qwen2.5-7b", - baseURL: "http://127.0.0.1:8080/v1", - apiKey: "none", - model: "qwen2.5-7b", - maxTokens: 512, -}; - -// ── OpenRouter — free tier ──────────────────────────────────────────────────── - -export const OR_QWEN3_480B: ProviderConfig = { - name: "or-qwen3-480b", - baseURL: "https://openrouter.ai/api/v1", - apiKey: process.env["OPENROUTER_API_KEY"] ?? "", - model: "qwen/qwen3-coder:free", - maxTokens: 512, -}; - -export const OR_GEMMA4_31B: ProviderConfig = { - name: "or-gemma4-31b", - baseURL: "https://openrouter.ai/api/v1", - apiKey: process.env["OPENROUTER_API_KEY"] ?? "", - model: "google/gemma-4-31b-it:free", - maxTokens: 512, -}; - -export const OR_QWEN3_80B: ProviderConfig = { - name: "or-qwen3-80b", - baseURL: "https://openrouter.ai/api/v1", - apiKey: process.env["OPENROUTER_API_KEY"] ?? "", - model: "qwen/qwen3-next-80b-a3b-instruct:free", - maxTokens: 512, -}; - -export const OR_NEMOTRON: ProviderConfig = { - name: "or-nemotron-120b", - baseURL: "https://openrouter.ai/api/v1", - apiKey: process.env["OPENROUTER_API_KEY"] ?? "", - model: "nvidia/nemotron-3-super-120b-a12b:free", - maxTokens: 512, -}; - -// ── Anthropic — reference baseline ─────────────────────────────────────────── -// Note: Anthropic uses a different API format. An adapter is required. -// See llm-setup.md for details. - -export const ANTHROPIC_SONNET: ProviderConfig = { - name: "anthropic-sonnet-4", - baseURL: "https://api.anthropic.com/v1", - apiKey: process.env["ANTHROPIC_API_KEY"] ?? "", - model: "claude-sonnet-4-6", - maxTokens: 512, -}; - -// ── All configured providers ────────────────────────────────────────────────── -// The pipeline runs through these in order — local models first, then cloud. -// Add new providers here to include them in the voting pool. - -export const ALL_PROVIDERS: ProviderConfig[] = [ - LOCAL_QWEN35_4B, - // LOCAL_GEMMA4, - // LOCAL_QWEN7B, - // OR_QWEN3_480B, - // OR_GEMMA4_31B, - // OR_QWEN3_80B, - // OR_NEMOTRON, - // ANTHROPIC_SONNET, -]; - -// ── Key validation ──────────────────────────────────────────────────────────── - -const LOCAL_PROVIDERS = new Set(["none"]); - -export function validateProviderKey(provider: ProviderConfig): void { - if (LOCAL_PROVIDERS.has(provider.apiKey)) return; - - if (!provider.apiKey) { - const keyName = provider.name.startsWith("anthropic") - ? "ANTHROPIC_API_KEY" - : "OPENROUTER_API_KEY"; - console.error(`\n ERROR: ${keyName} is not set in .env`); - console.error(` Provider "${provider.name}" requires this key to run.\n`); - process.exit(1); - } -} diff --git a/data-pipeline/stage-3-enrich/scripts/enrich.ts b/data-pipeline/stage-3-enrich/scripts/enrich.ts deleted file mode 100644 index e732e0a..0000000 --- a/data-pipeline/stage-3-enrich/scripts/enrich.ts +++ /dev/null @@ -1,877 +0,0 @@ -import { openDb } from "../../db/index.js"; -import type { ProviderConfig } from "../config.js"; -import { CEFR_LEVELS, SUPPORTED_LANGUAGE_CODES } from "@lila/shared"; -import type { SupportedLanguageCode } from "@lila/shared"; - -// ── Types ───────────────────────────────────────────────────────────────────── - -type EntryRow = { - id: number; - headword: string; - language: SupportedLanguageCode; - pos: string; - gloss: string | null; - examples: string; // JSON array string -}; - -type TranslationRow = { - id: number; - target_lang: SupportedLanguageCode; - word: string; -}; - -type GlossResult = { status: "ok" } | { status: "improved"; gloss: string }; - -type ExampleResult = { status: "ok" } | { status: "improved"; example: string }; - -type TranslationResult = { - translations: Partial< - Record> - >; - generated?: Partial>; -}; - -type CefrResult = { - headword_cefr: string; - translation_cefr: Partial< - Record> - >; -}; - -type SubStage = - | "round1_gloss" - | "round1_example" - | "round1_translations" - | "round1_cefr"; - -// ── Constants ───────────────────────────────────────────────────────────────── - -const SUPPORTED_LANG_SET = new Set(SUPPORTED_LANGUAGE_CODES); -const CEFR_SET = new Set(CEFR_LEVELS); - -// ── Shutdown ────────────────────────────────────────────────────────────────── - -let shutdownRequested = false; -let currentCallController: AbortController | null = null; - -export function registerEnrichShutdown(): void { - const handler = (): void => { - if (shutdownRequested) return; - shutdownRequested = true; - console.log("\n\n Shutdown requested — aborting current LLM call..."); - currentCallController?.abort(); - }; - process.on("SIGINT", handler); - process.on("SIGTERM", handler); -} - -// ── Prompt builders ─────────────────────────────────────────────────────────── - -function buildGlossPrompt(entry: EntryRow): string { - const glossText = entry.gloss ?? "none"; - const examples: string[] = JSON.parse(entry.examples) as string[]; - const examplesText = - examples.length > 0 ? examples.map((e) => ` - ${e}`).join("\n") : " none"; - - return `You are a language learning expert. - -Review this gloss for the ${entry.pos} "${entry.headword}" (sense ${entry.sense_index}). -Gloss: "${glossText}" -Examples of this specific sense: -${examplesText} - -Is this gloss clear, accurate for this specific sense, and suitable for a language learner? -- If yes, respond with: {"status": "ok"} -- If no or if gloss is "none", respond with: {"status": "improved", "gloss": "your improved gloss here"} - -IMPORTANT: Your improved gloss must describe THIS SPECIFIC SENSE shown by the examples above, -not a more common or general meaning of the word. - -Respond ONLY with valid JSON and nothing else.`; -} - -function buildTranslationsPrompt( - entry: EntryRow, - translations: TranslationRow[], - verifiedGloss: string, -): string { - const byLang = new Map(); - for (const t of translations) { - if (!byLang.has(t.target_lang)) byLang.set(t.target_lang, []); - byLang.get(t.target_lang)!.push(t.word); - } - - const coveredLangs = new Set(byLang.keys()); - const missingLangs = SUPPORTED_LANGUAGE_CODES.filter( - (l) => l !== entry.language && !coveredLangs.has(l), - ); - - const translationsText = - byLang.size > 0 - ? [...byLang.entries()] - .map(([lang, words]) => ` ${lang}: ${words.join(", ")}`) - .join("\n") - : " none"; - - const missingText = - missingLangs.length > 0 ? missingLangs.join(", ") : "none"; - - const exampleResponse: Record = { - translations: { - de: { frei: "ok", "-frei": "reject" }, - it: { libero: "ok", free: "reject" }, - }, - }; - if (missingLangs.length > 0) { - exampleResponse["generated"] = { es: "libre", fr: "libre" }; - } - - return `You are a language learning expert. - -For the ${entry.language} ${entry.pos} "${entry.headword}" (meaning: "${verifiedGloss}"), review these translations: -${translationsText} - -For each translation: -- Write "ok" if it is a valid translation for this specific meaning -- Write "reject" if it is wrong, a suffix (starts with -), garbled text, or the wrong language - -Examples of correct behaviour: -- "free" listed as Italian → "reject" (it is English, not Italian) -- "-frei" listed as German → "reject" (it is a suffix, not a standalone word) -- "libre" listed as Spanish → "ok" (it is a valid Spanish word) - -${missingLangs.length > 0 ? `Also generate the single best translation for these missing languages: ${missingText}` : ""} - -Respond ONLY with valid JSON and nothing else: -${JSON.stringify(exampleResponse, null, 2)}`; -} - -function buildCefrPrompt( - entry: EntryRow, - verifiedGloss: string, - validatedTranslations: Map, -): string { - const translationsText = - validatedTranslations.size > 0 - ? [...validatedTranslations.entries()] - .map(([lang, words]) => ` ${lang}: ${words.join(", ")}`) - .join("\n") - : " none"; - - return `You are a language learning expert. - -Assign CEFR levels (A1, A2, B1, B2, C1, or C2) to this word and its validated translations. -Base your levels on how commonly a language learner at that level would encounter this specific sense. -Consider register — slang, technical, and archaic words should be rated higher. - -WORD: ${entry.headword} (${entry.pos}) -MEANING: ${verifiedGloss} -VALIDATED TRANSLATIONS: -${translationsText} - -Respond ONLY with valid JSON and nothing else: -{ - "headword_cefr": "B1", - "translation_cefr": { - "de": { "frei": "A2" }, - "it": { "libero": "A2" } - } -}`; -} - -// ── Validation ──────────────────────────────────────────────────────────────── - -function validateGloss(raw: string): GlossResult | null { - try { - const obj = JSON.parse(raw) as Record; - if (obj["status"] === "ok") return { status: "ok" }; - if ( - obj["status"] === "improved" && - typeof obj["gloss"] === "string" && - obj["gloss"].trim() - ) { - return { status: "improved", gloss: obj["gloss"].trim() }; - } - return null; - } catch { - return null; - } -} - -function validateExample(raw: string): ExampleResult | null { - try { - const obj = JSON.parse(raw) as Record; - if (obj["status"] === "ok") return { status: "ok" }; - if ( - obj["status"] === "improved" && - typeof obj["example"] === "string" && - obj["example"].trim() - ) { - return { status: "improved", example: obj["example"].trim() }; - } - return null; - } catch { - return null; - } -} - -function validateTranslations( - raw: string, - translations: TranslationRow[], -): TranslationResult | null { - try { - const obj = JSON.parse(raw) as Record; - if (typeof obj["translations"] !== "object" || obj["translations"] === null) - return null; - - const result: TranslationResult = { translations: {} }; - const translationsObj = obj["translations"] as Record; - - // Validate each language's votes - for (const [lang, votes] of Object.entries(translationsObj)) { - if (!SUPPORTED_LANG_SET.has(lang)) continue; - if (typeof votes !== "object" || votes === null) continue; - - result.translations[lang as SupportedLanguageCode] = {}; - for (const [word, status] of Object.entries( - votes as Record, - )) { - if (status === "ok" || status === "reject") { - result.translations[lang as SupportedLanguageCode]![word] = status; - } - } - } - - // Validate generated translations - if (obj["generated"] !== undefined && obj["generated"] !== null) { - if (typeof obj["generated"] !== "object") return null; - result.generated = {}; - for (const [lang, word] of Object.entries( - obj["generated"] as Record, - )) { - if (!SUPPORTED_LANG_SET.has(lang)) continue; - if (typeof word === "string" && word.trim()) { - result.generated[lang as SupportedLanguageCode] = word.trim(); - } - } - } - - // Check all translations got a vote - const byLang = new Map>(); - for (const t of translations) { - if (!byLang.has(t.target_lang)) byLang.set(t.target_lang, new Set()); - byLang.get(t.target_lang)!.add(t.word); - } - - for (const [lang, words] of byLang.entries()) { - const votes = result.translations[lang as SupportedLanguageCode]; - if (!votes) return null; - for (const word of words) { - if (!votes[word]) return null; - } - } - - return result; - } catch { - return null; - } -} - -function validateCefr( - raw: string, - validatedTranslations: Map, -): CefrResult | null { - try { - const obj = JSON.parse(raw) as Record; - if (typeof obj["headword_cefr"] !== "string") return null; - if (!CEFR_SET.has(obj["headword_cefr"])) return null; - if ( - typeof obj["translation_cefr"] !== "object" || - obj["translation_cefr"] === null - ) - return null; - - const translationCefr = obj["translation_cefr"] as Record; - - // Verify all validated translations have a CEFR vote - for (const [lang, words] of validatedTranslations.entries()) { - const votes = translationCefr[lang] as Record | undefined; - if (!votes) return null; - for (const word of words) { - if (!votes[word] || !CEFR_SET.has(votes[word])) return null; - } - } - - return { - headword_cefr: obj["headword_cefr"], - translation_cefr: translationCefr as Partial< - Record> - >, - }; - } catch { - return null; - } -} - -// ── LLM call ────────────────────────────────────────────────────────────────── - -async function callLlm( - prompt: string, - provider: ProviderConfig, -): Promise { - currentCallController = new AbortController(); - const timeout = setTimeout(() => currentCallController?.abort(), 120_000); - - let response: Response; - try { - response = await fetch(`${provider.baseURL}/chat/completions`, { - method: "POST", - signal: currentCallController.signal, - headers: { - "Content-Type": "application/json", - Authorization: `Bearer ${provider.apiKey}`, - }, - body: JSON.stringify({ - model: provider.model, - max_tokens: provider.maxTokens, - messages: [{ role: "user", content: prompt }], - temperature: 0.1, - }), - }); - } finally { - clearTimeout(timeout); - currentCallController = null; - } - - if (!response.ok) { - throw new Error(`LLM API error: ${response.status} ${response.statusText}`); - } - - const data = (await response.json()) as { - choices?: { message?: { content?: string } }[]; - }; - - const content = data.choices?.[0]?.message?.content; - if (!content) throw new Error("LLM returned empty response"); - - return content - .replace(/```json\n?/g, "") - .replace(/```\n?/g, "") - .trim(); -} - -// ── Status helpers ──────────────────────────────────────────────────────────── - -function getSubStageStatus( - entryId: number, - modelName: string, - stage: SubStage, -): "complete" | "needs_review" | "pending" { - const db = openDb(); - const row = db - .prepare( - `SELECT status FROM run_status - WHERE entry_id = ? AND model_name = ? AND stage = ?`, - ) - .get(entryId, modelName, stage) as { status: string } | undefined; - db.close(); - if (!row) return "pending"; - if (row.status === "complete") return "complete"; - if (row.status === "needs_review") return "needs_review"; - return "pending"; -} - -function markSubStage( - entryId: number, - modelName: string, - stage: SubStage, - status: "complete" | "needs_review", -): void { - const db = openDb(); - db.prepare( - `INSERT INTO run_status (entry_id, model_name, stage, status) - VALUES (?, ?, ?, ?) - ON CONFLICT (entry_id, model_name, stage) - DO UPDATE SET status = ?, updated_at = datetime('now')`, - ).run(entryId, modelName, stage, status, status); - db.close(); -} - -// ── Write helpers ───────────────────────────────────────────────────────────── - -function writeGloss( - entryId: number, - modelName: string, - result: GlossResult, -): void { - if (result.status === "improved") { - const db = openDb(); - db.prepare( - `INSERT INTO generated_glosses (entry_id, model_name, text) - VALUES (?, ?, ?) - ON CONFLICT (entry_id, model_name) DO NOTHING`, - ).run(entryId, modelName, result.gloss); - db.close(); - } -} - -function writeExample( - entryId: number, - modelName: string, - result: ExampleResult, -): void { - if (result.status === "improved") { - const db = openDb(); - db.prepare( - `INSERT INTO generated_examples (entry_id, model_name, text) - VALUES (?, ?, ?) - ON CONFLICT (entry_id, model_name) DO NOTHING`, - ).run(entryId, modelName, result.example); - db.close(); - } -} - -function writeTranslations( - entryId: number, - modelName: string, - result: TranslationResult, - translations: TranslationRow[], -): void { - const db = openDb(); - - db.transaction(() => { - // Write rejections - for (const t of translations) { - const vote = result.translations[t.target_lang]?.[t.word]; - if (vote === "reject") { - db.prepare( - `INSERT INTO model_translation_rejections (translation_id, model_name) - VALUES (?, ?) - ON CONFLICT (translation_id, model_name) DO NOTHING`, - ).run(t.id, modelName); - } - } - - // Write generated translations - if (result.generated) { - for (const [lang, word] of Object.entries(result.generated)) { - db.prepare( - `INSERT INTO generated_translations (entry_id, model_name, target_lang, word) - VALUES (?, ?, ?, ?) - ON CONFLICT (entry_id, model_name, target_lang) DO NOTHING`, - ).run(entryId, modelName, lang, word); - } - } - })(); - - db.close(); -} - -function writeCefr( - entryId: number, - modelName: string, - result: CefrResult, - translations: TranslationRow[], -): void { - const db = openDb(); - - db.transaction(() => { - // Headword CEFR - db.prepare( - `INSERT INTO model_entry_cefr_votes (entry_id, model_name, cefr_level) - VALUES (?, ?, ?) - ON CONFLICT (entry_id, model_name) DO NOTHING`, - ).run(entryId, modelName, result.headword_cefr); - - // Translation CEFR votes - for (const t of translations) { - const level = result.translation_cefr[t.target_lang]?.[t.word]; - if (level && CEFR_SET.has(level)) { - db.prepare( - `INSERT INTO model_translation_cefr_votes (translation_id, model_name, cefr_level) - VALUES (?, ?, ?) - ON CONFLICT (translation_id, model_name) DO NOTHING`, - ).run(t.id, modelName, level); - } - } - })(); - - db.close(); -} - -// ── Progress ────────────────────────────────────────────────────────────────── - -function updateProgress( - processed: number, - needsReview: number, - total: number, - llmMs: number, - startTime: number, -): void { - const totalProcessed = processed + needsReview; - const pct = ((totalProcessed / total) * 100).toFixed(1); - const elapsed = (Date.now() - startTime) / 1000; - const rate = elapsed > 0 ? totalProcessed / elapsed : 0; - const remaining = rate > 0 ? (total - totalProcessed) / rate : 0; - const eta = - remaining === 0 - ? "calculating..." - : remaining < 60 - ? `${Math.round(remaining)}s` - : `${Math.round(remaining / 60)}m`; - const totalElapsedStr = - elapsed < 60 - ? `${Math.round(elapsed)}s` - : `${Math.floor(elapsed / 60)}m ${Math.round(elapsed % 60)}s`; - - process.stdout.write( - `\r ${totalProcessed}/${total} (${pct}%) — entry: ${(llmMs / 1000).toFixed(1)}s — total: ${totalElapsedStr} — ETA: ${eta} `, - ); -} - -// ── Main enrich function ────────────────────────────────────────────────────── - -export async function enrich( - provider: ProviderConfig, -): Promise<{ processed: number; skipped: number; needsReview: number }> { - registerEnrichShutdown(); - const db = openDb(); - - const allEntries = db - .prepare(`SELECT * FROM entries WHERE language = 'en'`) - .all() as EntryRow[]; - - // An entry is fully complete when all 4 sub-stages are complete - const completeEntries = db - .prepare( - `SELECT entry_id FROM run_status - WHERE model_name = ? AND stage = 'round1_gloss' - AND status = 'complete'`, - ) - .all(provider.name) as { entry_id: number }[]; - - const completeIds = new Set(completeEntries.map((r) => r.entry_id)); - const pending = allEntries.filter((e) => !completeIds.has(e.id)).slice(0, 50); - - db.close(); - - console.log(`\n Model: ${provider.name}`); - console.log(` Total entries: ${allEntries.length.toLocaleString()}`); - console.log(` Already complete: ${completeIds.size.toLocaleString()}`); - console.log(` Pending: ${pending.length.toLocaleString()}`); - - if (pending.length === 0) { - console.log(" Nothing to process."); - return { processed: 0, skipped: completeIds.size, needsReview: 0 }; - } - - let processedCount = 0; - let needsReviewCount = 0; - let llmMs = 0; - const startTime = Date.now(); - - for (const entry of pending) { - if (shutdownRequested) break; - - const db2 = openDb(); - const translations = db2 - .prepare( - `SELECT id, target_lang, word FROM translations WHERE entry_id = ? AND source = 'kaikki'`, - ) - .all(entry.id) as TranslationRow[]; - db2.close(); - - let entryFailed = false; - - // ── Sub-stage 1: Gloss ──────────────────────────────────────────────────── - - let verifiedGloss = entry.gloss ?? ""; - - if ( - getSubStageStatus(entry.id, provider.name, "round1_gloss") !== "complete" - ) { - try { - const llmStart = Date.now(); - const raw = await callLlm(buildGlossPrompt(entry), provider); - llmMs = Date.now() - llmStart; - - const result = validateGloss(raw); - if (!result) { - markSubStage(entry.id, provider.name, "round1_gloss", "needs_review"); - console.warn( - `\n needs_review: entry ${entry.id} round1_gloss — invalid response`, - ); - entryFailed = true; - } else { - writeGloss(entry.id, provider.name, result); - if (result.status === "improved") verifiedGloss = result.gloss; - markSubStage(entry.id, provider.name, "round1_gloss", "complete"); - } - } catch (err) { - llmMs = 0; - const message = err instanceof Error ? err.message : String(err); - markSubStage(entry.id, provider.name, "round1_gloss", "needs_review"); - console.warn( - `\n needs_review: entry ${entry.id} round1_gloss — ${message}`, - ); - entryFailed = true; - } - } - - if (entryFailed) { - needsReviewCount++; - updateProgress( - processedCount, - needsReviewCount, - pending.length, - llmMs, - startTime, - ); - continue; - } - - /* - // ── Sub-stages 2, 3, 4 — not yet active ────────────────────────────────── - // ── Sub-stage 2: Example ────────────────────────────────────────────────── - - if ( - getSubStageStatus(entry.id, provider.name, "round1_example") !== - "complete" - ) { - try { - const llmStart = Date.now(); - const raw = await callLlm( - buildExamplePrompt(entry, verifiedGloss), - provider, - ); - llmMs = Date.now() - llmStart; - - const result = validateExample(raw); - if (!result) { - markSubStage( - entry.id, - provider.name, - "round1_example", - "needs_review", - ); - console.warn( - `\n needs_review: entry ${entry.id} round1_example — invalid response`, - ); - entryFailed = true; - } else { - writeExample(entry.id, provider.name, result); - markSubStage(entry.id, provider.name, "round1_example", "complete"); - } - } catch (err) { - llmMs = 0; - const message = err instanceof Error ? err.message : String(err); - markSubStage(entry.id, provider.name, "round1_example", "needs_review"); - console.warn( - `\n needs_review: entry ${entry.id} round1_example — ${message}`, - ); - entryFailed = true; - } - } - - if (entryFailed) { - needsReviewCount++; - updateProgress( - processedCount, - needsReviewCount, - pending.length, - llmMs, - startTime, - ); - continue; - } - - // ── Sub-stage 3: Translations ───────────────────────────────────────────── - - const validatedTranslations = new Map(); - - if ( - getSubStageStatus(entry.id, provider.name, "round1_translations") !== - "complete" - ) { - try { - const llmStart = Date.now(); - const raw = await callLlm( - buildTranslationsPrompt(entry, translations, verifiedGloss), - provider, - ); - llmMs = Date.now() - llmStart; - - const result = validateTranslations(raw, translations); - if (!result) { - markSubStage( - entry.id, - provider.name, - "round1_translations", - "needs_review", - ); - console.warn( - `\n needs_review: entry ${entry.id} round1_translations — invalid response`, - ); - entryFailed = true; - } else { - writeTranslations(entry.id, provider.name, result, translations); - markSubStage( - entry.id, - provider.name, - "round1_translations", - "complete", - ); - - // Build validated translations map for CEFR sub-stage - // Include kaikki translations that were ok'd + generated translations - for (const t of translations) { - const vote = result.translations[t.target_lang]?.[t.word]; - if (vote === "ok") { - if (!validatedTranslations.has(t.target_lang)) { - validatedTranslations.set(t.target_lang, []); - } - validatedTranslations.get(t.target_lang)!.push(t.word); - } - } - if (result.generated) { - for (const [lang, word] of Object.entries(result.generated)) { - const l = lang as SupportedLanguageCode; - if (!validatedTranslations.has(l)) - validatedTranslations.set(l, []); - validatedTranslations.get(l)!.push(word); - } - } - } - } catch (err) { - llmMs = 0; - const message = err instanceof Error ? err.message : String(err); - markSubStage( - entry.id, - provider.name, - "round1_translations", - "needs_review", - ); - console.warn( - `\n needs_review: entry ${entry.id} round1_translations — ${message}`, - ); - entryFailed = true; - } - } else { - // Already complete — rebuild validated translations from db - const db3 = openDb(); - const rejections = new Set( - ( - db3 - .prepare( - `SELECT translation_id FROM model_translation_rejections WHERE model_name = ?`, - ) - .all(provider.name) as { translation_id: number }[] - ).map((r) => r.translation_id), - ); - for (const t of translations) { - if (!rejections.has(t.id)) { - if (!validatedTranslations.has(t.target_lang)) { - validatedTranslations.set(t.target_lang, []); - } - validatedTranslations.get(t.target_lang)!.push(t.word); - } - } - const generated = db3 - .prepare( - `SELECT target_lang, word FROM generated_translations WHERE entry_id = ? AND model_name = ?`, - ) - .all(entry.id, provider.name) as { - target_lang: SupportedLanguageCode; - word: string; - }[]; - for (const g of generated) { - if (!validatedTranslations.has(g.target_lang)) - validatedTranslations.set(g.target_lang, []); - validatedTranslations.get(g.target_lang)!.push(g.word); - } - db3.close(); - } - - if (entryFailed) { - needsReviewCount++; - updateProgress( - processedCount, - needsReviewCount, - pending.length, - llmMs, - startTime, - ); - continue; - } - - // ── Sub-stage 4: CEFR ───────────────────────────────────────────────────── - - if ( - getSubStageStatus(entry.id, provider.name, "round1_cefr") !== "complete" - ) { - try { - const llmStart = Date.now(); - const raw = await callLlm( - buildCefrPrompt(entry, verifiedGloss, validatedTranslations), - provider, - ); - llmMs = Date.now() - llmStart; - - const result = validateCefr(raw, validatedTranslations); - if (!result) { - markSubStage(entry.id, provider.name, "round1_cefr", "needs_review"); - console.warn( - `\n needs_review: entry ${entry.id} round1_cefr — invalid response`, - ); - needsReviewCount++; - } else { - // Get translation rows for validated words only - const validatedRows = translations.filter((t) => { - return validatedTranslations.get(t.target_lang)?.includes(t.word); - }); - writeCefr(entry.id, provider.name, result, validatedRows); - markSubStage(entry.id, provider.name, "round1_cefr", "complete"); - processedCount++; - } - } catch (err) { - llmMs = 0; - const message = err instanceof Error ? err.message : String(err); - markSubStage(entry.id, provider.name, "round1_cefr", "needs_review"); - console.warn( - `\n needs_review: entry ${entry.id} round1_cefr — ${message}`, - ); - needsReviewCount++; - } - } else { - processedCount++; - } - - */ - - processedCount++; - updateProgress( - processedCount, - needsReviewCount, - pending.length, - llmMs, - startTime, - ); - } - - process.stdout.write("\n"); - const totalMs = Date.now() - startTime; - const totalMin = Math.floor(totalMs / 60_000); - const totalSec = Math.round((totalMs % 60_000) / 1000); - console.log(` Total time: ${totalMin}m ${totalSec}s`); - console.log( - ` Avg per entry: ${(totalMs / Math.max(processedCount + needsReviewCount, 1) / 1000).toFixed(1)}s`, - ); - console.log(` Processed: ${processedCount.toLocaleString()}`); - console.log(` Needs review: ${needsReviewCount.toLocaleString()}`); - - return { - processed: processedCount, - skipped: completeIds.size, - needsReview: needsReviewCount, - }; -} diff --git a/data-pipeline/tests/validation/db-import.validation.test.ts b/data-pipeline/tests/validation/db-import.validation.test.ts deleted file mode 100644 index f58f35e..0000000 --- a/data-pipeline/tests/validation/db-import.validation.test.ts +++ /dev/null @@ -1,230 +0,0 @@ -import fs from "node:fs/promises"; -import path from "node:path"; -import { describe, it, expect, beforeAll } from "vitest"; -import { SUPPORTED_LANGUAGE_CODES } from "@lila/shared"; -import type { SupportedLanguageCode, SupportedPos } from "@lila/shared"; - -// ── Types ───────────────────────────────────────────────────────────────────── - -type ExtractedSense = { - headword: string; - language: SupportedLanguageCode; - pos: SupportedPos; - sense_index: number; - gloss: string | null; - examples: string[]; - translations: { - target_lang: SupportedLanguageCode; - word: string; - sense_hint: string | null; - }[]; -}; - -// ── Paths ───────────────────────────────────────────────────────────────────── - -const DB_PATH = path.resolve("db/pipeline.db"); -const OUTPUT_DIR = path.resolve("stage-1-extract/output"); - -// ── Helpers ─────────────────────────────────────────────────────────────────── - -async function dbExists(): Promise { - try { - await fs.access(DB_PATH); - return true; - } catch { - return false; - } -} - -// ── Tests ───────────────────────────────────────────────────────────────────── - -describe("pipeline.db — import validation", () => { - let db: import("better-sqlite3").Database; - let expectedEntriesByLang: Map; - let expectedTotalTranslations: number; - - beforeAll(async () => { - if (!(await dbExists())) return; - - const Database = (await import("better-sqlite3")).default; - db = new Database(DB_PATH, { readonly: true }); - db.pragma("foreign_keys = ON"); - - expectedEntriesByLang = new Map(); - expectedTotalTranslations = 0; - - for (const lang of SUPPORTED_LANGUAGE_CODES) { - try { - const raw = await fs.readFile( - path.join(OUTPUT_DIR, `${lang}.json`), - "utf-8", - ); - const senses = JSON.parse(raw) as ExtractedSense[]; - expectedEntriesByLang.set(lang, senses.length); - if (lang === "en") { - for (const sense of senses) { - expectedTotalTranslations += sense.translations.length; - } - } - } catch { - expectedEntriesByLang.set(lang, 0); - } - } - }, 30_000); - - it("pipeline.db exists — skipping all tests if not", async () => { - const exists = await dbExists(); - if (!exists) { - console.warn( - "\n pipeline.db not found — run pnpm db:init and pnpm db:import first\n", - ); - } - expect(exists).toBe(true); - }); - - it("entry count per language matches source files", () => { - if (!db) return; - const errors: string[] = []; - - for (const lang of SUPPORTED_LANGUAGE_CODES) { - const expected = expectedEntriesByLang.get(lang) ?? 0; - const row = db - .prepare("SELECT COUNT(*) as count FROM entries WHERE language = ?") - .get(lang) as { count: number }; - - if (row.count !== expected) { - errors.push(`${lang}: expected ${expected} entries, got ${row.count}`); - } - } - - expect(errors, `\n${errors.join("\n")}`).toHaveLength(0); - }); - - it("translation count matches source files plus reverse links", () => { - if (!db) return; - const row = db - .prepare("SELECT COUNT(*) as count FROM translations") - .get() as { count: number }; - const reverseLinks = db - .prepare( - "SELECT COUNT(*) as count FROM translations WHERE source = 'reverse_link'", - ) - .get() as { count: number }; - expect(row.count).toBe(expectedTotalTranslations + reverseLinks.count); - }); - - it("every translation references a valid entry", () => { - if (!db) return; - const rows = db - .prepare( - `SELECT t.id, t.entry_id - FROM translations t - LEFT JOIN entries e ON e.id = t.entry_id - WHERE e.id IS NULL`, - ) - .all() as { id: number; entry_id: number }[]; - - const errors = rows.map( - (r) => `translation ${r.id}: references missing entry ${r.entry_id}`, - ); - expect(errors, `\n${errors.join("\n")}`).toHaveLength(0); - }); - - it("every entry has a valid language code", () => { - if (!db) return; - const validLangs = SUPPORTED_LANGUAGE_CODES.map((l) => `'${l}'`).join(", "); - const rows = db - .prepare( - `SELECT id, headword, language FROM entries - WHERE language NOT IN (${validLangs})`, - ) - .all() as { id: number; headword: string; language: string }[]; - - const errors = rows.map( - (r) => `entry ${r.id} "${r.headword}": invalid language "${r.language}"`, - ); - expect(errors, `\n${errors.join("\n")}`).toHaveLength(0); - }); - - it("every entry has a valid pos", () => { - if (!db) return; - const rows = db - .prepare( - `SELECT id, headword, pos FROM entries - WHERE pos NOT IN ('noun', 'verb', 'adjective', 'adverb')`, - ) - .all() as { id: number; headword: string; pos: string }[]; - - const errors = rows.map( - (r) => `entry ${r.id} "${r.headword}": invalid pos "${r.pos}"`, - ); - expect(errors, `\n${errors.join("\n")}`).toHaveLength(0); - }); - - it("sense_index is unique per headword, language, pos", () => { - if (!db) return; - const rows = db - .prepare( - `SELECT headword, language, pos, sense_index, COUNT(*) as c - FROM entries - GROUP BY headword, language, pos, sense_index - HAVING c > 1`, - ) - .all() as { - headword: string; - language: string; - pos: string; - sense_index: number; - c: number; - }[]; - - const errors = rows.map( - (r) => - `"${r.headword}" (${r.language} ${r.pos}): duplicate sense_index ${r.sense_index} (${r.c} rows)`, - ); - expect(errors, `\n${errors.join("\n")}`).toHaveLength(0); - }); - - it("non-English entries have no Kaikki translations", () => { - if (!db) return; - const nonEnLangs = SUPPORTED_LANGUAGE_CODES.filter((l) => l !== "en") - .map((l) => `'${l}'`) - .join(", "); - - const rows = db - .prepare( - `SELECT e.headword, e.language, COUNT(t.id) as c - FROM entries e - JOIN translations t ON t.entry_id = e.id - WHERE e.language IN (${nonEnLangs}) - AND t.source = 'kaikki' - GROUP BY e.id`, - ) - .all() as { headword: string; language: string; c: number }[]; - - const errors = rows.map( - (r) => - `"${r.headword}" (${r.language}): unexpected ${r.c} Kaikki translations`, - ); - expect(errors, `\n${errors.join("\n")}`).toHaveLength(0); - }); - - it("all Kaikki translation target languages are supported and not English", () => { - if (!db) return; - const validLangs = SUPPORTED_LANGUAGE_CODES.map((l) => `'${l}'`).join(", "); - - const rows = db - .prepare( - `SELECT t.id, t.target_lang - FROM translations t - WHERE t.source = 'kaikki' - AND (t.target_lang NOT IN (${validLangs}) OR t.target_lang = 'en')`, - ) - .all() as { id: number; target_lang: string }[]; - - const errors = rows.map( - (r) => `translation ${r.id}: invalid target_lang "${r.target_lang}"`, - ); - expect(errors, `\n${errors.join("\n")}`).toHaveLength(0); - }); -}); diff --git a/data-pipeline/tests/validation/stage-1.validation.test.ts b/data-pipeline/tests/validation/stage-1.validation.test.ts deleted file mode 100644 index 86edac5..0000000 --- a/data-pipeline/tests/validation/stage-1.validation.test.ts +++ /dev/null @@ -1,192 +0,0 @@ -import fs from "node:fs/promises"; -import path from "node:path"; -import { describe, it, expect, beforeAll } from "vitest"; -import { SUPPORTED_LANGUAGE_CODES, SUPPORTED_POS } from "@lila/shared"; -import type { SupportedLanguageCode, SupportedPos } from "@lila/shared"; - -// ── Types ───────────────────────────────────────────────────────────────────── - -type ExtractedSense = { - headword: string; - language: SupportedLanguageCode; - pos: SupportedPos; - sense_index: number; - gloss: string | null; - examples: string[]; - translations: { - target_lang: SupportedLanguageCode; - word: string; - sense_hint: string | null; - }[]; -}; - -// ── Paths ───────────────────────────────────────────────────────────────────── - -const OUTPUT_DIR = path.resolve("stage-1-extract/output"); - -// ── Tests ───────────────────────────────────────────────────────────────────── - -describe("stage 1 — Kaikki extraction output validation", () => { - const sensesByLang = new Map(); - - beforeAll(async () => { - for (const lang of SUPPORTED_LANGUAGE_CODES) { - const filePath = path.join(OUTPUT_DIR, `${lang}.json`); - const raw = await fs.readFile(filePath, "utf-8"); - sensesByLang.set(lang, JSON.parse(raw) as ExtractedSense[]); - } - }, 30_000); - - it("all five language output files exist", async () => { - const errors: string[] = []; - for (const lang of SUPPORTED_LANGUAGE_CODES) { - try { - await fs.access(path.join(OUTPUT_DIR, `${lang}.json`)); - } catch { - errors.push(`missing: ${lang}.json`); - } - } - expect(errors, `\n${errors.join("\n")}`).toHaveLength(0); - }); - - it("every language file is a non-empty array", () => { - const errors: string[] = []; - for (const lang of SUPPORTED_LANGUAGE_CODES) { - const senses = sensesByLang.get(lang)!; - if (!Array.isArray(senses)) errors.push(`${lang}: not an array`); - else if (senses.length === 0) errors.push(`${lang}: empty array`); - } - expect(errors, `\n${errors.join("\n")}`).toHaveLength(0); - }); - - it("every sense has required fields", () => { - const errors: string[] = []; - for (const lang of SUPPORTED_LANGUAGE_CODES) { - for (const sense of sensesByLang.get(lang)!) { - if (!sense.headword) errors.push(`${lang}: sense missing headword`); - if (!sense.language) - errors.push(`${lang} ${sense.headword}: missing language`); - if (!sense.pos) errors.push(`${lang} ${sense.headword}: missing pos`); - if (sense.sense_index === undefined) - errors.push(`${lang} ${sense.headword}: missing sense_index`); - if (!Array.isArray(sense.examples)) - errors.push(`${lang} ${sense.headword}: examples not an array`); - if (!Array.isArray(sense.translations)) - errors.push(`${lang} ${sense.headword}: translations not an array`); - } - } - expect(errors, `\n${errors.join("\n")}`).toHaveLength(0); - }); - - it("every sense has a valid pos", () => { - const errors: string[] = []; - const validPos = new Set(SUPPORTED_POS); - for (const lang of SUPPORTED_LANGUAGE_CODES) { - for (const sense of sensesByLang.get(lang)!) { - if (!validPos.has(sense.pos)) { - errors.push(`${lang} ${sense.headword}: invalid pos "${sense.pos}"`); - } - } - } - expect(errors, `\n${errors.join("\n")}`).toHaveLength(0); - }); - - it("every sense language code matches its file", () => { - const errors: string[] = []; - for (const lang of SUPPORTED_LANGUAGE_CODES) { - for (const sense of sensesByLang.get(lang)!) { - if (sense.language !== lang) { - errors.push( - `${lang} ${sense.headword}: language field "${sense.language}" does not match file`, - ); - } - } - } - expect(errors, `\n${errors.join("\n")}`).toHaveLength(0); - }); - - it("no abbreviation senses in output", () => { - const errors: string[] = []; - for (const lang of SUPPORTED_LANGUAGE_CODES) { - for (const sense of sensesByLang.get(lang)!) { - if (sense.gloss?.toLowerCase().startsWith("abbreviation of")) { - errors.push( - `${lang} ${sense.headword}: abbreviation sense not filtered`, - ); - } - } - } - expect(errors, `\n${errors.join("\n")}`).toHaveLength(0); - }); - - it("English senses all have at least one translation", () => { - const errors: string[] = []; - for (const sense of sensesByLang.get("en")!) { - if (sense.translations.length === 0) { - errors.push( - `en ${sense.headword} (sense ${sense.sense_index}): no translations`, - ); - } - } - expect(errors, `\n${errors.join("\n")}`).toHaveLength(0); - }); - - it("non-English senses have no translations", () => { - const errors: string[] = []; - for (const lang of SUPPORTED_LANGUAGE_CODES) { - if (lang === "en") continue; - for (const sense of sensesByLang.get(lang)!) { - if (sense.translations.length > 0) { - errors.push( - `${lang} ${sense.headword}: unexpected translations in non-English file`, - ); - } - } - } - expect(errors, `\n${errors.join("\n")}`).toHaveLength(0); - }); - - it("all translation target languages are supported and not English", () => { - const errors: string[] = []; - const validLangs = new Set(SUPPORTED_LANGUAGE_CODES); - for (const sense of sensesByLang.get("en")!) { - for (const t of sense.translations) { - if (!validLangs.has(t.target_lang)) { - errors.push( - `en ${sense.headword}: unsupported translation language "${t.target_lang}"`, - ); - } - if (t.target_lang === "en") { - errors.push( - `en ${sense.headword}: translation to same language "en"`, - ); - } - if (!t.word?.trim()) { - errors.push( - `en ${sense.headword}: empty translation word for ${t.target_lang}`, - ); - } - } - } - expect(errors, `\n${errors.join("\n")}`).toHaveLength(0); - }); - - it("sense_index is unique per headword and pos within each language", () => { - const errors: string[] = []; - for (const lang of SUPPORTED_LANGUAGE_CODES) { - const seen = new Map>(); - for (const sense of sensesByLang.get(lang)!) { - const key = `${sense.headword}|${sense.pos}`; - if (!seen.has(key)) seen.set(key, new Set()); - const indexes = seen.get(key)!; - if (indexes.has(sense.sense_index)) { - errors.push( - `${lang} ${sense.headword} (${sense.pos}): duplicate sense_index ${sense.sense_index}`, - ); - } - indexes.add(sense.sense_index); - } - } - expect(errors, `\n${errors.join("\n")}`).toHaveLength(0); - }); -}); diff --git a/documentation/pipeline/ENGLISH_NOUNS.md b/documentation/pipeline/ENGLISH_NOUNS.md new file mode 100644 index 0000000..72fa88a --- /dev/null +++ b/documentation/pipeline/ENGLISH_NOUNS.md @@ -0,0 +1,342 @@ +# english nouns + +## step 1a + +get freqency source list + verify lemmatization + confirm language/POS tagging + +example: + +house +bank +asdf + +## step 1b + +transform list into JSON (headword, language, POS) + +example: + +```json +{ + "headword": "house", + "language": "en", + "pos": "noun" +}, +{ + "headword": "bank", + "language": "en", + "pos": "noun" +}, +{ + "headword": "asdf", + "language": "en", + "pos": "noun" +} +``` + +--- + +## step 2a + +check existence in kaikki (eg. is there a english noun "asdf" in kaikki, if not put it separate list) + +example output: + +```json +{ + "headword": "house", + "language": "en", + "pos": "noun" +}, +{ + "headword": "bank", + "language": "en", + "pos": "noun" +} +``` + +miss list (triage queue, not trash! contains junk, slipped inflections, and real Kaikki gaps, gets handled separately, never auto-drop!) + +```json +{ "headword": "asdf", "language": "en", "pos": "noun" } +``` + +## step 2b + +for each sense, extract: glosses, translations (with gender), examples +id will be generated via: `headword:lang:pos:sense_index` + +words without glosses will be dropped! (put in special list) + +example output: + +```json +{ + "id": "house:en:noun:0", + "headword": "house", + "language": "en", + "pos": "noun", + "glosses": ["A building for human habitation."], + "examples": ["They bought a house in the city."], + "translations": { + "de": [{ "word": "Haus", "gender": "neuter" }], + "it": [{ "word": "casa", "gender": "feminine" }], + "es": [{ "word": "casa", "gender": "feminine" }], + "fr": [{ "word": "maison", "gender": "feminine" }] + } +}, +{ + "id": "house:en:noun:1", + "headword": "house", + "language": "en", + "pos": "noun", + "glosses": ["A noble family or lineage."], + "examples": ["The House of Tudor ruled England."], + "translations": { + "de": [ + { "word": "Adelsgeschlecht", "gender": "neuter" }, + { "word": "Haus", "gender": "neuter" } + ] + } +}, +{ + "id": "bank:en:noun:0", + "headword": "bank", + "language": "en", + "pos": "noun", + "glosses": ["An institution where one can place and borrow money."], + "examples": ["She deposited her paycheck at the bank."], + "translations": { + "de": [{ "word": "Bank", "gender": "feminine" }], + "it": [{ "word": "banca", "gender": "feminine" }], + "es": [{ "word": "banco", "gender": "masculine" }], + "fr": [{ "word": "banque", "gender": "feminine" }] + } +}, +{ + "id": "bank:en:noun:1", + "headword": "bank", + "language": "en", + "pos": "noun", + "glosses": ["The land alongside a river or lake."], + "examples": ["They picnicked on the bank of the river."], + "translations": { + "de": [{ "word": "Ufer", "gender": "neuter" }], + "it": [{ "word": "riva", "gender": "feminine" }], + "es": [{ "word": "orilla", "gender": "feminine" }], + "fr": [{ "word": "rive", "gender": "feminine" }] + } +}, +{ + "id": "bank:en:noun:2", + "headword": "bank", + "language": "en", + "pos": "noun", + "glosses": ["A collection or store of something held in reserve."], + "examples": ["The hospital keeps a blood bank."], + "translations": { + "de": [{ "word": "Bank", "gender": "feminine" }] + } +} +``` + +## step 2c + +fill gaps Kaikki left (LLM, 3 models, generate-then-vote) + +takes 2b's partial cards. 3 different-family models (qwen, llama, gemma) generate, then vote. + +separate focused sub-passes, one field at a time — never combined: + +- **translations** → per missing language, generated with the gloss as context. generate → vote. no agreement → gap stays (cloud audit later) +- **examples** → for senses with no Kaikki example. generate → verify-vote ("is this a valid example of the gloss?"), since freeform sentences never exact-match. no agreement → no example (card still valid) no tiebreak: no agreement leaves the gap, never escalates to more models. vote records stored in pipeline.db for the cloud audit. +- **invariant**: the gloss is never LLM-generated — translations are filled, examples generated, difficulty graded, but the gloss must always be Kaikki's (no gloss → sense dropped in 2b) + +example input: + +```json +{ + "id": "harbor:en:noun:0", + "headword": "harbor", + "language": "en", + "pos": "noun", + "glosses": ["A sheltered area of water where ships can dock safely."], + "examples": [], + "translations": { "de": [{ "word": "Hafen", "gender": "masculine" }] } +} +``` + +next substage is the translation generation: + +```json +{ + "id": "harbor:en:noun:0", + "headword": "harbor", + "language": "en", + "pos": "noun", + "glosses": ["A sheltered area of water where ships can dock safely."], + "examples": [], + "translations": { + "de": [{ "word": "Hafen", "gender": "masculine" }], + "it": [{ "word": "porto", "gender": null }], + "es": [{ "word": "puerto", "gender": null }], + "fr": [{ "word": "port", "gender": null }] + } +} +``` + +next substage is the gender modification (by looking up kaikki for it/es/fr or corresponding wiktionary, fill only nulls, not touching existing genders): +**runs only after all genders are present** + +```json +{ + "id": "harbor:en:noun:0", + "headword": "harbor", + "language": "en", + "pos": "noun", + "glosses": ["A sheltered area of water where ships can dock safely."], + "examples": [], + "translations": { + "de": [{ "word": "Hafen", "gender": "masculine" }], + "it": [{ "word": "porto", "gender": "masculine" }], + "es": [{ "word": "puerto", "gender": "masculine" }], + "fr": [{ "word": "port", "gender": "masculine" }] + } +} +``` + +next substage is the example generation: + +```json +{ + "id": "harbor:en:noun:0", + "headword": "harbor", + "language": "en", + "pos": "noun", + "glosses": ["A sheltered area of water where ships can dock safely."], + "examples": ["The fishing boats returned to the harbor at dusk."], + "translations": { + "de": [{ "word": "Hafen", "gender": "masculine" }], + "it": [{ "word": "porto", "gender": "masculine" }], + "es": [{ "word": "puerto", "gender": "masculine" }], + "fr": [{ "word": "port", "gender": "masculine" }] + } +} +``` + +--- + +## step 3 + +adding difficulty level + +CEFR is mapped to three buckets: +A1/A2 → easy +B1/B2 → intermediate +C1/C2 → hard + +depending on number of senses per headword: + +- One sense → derive difficulty from the CEFRLex distribution (first CEFR level crossing a frequency threshold, mapped to a bucket, not the peak), for the four covered languages (en/de/es/fr). Deterministic, no LLM +- Multiple senses → the three local LLMs grade each sense (easy/intermediate/hard) from the gloss. CEFRLex not involved. +- No CEFRLex entry at all (Italian, or single-sense word that's missing) → LLMs grade from gloss + +to get coverage (not quality), 3 local llms are going to be used: gemma, qwen and llama + +on the llm votes: + +- **Majority agrees (3-0 or 2-1)** → ship the majority value. +- **3-way split (all three differ)** → no consensus → exclude the card to the review queue. Not shipped. + +example output: + +```json +{ + "id": "house:en:noun:0", + "headword": "house", + "language": "en", + "pos": "noun", + "difficulty_level": "easy", + "glosses": ["A building for human habitation."], + "examples": ["They bought a house in the city."], + "translations": { + "de": [{ "word": "Haus", "gender": "neuter" }], + "it": [{ "word": "casa", "gender": "feminine" }], + "es": [{ "word": "casa", "gender": "feminine" }], + "fr": [{ "word": "maison", "gender": "feminine" }] + } +}, +{ + "id": "house:en:noun:1", + "headword": "house", + "language": "en", + "pos": "noun", + "difficulty_level": "hard", + "glosses": ["A noble family or lineage."], + "examples": ["The House of Tudor ruled England."], + "translations": { + "de": [ + { "word": "Adelsgeschlecht", "gender": "neuter" }, + { "word": "Haus", "gender": "neuter" } + ] + } +} +``` + +**important**: `difficulty_level` is mandatory on every shipped card (the user sets difficulty before a game) + +This is coverage, not final quality: three local models make every shipped card's difficulty present and plausible, but not guaranteed correct + +no cefr list: needs to be graded by online llms (see notes) + +--- + +## coverage report (trial deliverable) + +- not a data stage => reads the finished cards and summarises them +- the trial (english + italian nouns, local models) runs steps 1a→3 then emits a report + +### pipeline health — did it run? + +- input lemmas → output cards (ratio; multi-sense makes cards > lemmas) +- dropped per stage + why: 2a misses (junk/inflection/gap), 2b no-gloss drops, 3 no-consensus exclusions +- parse failures / errors + +### coverage — how complete? + +- translation completeness: all 4 langs vs gaps, per language +- examples: from Kaikki vs LLM-generated vs none +- gender: translations still null after Wiktionary fill +- difficulty distribution per bucket (lopsided = broken CEFRLex threshold or bad grading) + +### local good enough? — model agreement + +- difficulty votes: 3-0 / 2-1 / 3-way split rates (high 3-way = locals can't do it → need cloud) +- translation gap-fill: agreement rate, gaps left unfilled +- → go/no-go on local-for-launch + +### rent vs API? — workload volume + +- total LLM calls across 2c + 3 (× per-token rate = API cost) +- cards needing LLM vs handled deterministically (english: high deterministic; italian: ~0, no CEFRLex) +- per-language call counts → scale english (low) and italian (high) to estimate the middle three + +### review backlog — what's deferred + +- miss-list size + composition, per language +- no-consensus exclusions (cloud-audit queue size) +- unfilled translation gaps (also cloud-audit work) + +**framing**: english = optimistic floor (rich Kaikki, CEFRLex exists, models strongest). +italian = pessimistic ceiling (no CEFRLex, thinner Kaikki). all five languages sit between. +read to decide, not to archive. + +--- + +Notes: + +- the miss list needs to get verified/re-worked later on +- the kaikki data contains ipas and links to audio files, add them later if needed, they are not needed now" +- use online llms to set the difficulty(cefr) of the not shipped words +- add plurals from kaikki/wiktionary +- postgres sync to prod db diff --git a/documentation/pipeline/TRIAL_IMPLEMENTATION_ROADMAP.md b/documentation/pipeline/TRIAL_IMPLEMENTATION_ROADMAP.md new file mode 100644 index 0000000..6f0c7d2 --- /dev/null +++ b/documentation/pipeline/TRIAL_IMPLEMENTATION_ROADMAP.md @@ -0,0 +1,41 @@ +# trial run — implementation roadmap (pure vertical) + +goal: english + italian nouns through the full pipeline on local models, emit +coverage report. build a thin end-to-end slice first, then widen each pass. + +## phase 0 — foundation + +- confirm @lila/shared types (lang codes, POS) exist and match +- package.json deps for what phase 1 needs +- one hardcoded test word to carry through the slice (e.g. "house") + +## phase 1 — thin vertical slice (ONE word, 1a→3, crudest possible) + +goal: prove a single card can travel the whole pipeline and come out the end. +allowed to be ugly — hardcode, skip voting, one model, fake CEFRLex. + +- 1a: one lemma record, by hand or trivial read +- 2a: look it up in Kaikki, confirm it exists +- 2b: extract its senses → card(s) with gloss +- 2c: fill one missing translation with ONE local model, no voting +- 3: assign a difficulty crudely (even hardcoded "easy") +- OUT: one finished card. the pipeline has a shape. + +## phase 2 — widen: real deterministic front (1a–2b, all words) + +- 1a: real frequency list, verified +- 2a: real existence gate + miss-list triage +- 2b: real sense extraction, gender from tags, gloss-required drop +- still JSON output. inspect cards by hand. + +## phase 3 — widen: real LLM back (2c + 3) + +- storage seam: JSON → SQLite, schema, resumability +- 2c: real generate→vote, three families, all sub-passes +- 3: real CEFRLex single-sense + 3-model vote multi-sense + exclude-on-split + +## phase 4 — coverage report + runs + +- emit report +- run english, then italian +- decide: local good enough? rent vs API?