diff --git a/README.md b/README.md index 6421048..043d717 100644 --- a/README.md +++ b/README.md @@ -99,9 +99,3 @@ lila/ ├── documentation/ — Project docs (this directory) └── Caddyfile, docker-compose.yml, etc. ``` - ---- - -## License - -TBD diff --git a/data-pipeline/config/batch.ts b/data-pipeline/config/batch.ts new file mode 100644 index 0000000..289fe55 --- /dev/null +++ b/data-pipeline/config/batch.ts @@ -0,0 +1,3 @@ +export const BATCH_CONFIG = { size: 4, maxRetries: 3 } as const; + +//1, 2, 5, 10, 20 diff --git a/data-pipeline/config/llm.ts b/data-pipeline/config/llm.ts new file mode 100644 index 0000000..8b416da --- /dev/null +++ b/data-pipeline/config/llm.ts @@ -0,0 +1,5 @@ +export const LLM_CONFIG = { + provider: "local" as "local" | "openrouter" | "deepseek" | "gemini", + url: "http://127.0.0.1:8080/v1/chat/completions", + model: undefined as string | undefined, +} as const; diff --git a/data-pipeline/config/prompt.ts b/data-pipeline/config/prompt.ts new file mode 100644 index 0000000..060e650 --- /dev/null +++ b/data-pipeline/config/prompt.ts @@ -0,0 +1,34 @@ +export const ENRICHMENT_SYSTEM_PROMPT = `You are a multilingual dictionary engine. Output ONLY a JSON object. No markdown, no explanations. + +For each English noun provided, generate 1-2 distinct senses. + +CEFR difficulty mapping: +- A1/A2 → easy +- B1/B2 → medium +- C1/C2 → hard + +Each sense must have: +- sense: student-friendly definition, max 15 words +- example: natural sentence using the word +- difficulty_level: easy, medium, or hard +- translations: object with keys de, it, es, fr; each value is an array of {word, gender} where gender MUST be masculine, feminine, or neuter. Use null ONLY if the language has no grammatical gender for that word. + +Output format: JSON object where keys are the input words, values are arrays of sense objects. + +Example for ["house"]: +{ + "house": [ + { + "sense": "A building for human habitation.", + "example": "They bought a house in the city.", + "difficulty_level": "easy", + "translations": { + "de": [{"word": "Haus", "gender": "neuter"}], + "it": [{"word": "casa", "gender": "feminine"}], + "es": [{"word": "casa", "gender": "feminine"}], + "fr": [{"word": "maison", "gender": "feminine"}] + } + } + ] +} +`; diff --git a/data-pipeline/pipeline.ts b/data-pipeline/pipeline.ts new file mode 100644 index 0000000..863c4af --- /dev/null +++ b/data-pipeline/pipeline.ts @@ -0,0 +1,160 @@ +import { isWordProcessed } from "./utils/check-if-json-exists.js"; +import { createBaseJson } from "./utils/create-base-json.js"; +import { ensureOutputFolders } from "./utils/create-output-dirs.js"; +import { scanSourceData } from "./utils/scanning-source-files.js"; +import { createLineReader } from "./utils/create-line-reader.js"; +import { checkLlmServer } from "./utils/check-llm-server.js"; +import { getWordFilePath } from "./utils/get-word-file-path.js"; +import { mergeEnrichedData } from "./utils/merge-enriched-data.js"; +import { enrichWordWithRetry } from "./utils/enrich-word.js"; +import { writeJsonFile } from "./utils/write-json-file.js"; +import { deleteFileIfExists } from "./utils/delete-file.js"; +import { PipelineTimer } from "./utils/pipeline-timer.js"; +import { ProgressTracker } from "./utils/progress-tracker.js"; +import { verifyEnrichedFile } from "./utils/verify-enriched-file.js"; +import { BATCH_CONFIG } from "./config/batch.js"; + +async function main() { + console.log("Starting data pipeline...\n"); + + const timer = new PipelineTimer(); + + // step 1: scanning for source files + console.log("\n step 1: scanning the source files..."); + const wordlists = scanSourceData(import.meta.dirname); + + // step 2: ensuring output folders exist + console.log("\n step 2: creating necessary output folders..."); + ensureOutputFolders(wordlists); + + // step 3: check to verify the AI engine is ready before touching anything + console.log("\n step 3: verifying local AI engine status..."); + await checkLlmServer(); + + // Step 4: Loop through the wordlists array + console.log("\n step 4: looping through the wordlists..."); + + for (const wordlist of wordlists) { + console.log( + `\nReading list: [${wordlist.language.toUpperCase()}] -> [${wordlist.pos.toUpperCase()}]`, + ); + + const rl = createLineReader(wordlist.sourcePath); + + // Collect words and count them + const words: string[] = []; + for await (const line of rl) { + const word = line.trim().toLowerCase(); + if (word) words.push(word); + } + + // Filter out already-processed words + const unprocessedWords = words.filter( + (word) => !isWordProcessed(word, wordlist.outputDir), + ); + + const skippedCount = words.length - unprocessedWords.length; + if (skippedCount > 0) { + console.log(` Skipped ${skippedCount} already-processed words`); + } + + const progress = new ProgressTracker(unprocessedWords.length); + + // Step 5: Process in batches + for (let i = 0; i < unprocessedWords.length; i += BATCH_CONFIG.size) { + const batch = unprocessedWords.slice(i, i + BATCH_CONFIG.size); + const batchNum = Math.floor(i / BATCH_CONFIG.size) + 1; + const totalBatches = Math.ceil( + unprocessedWords.length / BATCH_CONFIG.size, + ); + const batchLabel = `Batch ${batchNum}/${totalBatches}`; + + console.log(`\n ${batchLabel}: [${batch.join(", ")}]`); + + // Create skeletons for all words in batch + for (const word of batch) { + createBaseJson( + word, + wordlist.outputDir, + wordlist.language, + wordlist.pos, + ); + } + + timer.startWord(); + + try { + // Step 6: enrich batch with senses (with retry/split) + const result = await enrichWordWithRetry( + batch, + wordlist.language, + wordlist.pos, + ); + + // Step 7: write each word's result + for (const [word, senses] of result.results) { + progress.next(); + console.log( + ` ${progress.format(`Enriched and saved: ${word}.json`)}`, + ); + + const targetFilePath = getWordFilePath(word, wordlist.outputDir); + const enrichedData = mergeEnrichedData( + word, + wordlist.language, + wordlist.pos, + senses, + ); + + writeJsonFile(targetFilePath, enrichedData); + + // Verify the generated file + const verification = verifyEnrichedFile(targetFilePath); + if (!verification.valid) { + console.error(` Warning: Schema violations in ${word}.json:`); + for (const error of verification.errors) { + console.error(` - ${error}`); + } + } + + timer.recordProcessed({ + promptTokens: result.metrics.promptTokens / batch.length, + completionTokens: result.metrics.completionTokens / batch.length, + totalTokens: result.metrics.totalTokens / batch.length, + promptTimeMs: result.metrics.promptTimeMs / batch.length, + completionTimeMs: result.metrics.completionTimeMs / batch.length, + promptTokensPerSecond: result.metrics.promptTokensPerSecond, + completionTokensPerSecond: result.metrics.completionTokensPerSecond, + }); + } + + console.log(` ${timer.getWordTiming()}`); + } catch (error: unknown) { + const errorMessage = + error instanceof Error ? error.message : String(error); + console.error( + ` Failed to enrich batch [${batch.join(", ")}]: ${errorMessage}`, + ); + + // Cleanup: delete skeleton files for failed batch + for (const word of batch) { + const targetFilePath = getWordFilePath(word, wordlist.outputDir); + deleteFileIfExists(targetFilePath); + console.log(` Removed incomplete file: ${word}.json`); + progress.recordFailed(); + } + + timer.recordFailed(); + } + } + } + + timer.stop(); + console.log("\n" + timer.getSummary()); + console.log("\nGlobal data pipeline run completed successfully."); +} + +// Fire the orchestrator block +main().catch((err) => { + console.error("Critical unexpected pipeline failure:", err); +}); diff --git a/data-pipeline/source-data/english/nouns b/data-pipeline/source-data/english/nouns new file mode 100644 index 0000000..028aff6 --- /dev/null +++ b/data-pipeline/source-data/english/nouns @@ -0,0 +1,20 @@ +house +time +water +year +people +day +way +man +woman +child +work +life +world +hand +eye +book +friend +school +city +family diff --git a/data-pipeline/utils/check-if-json-exists.ts b/data-pipeline/utils/check-if-json-exists.ts new file mode 100644 index 0000000..00ab535 --- /dev/null +++ b/data-pipeline/utils/check-if-json-exists.ts @@ -0,0 +1,23 @@ +import fs from "fs"; +import path from "path"; + +/** + * Checks if a JSON file for the given word exists AND contains enriched data. + * Returns false for skeleton files (missing senses array). + */ +export function isWordProcessed(word: string, outputDir: string): boolean { + const targetFilePath = path.join(outputDir, `${word}.json`); + + if (!fs.existsSync(targetFilePath)) { + return false; + } + + try { + const content = fs.readFileSync(targetFilePath, "utf-8"); + const data = JSON.parse(content) as Record; + return Array.isArray(data["senses"]) && data["senses"].length > 0; + } catch (_error: unknown) { + // Corrupted file => treat as not processed + return false; + } +} diff --git a/data-pipeline/utils/check-llm-server.ts b/data-pipeline/utils/check-llm-server.ts new file mode 100644 index 0000000..b4127a0 --- /dev/null +++ b/data-pipeline/utils/check-llm-server.ts @@ -0,0 +1,43 @@ +/** + * Pings the local llama.cpp server to ensure it's up, running, and has a model loaded. + * If the server is offline or still loading, it terminates the pipeline gracefully. + */ +export async function checkLlmServer( + url = "http://127.0.0.1:8080/health", +): Promise { + try { + const response = await fetch(url); + + // llama.cpp returns a 503 status if the server is up but the model weights are still loading + if (response.status === 503) { + console.error( + "\n ⏳ Local AI engine is starting up, but the model is still loading into memory.", + ); + console.error( + "👉 Please wait a minute for the weights to load, then run the pipeline again.\n", + ); + process.exit(1); + } + + // Parse the JSON health response (expected: { status: "ok" }) + const data = (await response.json()) as { status?: string }; + + if (response.ok && data.status === "ok") { + console.log("🟢 Local AI engine is connected and ready for inference!"); + return; + } + + // Catch-all for unexpected active server responses + console.error( + `\n ❌ Unknown response from local AI engine health check (Status: ${response.status}).`, + ); + process.exit(1); + } catch (_error: unknown) { + console.error("\n ❌ Could not connect to the local AI engine."); + console.error(`🔗 Attempted endpoint: ${url}`); + console.error( + "👉 Make sure your './llama-server' command is actively running in another terminal tab!\n", + ); + process.exit(1); + } +} diff --git a/data-pipeline/utils/create-base-json.ts b/data-pipeline/utils/create-base-json.ts new file mode 100644 index 0000000..2d21ecc --- /dev/null +++ b/data-pipeline/utils/create-base-json.ts @@ -0,0 +1,41 @@ +import fs from "fs"; +import path from "path"; + +const LANG_MAP: Record = { + english: "en", + italian: "it", + german: "de", + french: "fr", + spanish: "es", +}; + +const POS_MAP: Record = { + nouns: "noun", + verbs: "verb", + adverbs: "adverb", + adjectives: "adjective", +}; + +/** + * Creates the base JSON file with word, language, and pos. + * No logging — the orchestrator handles all console output. + */ +export function createBaseJson( + word: string, + outputDir: string, + rawLanguage: string, + rawPos: string, +): void { + const targetFilePath = path.join(outputDir, `${word}.json`); + + const dbLanguage = LANG_MAP[rawLanguage] || rawLanguage; + const dbPos = POS_MAP[rawPos] || rawPos; + + const initialData = { word, language: dbLanguage, pos: dbPos }; + + fs.writeFileSync( + targetFilePath, + JSON.stringify(initialData, null, 2), + "utf-8", + ); +} diff --git a/data-pipeline/utils/create-line-reader.ts b/data-pipeline/utils/create-line-reader.ts new file mode 100644 index 0000000..c3b166a --- /dev/null +++ b/data-pipeline/utils/create-line-reader.ts @@ -0,0 +1,11 @@ +import fs from "fs"; +import readline from "readline"; + +/** + * Creates a line-by-line reader stream for a given file path. + */ +export function createLineReader(sourcePath: string): readline.Interface { + const fileStream = fs.createReadStream(sourcePath, "utf-8"); + + return readline.createInterface({ input: fileStream, crlfDelay: Infinity }); +} diff --git a/data-pipeline/utils/create-output-dirs.ts b/data-pipeline/utils/create-output-dirs.ts new file mode 100644 index 0000000..d86b0e7 --- /dev/null +++ b/data-pipeline/utils/create-output-dirs.ts @@ -0,0 +1,20 @@ +import fs from "fs"; +import type { Wordlist } from "./scanning-source-files.js"; + +/** + * Takes a list of scanned datasets and creates their output folders if missing. + */ +export function ensureOutputFolders(wordlists: Wordlist[]): void { + for (const wordlist of wordlists) { + if (!fs.existsSync(wordlist.outputDir)) { + fs.mkdirSync(wordlist.outputDir, { recursive: true }); + console.log( + `📁 Created target folder: worddata/${wordlist.language}/${wordlist.pos}`, + ); + } + } + + console.log( + "✅ All required output directories have been verified and created successfully.", + ); +} diff --git a/data-pipeline/utils/delete-file.ts b/data-pipeline/utils/delete-file.ts new file mode 100644 index 0000000..3c545ce --- /dev/null +++ b/data-pipeline/utils/delete-file.ts @@ -0,0 +1,10 @@ +import fs from "fs"; + +/** + * Deletes a file if it exists. Silently ignores missing files. + */ +export function deleteFileIfExists(filePath: string): void { + if (fs.existsSync(filePath)) { + fs.unlinkSync(filePath); + } +} diff --git a/data-pipeline/utils/enrich-word.ts b/data-pipeline/utils/enrich-word.ts new file mode 100644 index 0000000..d970ee3 --- /dev/null +++ b/data-pipeline/utils/enrich-word.ts @@ -0,0 +1,240 @@ +// utils/enrich-word.ts +import { ENRICHMENT_SYSTEM_PROMPT } from "../config/prompt.js"; +import { createAdapter } from "./llm-adapters/factory.js"; +import { BATCH_CONFIG } from "../config/batch.js"; +import type { Language, Pos, EnrichedSense } from "./merge-enriched-data.js"; + +const LANG_MAP: Record = { + english: "en", + italian: "it", + german: "de", + french: "fr", + spanish: "es", +}; + +const POS_MAP: Record = { + nouns: "noun", + verbs: "verb", + adverbs: "adverb", + adjectives: "adjective", +}; + +interface LlmResponse { + content: string; + promptTokens: number; + completionTokens: number; + totalTokens: number; + promptTimeMs: number; + completionTimeMs: number; + promptTokensPerSecond: number; + completionTokensPerSecond: number; +} + +export interface EnrichmentResult { + results: Map; + metrics: { + promptTokens: number; + completionTokens: number; + totalTokens: number; + promptTimeMs: number; + completionTimeMs: number; + promptTokensPerSecond: number; + completionTokensPerSecond: number; + }; +} + +/** + * Calls the local LLM with the enrichment prompt. + * Returns the response content and timing metrics. + */ +async function callLlm(words: string[]): Promise { + const adapter = createAdapter(); + return adapter.call(words, ENRICHMENT_SYSTEM_PROMPT); +} + +/** + * Strips markdown code blocks and extracts the JSON object from raw LLM output. + * Throws if no valid JSON object braces are found. + */ +function sanitizeLlmOutput(raw: string): string { + const cleaned = raw.replace(/```json\s*/g, "").replace(/```\s*$/g, ""); + const start = cleaned.indexOf("{"); + const end = cleaned.lastIndexOf("}"); + if (start === -1 || end === -1) { + throw new Error("No JSON object found in LLM output"); + } + return cleaned.slice(start, end + 1); +} + +/** + * Parses the LLM response string into a JavaScript object. + * Throws if the response is not valid JSON or not an object with expected keys. + */ +export function parseLlmResponse( + rawJson: string, + expectedWords: string[], +): Record { + let parsed: unknown; + + try { + const sanitized = sanitizeLlmOutput(rawJson); + parsed = JSON.parse(sanitized); + } catch (error: unknown) { + throw new Error(`Failed to parse LLM output as JSON: ${rawJson}`, { + cause: error, + }); + } + + if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) { + throw new Error("LLM output is not a JSON object"); + } + + const obj = parsed as Record; + + for (const word of expectedWords) { + if (!(word in obj)) { + throw new Error(`Missing key in LLM output: "${word}"`); + } + if (!Array.isArray(obj[word]) || (obj[word] as unknown[]).length === 0) { + throw new Error(`LLM output for "${word}" is not a non-empty array`); + } + } + + return obj; +} + +/** + * Takes parsed LLM output and builds final enriched objects with composite IDs. + */ +export function buildEnrichedData( + parsed: Record, + rawLanguage: string, + rawPos: string, +): Map { + const language = (LANG_MAP[rawLanguage] || rawLanguage) as Language; + const pos = (POS_MAP[rawPos] || rawPos) as Pos; + const results = new Map(); + + for (const [word, sensesArray] of Object.entries(parsed)) { + const senses = (sensesArray as unknown[]).map((item, index) => { + const sense = item as Omit< + EnrichedSense, + "id" | "word" | "language" | "pos" + >; + + return { + id: `${word}:${language}:${pos}:${index}`, + word, + language, + pos, + ...sense, + } as EnrichedSense; + }); + + results.set(word, senses); + } + + return results; +} + +/** + * Enriches a batch of words by calling the LLM, parsing the response, and building final data. + */ +export async function enrichWord( + words: string[], + rawLanguage: string, + rawPos: string, +): Promise { + const llmResponse = await callLlm(words); + const parsed = parseLlmResponse(llmResponse.content, words); + const results = buildEnrichedData(parsed, rawLanguage, rawPos); + + return { + results, + metrics: { + promptTokens: llmResponse.promptTokens, + completionTokens: llmResponse.completionTokens, + totalTokens: llmResponse.totalTokens, + promptTimeMs: llmResponse.promptTimeMs, + completionTimeMs: llmResponse.completionTimeMs, + promptTokensPerSecond: llmResponse.promptTokensPerSecond, + completionTokensPerSecond: llmResponse.completionTokensPerSecond, + }, + }; +} + +/** + * Enriches a batch of words with retry and split-on-failure logic. + * Retries up to BATCH_CONFIG.maxRetries times, then splits batch in half and retries each half. + * Continues splitting until batch size is 1, then throws if still failing. + */ +export async function enrichWordWithRetry( + words: string[], + rawLanguage: string, + rawPos: string, + attempt: number = 1, +): Promise { + try { + return await enrichWord(words, rawLanguage, rawPos); + } catch (error: unknown) { + const errorMessage = error instanceof Error ? error.message : String(error); + + if (words.length === 1) { + throw new Error( + `Failed to enrich word "${words[0]}" after ${attempt} attempts: ${errorMessage}`, + { cause: error }, + ); + } + + if (attempt < BATCH_CONFIG.maxRetries) { + console.log( + ` Retry ${attempt}/${BATCH_CONFIG.maxRetries} for batch [${words.join(", ")}]: ${errorMessage}`, + ); + return enrichWordWithRetry(words, rawLanguage, rawPos, attempt + 1); + } + + // Max retries reached — split and retry + console.log( + ` Splitting batch [${words.join(", ")}] after ${BATCH_CONFIG.maxRetries} failed attempts`, + ); + + const half = Math.ceil(words.length / 2); + const left = words.slice(0, half); + const right = words.slice(half); + + const leftResult = await enrichWordWithRetry(left, rawLanguage, rawPos, 1); + const rightResult = await enrichWordWithRetry( + right, + rawLanguage, + rawPos, + 1, + ); + + // Merge results + const merged = new Map([...leftResult.results, ...rightResult.results]); + const mergedMetrics = { + promptTokens: + leftResult.metrics.promptTokens + rightResult.metrics.promptTokens, + completionTokens: + leftResult.metrics.completionTokens + + rightResult.metrics.completionTokens, + totalTokens: + leftResult.metrics.totalTokens + rightResult.metrics.totalTokens, + promptTimeMs: + leftResult.metrics.promptTimeMs + rightResult.metrics.promptTimeMs, + completionTimeMs: + leftResult.metrics.completionTimeMs + + rightResult.metrics.completionTimeMs, + promptTokensPerSecond: + (leftResult.metrics.promptTokensPerSecond + + rightResult.metrics.promptTokensPerSecond) / + 2, + completionTokensPerSecond: + (leftResult.metrics.completionTokensPerSecond + + rightResult.metrics.completionTokensPerSecond) / + 2, + }; + + return { results: merged, metrics: mergedMetrics }; + } +} diff --git a/data-pipeline/utils/get-word-file-path.ts b/data-pipeline/utils/get-word-file-path.ts new file mode 100644 index 0000000..81ef2d3 --- /dev/null +++ b/data-pipeline/utils/get-word-file-path.ts @@ -0,0 +1,5 @@ +import path from "path"; + +export function getWordFilePath(word: string, outputDir: string): string { + return path.join(outputDir, `${word}.json`); +} diff --git a/data-pipeline/utils/llm-adapters/factory.ts b/data-pipeline/utils/llm-adapters/factory.ts new file mode 100644 index 0000000..56e84a9 --- /dev/null +++ b/data-pipeline/utils/llm-adapters/factory.ts @@ -0,0 +1,30 @@ +import { LLM_CONFIG } from "../../config/llm.js"; +import { OpenAiCompatibleAdapter } from "./openai-compatible.js"; +import { GeminiAdapter } from "./gemini.js"; +import type { LlmAdapter } from "./types.js"; + +export function createAdapter(): LlmAdapter { + switch (LLM_CONFIG.provider) { + case "local": + return new OpenAiCompatibleAdapter(LLM_CONFIG.url); + case "openrouter": + return new OpenAiCompatibleAdapter( + LLM_CONFIG.url, + process.env["OPENROUTER_API_KEY"], + LLM_CONFIG.model, + ); + case "deepseek": + return new OpenAiCompatibleAdapter( + LLM_CONFIG.url, + process.env["DEEPSEEK_API_KEY"], + LLM_CONFIG.model, + ); + case "gemini": { + const apiKey = process.env["GEMINI_API_KEY"]; + if (!apiKey) throw new Error("GEMINI_API_KEY env var not set"); + if (!LLM_CONFIG.model) + throw new Error("LLM_CONFIG.model required for gemini"); + return new GeminiAdapter(apiKey, LLM_CONFIG.model); + } + } +} diff --git a/data-pipeline/utils/llm-adapters/gemini.ts b/data-pipeline/utils/llm-adapters/gemini.ts new file mode 100644 index 0000000..6fdb2de --- /dev/null +++ b/data-pipeline/utils/llm-adapters/gemini.ts @@ -0,0 +1,92 @@ +import type { LlmAdapter } from "./types.js"; + +interface GeminiResponse { + candidates: Array<{ content: { parts: Array<{ text: string }> } }>; + usageMetadata: { + promptTokenCount: number; + candidatesTokenCount: number; + totalTokenCount: number; + }; +} + +export class GeminiAdapter implements LlmAdapter { + private apiKey: string; + private model: string; + + constructor(apiKey: string, model: string) { + this.apiKey = apiKey; + this.model = model; + } + + async call( + words: string[], + systemPrompt: string, + ): Promise<{ + content: string; + promptTokens: number; + completionTokens: number; + totalTokens: number; + promptTimeMs: number; + completionTimeMs: number; + promptTokensPerSecond: number; + completionTokensPerSecond: number; + }> { + const url = `https://generativelanguage.googleapis.com/v1beta/models/${this.model}:generateContent?key=${this.apiKey}`; + + const payload = { + contents: [ + { + role: "user", + parts: [ + { text: systemPrompt + "\n\nWords: " + JSON.stringify(words) }, + ], + }, + ], + generationConfig: { + temperature: 0.1, + topP: 0.9, + maxOutputTokens: Math.ceil(words.length * 250 * 1.2), + }, + }; + + const startTime = Date.now(); + + const response = await fetch(url, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify(payload), + }); + + const totalTimeMs = Date.now() - startTime; + + if (!response.ok) { + throw new Error(`Gemini API responded with status: ${response.status}`); + } + + const json = (await response.json()) as GeminiResponse; + + const content = json.candidates[0]?.content?.parts[0]?.text; + if (!content) { + throw new Error("Gemini response content is empty"); + } + + const promptTokens = json.usageMetadata.promptTokenCount; + const completionTokens = json.usageMetadata.candidatesTokenCount; + const totalTokens = json.usageMetadata.totalTokenCount; + + // Gemini doesn't provide timing breakdown, so we estimate + const promptTimeMs = totalTimeMs * 0.3; // rough estimate + const completionTimeMs = totalTimeMs * 0.7; // rough estimate + + return { + content, + promptTokens, + completionTokens, + totalTokens, + promptTimeMs, + completionTimeMs, + promptTokensPerSecond: promptTokens / (promptTimeMs / 1000), + completionTokensPerSecond: completionTokens / (completionTimeMs / 1000), + }; + } +} diff --git a/data-pipeline/utils/llm-adapters/openai-compatible.ts b/data-pipeline/utils/llm-adapters/openai-compatible.ts new file mode 100644 index 0000000..f6e50a6 --- /dev/null +++ b/data-pipeline/utils/llm-adapters/openai-compatible.ts @@ -0,0 +1,92 @@ +import type { LlmAdapter } from "./types.js"; + +interface OpenAiResponse { + choices: Array<{ message: { content: string } }>; + usage: { + prompt_tokens: number; + completion_tokens: number; + total_tokens: number; + }; + timings: { + prompt_ms: number; + predicted_ms: number; + prompt_per_second: number; + predicted_per_second: number; + }; +} + +export class OpenAiCompatibleAdapter implements LlmAdapter { + private url: string; + private apiKey: string | undefined; + private model: string | undefined; + + constructor(url: string, apiKey?: string, model?: string) { + this.url = url; + this.apiKey = apiKey; + this.model = model; + } + + async call( + words: string[], + systemPrompt: string, + ): Promise<{ + content: string; + promptTokens: number; + completionTokens: number; + totalTokens: number; + promptTimeMs: number; + completionTimeMs: number; + promptTokensPerSecond: number; + completionTokensPerSecond: number; + }> { + const payload: Record = { + messages: [ + { role: "system", content: systemPrompt }, + { role: "user", content: JSON.stringify(words) }, + ], + temperature: 0.1, + top_p: 0.9, + max_tokens: Math.ceil(words.length * 250 * 1.2), + }; + + if (this.model) { + payload["model"] = this.model; + } + + const headers: Record = { + "Content-Type": "application/json", + }; + + if (this.apiKey) { + headers["Authorization"] = `Bearer ${this.apiKey}`; + } + + const response = await fetch(this.url, { + method: "POST", + headers, + body: JSON.stringify(payload), + }); + + if (!response.ok) { + throw new Error(`LLM server responded with status: ${response.status}`); + } + + const json = (await response.json()) as OpenAiResponse; + + const content = json.choices[0]?.message?.content; + if (!content) { + throw new Error("LLM response content is empty"); + } + + return { + content, + promptTokens: json.usage.prompt_tokens, + completionTokens: json.usage.completion_tokens, + totalTokens: json.usage.total_tokens, + promptTimeMs: json.timings.prompt_ms, + completionTimeMs: json.timings.predicted_ms, + promptTokensPerSecond: json.timings.prompt_per_second, + completionTokensPerSecond: json.timings.predicted_per_second, + }; + } +} diff --git a/data-pipeline/utils/llm-adapters/types.ts b/data-pipeline/utils/llm-adapters/types.ts new file mode 100644 index 0000000..c0c5b70 --- /dev/null +++ b/data-pipeline/utils/llm-adapters/types.ts @@ -0,0 +1,15 @@ +export interface LlmAdapter { + call( + words: string[], + systemPrompt: string, + ): Promise<{ + content: string; + promptTokens: number; + completionTokens: number; + totalTokens: number; + promptTimeMs: number; + completionTimeMs: number; + promptTokensPerSecond: number; + completionTokensPerSecond: number; + }>; +} diff --git a/data-pipeline/utils/merge-enriched-data.ts b/data-pipeline/utils/merge-enriched-data.ts new file mode 100644 index 0000000..85035e3 --- /dev/null +++ b/data-pipeline/utils/merge-enriched-data.ts @@ -0,0 +1,69 @@ +export type Language = "en" | "de" | "it" | "es" | "fr"; +export type Pos = "noun" | "verb" | "adjective" | "adverb"; +export type Gender = "masculine" | "feminine" | "neuter" | null; +export type Difficulty = "easy" | "medium" | "hard"; + +export interface Translation { + word: string; + gender: Gender; +} + +export interface EnrichedSense { + id: string; + word: string; + language: Language; + pos: Pos; + sense: string; + example: string; + difficulty_level: Difficulty; + translations: { + de: Translation[]; + it: Translation[]; + es: Translation[]; + fr: Translation[]; + }; +} + +const LANG_MAP: Record = { + english: "en", + italian: "it", + german: "de", + french: "fr", + spanish: "es", +}; + +const POS_MAP: Record = { + nouns: "noun", + verbs: "verb", + adverbs: "adverb", + adjectives: "adjective", +}; + +/** + * Merges skeleton data with enriched LLM senses into the final pipeline output. + */ +export function mergeEnrichedData( + word: string, + rawLanguage: string, + rawPos: string, + senses: EnrichedSense[], +): Record { + const language = LANG_MAP[rawLanguage] || (rawLanguage as Language); + const pos = POS_MAP[rawPos] || (rawPos as Pos); + + const fixedSenses = senses.map((sense, index) => ({ + ...sense, + id: `${word}:${language}:${pos}:${index}`, + language, + pos, + })); + + return { + word, + language, + pos, + senses: fixedSenses, + enrichedAt: new Date().toISOString(), + model: "qwen3.5-4b-q4_k_m", + }; +} diff --git a/data-pipeline/utils/pipeline-timer.ts b/data-pipeline/utils/pipeline-timer.ts new file mode 100644 index 0000000..56683e9 --- /dev/null +++ b/data-pipeline/utils/pipeline-timer.ts @@ -0,0 +1,162 @@ +interface LlmMetrics { + promptTokens: number; + completionTokens: number; + totalTokens: number; + promptTimeMs: number; + completionTimeMs: number; + promptTokensPerSecond: number; + completionTokensPerSecond: number; +} + +interface PipelineMetrics { + startTime: Date; + endTime?: Date; + wordsProcessed: number; + wordsSkipped: number; + wordsFailed: number; + llmCalls: number; + totalPromptTokens: number; + totalCompletionTokens: number; + totalTokens: number; + totalPromptTimeMs: number; + totalCompletionTimeMs: number; + currentWordStartTime?: Date; +} + +/** + * Simple timer and metrics tracker for the pipeline. + * Tracks both pipeline throughput and LLM performance. + */ +export class PipelineTimer { + private metrics: PipelineMetrics; + + constructor() { + this.metrics = { + startTime: new Date(), + wordsProcessed: 0, + wordsSkipped: 0, + wordsFailed: 0, + llmCalls: 0, + totalPromptTokens: 0, + totalCompletionTokens: 0, + totalTokens: 0, + totalPromptTimeMs: 0, + totalCompletionTimeMs: 0, + }; + } + + startWord(): void { + this.metrics.currentWordStartTime = new Date(); + } + + getWordDurationMs(): number { + if (!this.metrics.currentWordStartTime) return 0; + return new Date().getTime() - this.metrics.currentWordStartTime.getTime(); + } + + recordProcessed(llmMetrics?: LlmMetrics): void { + this.metrics.wordsProcessed++; + if (llmMetrics) { + this.metrics.llmCalls++; + this.metrics.totalPromptTokens += llmMetrics.promptTokens; + this.metrics.totalCompletionTokens += llmMetrics.completionTokens; + this.metrics.totalTokens += llmMetrics.totalTokens; + this.metrics.totalPromptTimeMs += llmMetrics.promptTimeMs; + this.metrics.totalCompletionTimeMs += llmMetrics.completionTimeMs; + } + } + + recordSkipped(): void { + this.metrics.wordsSkipped++; + } + + recordFailed(): void { + this.metrics.wordsFailed++; + } + + stop(): void { + this.metrics.endTime = new Date(); + } + + getWordTiming(): string { + const durationMs = this.getWordDurationMs(); + const durationSec = (durationMs / 1000).toFixed(1); + return `⏱️ Word took ${durationSec}s`; + } + + getEta(totalWords: number): string { + const processed = this.metrics.wordsProcessed; + const remaining = totalWords - processed - this.metrics.wordsSkipped; + + if (processed === 0 || remaining <= 0) return "ETA: calculating..."; + + const elapsedMs = new Date().getTime() - this.metrics.startTime.getTime(); + const avgMsPerWord = elapsedMs / processed; + const etaMs = avgMsPerWord * remaining; + + const etaMin = Math.round(etaMs / 60000); + const etaHour = (etaMs / 3600000).toFixed(1); + + if (etaMin < 60) { + return `ETA: ${etaMin} min`; + } + return `ETA: ${etaHour} hours`; + } + + getSummary(): string { + const end = this.metrics.endTime || new Date(); + const durationMs = end.getTime() - this.metrics.startTime.getTime(); + const durationSec = (durationMs / 1000).toFixed(1); + + const total = + this.metrics.wordsProcessed + + this.metrics.wordsSkipped + + this.metrics.wordsFailed; + const throughput = + this.metrics.wordsProcessed > 0 + ? (this.metrics.wordsProcessed / (durationMs / 1000)).toFixed(2) + : "0"; + + const avgPromptTokens = + this.metrics.llmCalls > 0 + ? (this.metrics.totalPromptTokens / this.metrics.llmCalls).toFixed(0) + : "0"; + const avgCompletionTokens = + this.metrics.llmCalls > 0 + ? (this.metrics.totalCompletionTokens / this.metrics.llmCalls).toFixed( + 0, + ) + : "0"; + const avgPromptSpeed = + this.metrics.totalPromptTimeMs > 0 + ? ( + this.metrics.totalPromptTokens / + (this.metrics.totalPromptTimeMs / 1000) + ).toFixed(1) + : "0"; + const avgCompletionSpeed = + this.metrics.totalCompletionTimeMs > 0 + ? ( + this.metrics.totalCompletionTokens / + (this.metrics.totalCompletionTimeMs / 1000) + ).toFixed(1) + : "0"; + + return [ + `⏱️ Pipeline Summary`, + ` Duration: ${durationSec}s`, + ` Processed: ${this.metrics.wordsProcessed}`, + ` Skipped: ${this.metrics.wordsSkipped}`, + ` Failed: ${this.metrics.wordsFailed}`, + ` Total: ${total}`, + ` Throughput: ${throughput} words/sec`, + ``, + `🤖 LLM Metrics`, + ` Calls: ${this.metrics.llmCalls}`, + ` Avg prompt tokens: ${avgPromptTokens}`, + ` Avg completion tokens: ${avgCompletionTokens}`, + ` Avg prompt speed: ${avgPromptSpeed} tok/s`, + ` Avg completion speed: ${avgCompletionSpeed} tok/s`, + ].join("\n"); + } +} diff --git a/data-pipeline/utils/progress-tracker.ts b/data-pipeline/utils/progress-tracker.ts new file mode 100644 index 0000000..d232918 --- /dev/null +++ b/data-pipeline/utils/progress-tracker.ts @@ -0,0 +1,27 @@ +/** + * Simple progress tracker for pipeline execution. + */ +export class ProgressTracker { + private current: number; + private failed: number; + private total: number; + + constructor(total: number) { + this.current = 0; + this.failed = 0; + this.total = total; + } + + next(): number { + this.current++; + return this.current; + } + + recordFailed(): void { + this.failed++; + } + + format(label: string): string { + return `[${this.current}/${this.total}] (${this.failed} failed) ${label}`; + } +} diff --git a/data-pipeline/utils/scanning-source-files.ts b/data-pipeline/utils/scanning-source-files.ts new file mode 100644 index 0000000..e9d05ca --- /dev/null +++ b/data-pipeline/utils/scanning-source-files.ts @@ -0,0 +1,61 @@ +import fs from "fs"; +import path from "path"; + +// Define a simple shape for what a discovered dataset looks like +export interface Wordlist { + language: string; + pos: string; + sourcePath: string; + outputDir: string; +} + +/** + * Scans the source-data directory to find all available word lists. + */ +export function scanSourceData(baseDir: string): Wordlist[] { + const sourceBaseDir = path.join(baseDir, "source-data"); + const discoveredWordlists: Wordlist[] = []; + + // Safety check: if there's no source-data folder, return an empty array + if (!fs.existsSync(sourceBaseDir)) { + return discoveredWordlists; + } + + // 1. Read the language directories (e.g., ['english']) + const languages = fs.readdirSync(sourceBaseDir); + + for (const lang of languages) { + const langFolderPath = path.join(sourceBaseDir, lang); + + // Make sure it's a directory, not a stray file + if (!fs.statSync(langFolderPath).isDirectory()) continue; + + // 2. Read the files inside the language folder (e.g., ['nouns']) + const posFiles = fs.readdirSync(langFolderPath); + + for (const pos of posFiles) { + const fullSourcePath = path.join(langFolderPath, pos); + + // Make sure it's a file (like your extensionless "nouns" file) + if (!fs.statSync(fullSourcePath).isFile()) continue; + + // 3. Package everything into a flat item and add it to our array + discoveredWordlists.push({ + language: lang, + pos: pos, + sourcePath: fullSourcePath, + outputDir: path.join(baseDir, "worddata", lang, pos), + }); + } + } + + // show summary + console.log( + `✅ Scan complete! Found ${discoveredWordlists.length} wordlist(s):`, + ); + for (const list of discoveredWordlists) { + console.log(` • ${list.language.toUpperCase()} (${list.pos})`); + } + + return discoveredWordlists; +} diff --git a/data-pipeline/utils/verify-enriched-file.ts b/data-pipeline/utils/verify-enriched-file.ts new file mode 100644 index 0000000..bbe0560 --- /dev/null +++ b/data-pipeline/utils/verify-enriched-file.ts @@ -0,0 +1,96 @@ +import fs from "fs"; + +interface VerificationResult { + valid: boolean; + errors: string[]; +} + +/** + * Verifies that an enriched JSON file matches the expected schema. + * Returns detailed error messages for any violations. + */ +export function verifyEnrichedFile(filePath: string): VerificationResult { + const errors: string[] = []; + + if (!fs.existsSync(filePath)) { + return { valid: false, errors: ["File does not exist"] }; + } + + let data: unknown; + try { + data = JSON.parse(fs.readFileSync(filePath, "utf-8")); + } catch (_error: unknown) { + return { valid: false, errors: ["Invalid JSON syntax"] }; + } + + if (typeof data !== "object" || data === null || Array.isArray(data)) { + return { valid: false, errors: ["Root must be an object"] }; + } + + const obj = data as Record; + + // Required top-level fields + const requiredFields = ["word", "language", "pos", "senses"]; + for (const field of requiredFields) { + if (!(field in obj)) { + errors.push(`Missing required field: "${field}"`); + } + } + + // Validate senses array + if (!Array.isArray(obj["senses"])) { + errors.push('"senses" must be an array'); + } else if (obj["senses"].length === 0) { + errors.push('"senses" array cannot be empty'); + } else { + for (let i = 0; i < obj["senses"].length; i++) { + const sense = obj["senses"][i] as Record; + const sensePrefix = `senses[${i}]`; + + if (!sense["sense"] || typeof sense["sense"] !== "string") { + errors.push(`${sensePrefix}: missing or invalid "sense"`); + } + if (!sense["example"] || typeof sense["example"] !== "string") { + errors.push(`${sensePrefix}: missing or invalid "example"`); + } + if ( + !["easy", "medium", "hard"].includes( + sense["difficulty_level"] as string, + ) + ) { + errors.push(`${sensePrefix}: invalid "difficulty_level"`); + } + if (!sense["translations"] || typeof sense["translations"] !== "object") { + errors.push(`${sensePrefix}: missing "translations"`); + } else { + const trans = sense["translations"] as Record; + for (const lang of ["de", "it", "es", "fr"]) { + if (!Array.isArray(trans[lang])) { + errors.push( + `${sensePrefix}: missing or invalid "${lang}" translations`, + ); + } else { + for (let j = 0; j < (trans[lang] as unknown[]).length; j++) { + const t = (trans[lang] as unknown[])[j] as Record< + string, + unknown + >; + if (!t["word"] || typeof t["word"] !== "string") { + errors.push(`${sensePrefix}.${lang}[${j}]: missing "word"`); + } + if ( + !["masculine", "feminine", "neuter", null].includes( + t["gender"] as string | null, + ) + ) { + errors.push(`${sensePrefix}.${lang}[${j}]: invalid "gender"`); + } + } + } + } + } + } + } + + return { valid: errors.length === 0, errors }; +} diff --git a/data-pipeline/utils/write-json-file.ts b/data-pipeline/utils/write-json-file.ts new file mode 100644 index 0000000..29c5c99 --- /dev/null +++ b/data-pipeline/utils/write-json-file.ts @@ -0,0 +1,11 @@ +import fs from "fs"; + +/** + * Writes data as formatted JSON to a file path. + * Safely catches and re-throws file system errors. + */ +export function writeJsonFile(filePath: string, data: unknown): void { + const tempPath = `${filePath}.tmp`; + fs.writeFileSync(tempPath, JSON.stringify(data, null, 2), "utf-8"); + fs.renameSync(tempPath, filePath); +} diff --git a/documentation/DATA_PIPELINE.md b/documentation/DATA_PIPELINE.md index e8899dc..f03f6ba 100644 --- a/documentation/DATA_PIPELINE.md +++ b/documentation/DATA_PIPELINE.md @@ -1,489 +1,901 @@ -# lila data pipeline +# Lila Data Pipeline — Technical Documentation -This pipeline extracts vocabulary data from Wiktionary via the Kaikki dataset, enriches it with CEFR levels and fills content gaps using local LLMs, and produces authoritative output in `pipeline.db`. This database is consumed by the sync script to populate the production database with vocabulary entries, translations, glosses, CEFR levels, and difficulty ratings. +> Multilingual dictionary enrichment pipeline for the Lila vocabulary trainer. +> Last updated: 2026-06-17 -## Overview +--- -```mermaid -flowchart LR - kaikki[(Kaikki JSONL)] - extract[Extract] - reverselink[Reverse Link Sync] - enrich[Enrich] - pipelinedb[(pipeline.db)] - merge[Merge] - tiebreak[Tiebreak] - compare[Compare] - sync[Sync] - db[(PostgreSQL)] +## Table of Contents - kaikki --> extract - extract --> pipelinedb - pipelinedb --> reverselink - reverselink --> pipelinedb - pipelinedb --> enrich - enrich --> pipelinedb - pipelinedb --> merge - merge --> pipelinedb - pipelinedb --> tiebreak - tiebreak --> pipelinedb - pipelinedb --> compare - pipelinedb --> sync - sync --> db +1. [Executive Summary](#1-executive-summary) +2. [Problem & Context](#2-problem--context) +3. [Architecture Overview](#3-architecture-overview) +4. [Current Implementation](#4-current-implementation) +5. [The LLM Layer](#5-the-llm-layer) + - 5.1 [Local Model Evaluation](#51-local-model-evaluation) + - 5.2 [Online API Options](#52-online-api-options) + - 5.3 [Model Selection Criteria](#53-model-selection-criteria) +6. [The Gender Problem & Kaikki Integration](#6-the-gender-problem--kaikki-integration) +7. [Batching Strategy](#7-batching-strategy) +8. [Hardware Constraints](#8-hardware-constraints) +9. [Testing & Quality Assurance](#9-testing--quality-assurance) +10. [Future Extensions & Roadmap](#10-future-extensions--roadmap) +11. [Decisions Log](#11-decisions-log) +12. [Known Issues & Dev Notes](#12-known-issues--dev-notes) +13. [How to Run](#13-how-to-run) +14. [Roadmap](#14-roadmap) + +--- + +## Quick Reference + +| What | Where | +| ----------------- | ----------------------------------- | +| Entry point | `pipeline.ts` | +| LLM config | `config/llm.ts` | +| System prompt | `config/prompt.ts` | +| Output schema | `utils/merge-enriched-data.ts` | +| Batch size config | `config/batch.ts` _(planned)_ | +| Current model | `qwen2.5-1.5b-instruct-q4_k_m.gguf` | +| Target scale | 100,000+ words | + +--- + +## 1. Executive Summary + +The Lila Data Pipeline is a TypeScript-based batch processing system that enriches raw word lists into structured multilingual dictionary entries for the Lila vocabulary trainer. It takes a source wordlist (e.g., English nouns) and, for each word, generates: + +- One or more **senses** (definitions) +- A **natural example sentence** per sense +- A **CEFR-based difficulty level** (`easy` / `medium` / `hard`) +- **Translations** into German, Italian, Spanish, and French, each with grammatical **gender** + +The pipeline is designed to scale to **100,000+ words** across multiple languages and parts of speech (nouns, verbs, adjectives, adverbs). It is currently in active development: the core architecture is stable, the LLM integration layer is being evaluated across local and online providers, and a configurable batching system is planned to unlock throughput at scale. + +### Key Design Principles + +| Principle | Rationale | +| ------------------------------- | ------------------------------------------------------------------------------------------------------------------------------ | +| **Quality first** | Definitions, examples, translations, and gender must be accurate. Speed and cost are secondary. | +| **Local-first, cloud-fallback** | Local LLMs (llama.cpp) are the default for cost control and data privacy. Online APIs are evaluated as alternatives for speed. | +| **Resumable & idempotent** | Each word writes to its own JSON file. The pipeline skips already-processed words on restart. | +| **Configurable batching** | Batch size (1, 5, 15, 50, etc.) is a single config value. The pipeline adapts without code changes. | + +### Open Question: Gender Accuracy + +Grammatical gender is currently generated by the LLM as part of the translation object. Early testing showed that **Qwen2.5-1.5B systematically defaults to `neuter`** for languages that do not have neuter grammatical gender (Italian, Spanish, French). Whether this is a **model size issue** (fixable by moving to 3B+) or a **training data issue** (requiring an external lookup) is unresolved. + +**Options under evaluation:** + +- Larger local models (Qwen2.5-3B, Qwen3.5-1.7B) +- Online models with stronger multilingual training (Gemini, DeepSeek) +- Post-processing lookup via **Kaikki Wiktionary dumps** as a fallback or replacement + +No decision made. Gender handling will be determined by the 20-word quality torture suite. + +### Current Status (2026-06-17) + +- Core pipeline: scanning, enrichment, merging, verification, writing +- Local LLM integration via llama.cpp server (OpenAI-compatible API) +- Schema validation for generated JSON +- Progress tracking and timing metrics +- **In progress:** Evaluating local models (Qwen2.5-1.5B tested; Qwen2.5-3B download pending) +- **In progress:** Designing configurable batching system +- **Pending:** 20-word quality torture suite (will decide gender approach) +- **Pending:** Online API evaluation (Gemini free tier, DeepSeek, Groq) + +### One-Line Architecture + +``` +source wordlists -> llama.cpp server (LLM) -> merge senses -> verify schema -> write .json + | + [gender: LLM-generated, accuracy TBD] ``` -Each stage is a standalone script that reads from and writes to `pipeline.db`. The pipeline is fully resumable — interrupted overnight runs pick up from the last processed record without losing work. +### Files at a Glance -Stage 1 is a manual prerequisite and is not run by the pipeline orchestrator. See **Stage 1 — Extract** for instructions. +| File | Purpose | +| ------------------------------- | ---------------------------------------------------------------- | +| `pipeline.ts` | Orchestrator. Scans sources, loops words, coordinates all stages | +| `config/llm.ts` | API URL, default parameters (`temperature`, `max_tokens`, etc.) | +| `config/prompt.ts` | System prompt sent to the LLM | +| `utils/enrich-word.ts` | Calls LLM, parses response, builds `EnrichedSense[]` | +| `utils/merge-enriched-data.ts` | Merges skeleton + enriched senses into final JSON | +| `utils/verify-enriched-file.ts` | Schema validation (required fields, types, gender enum) | +| `utils/check-llm-server.ts` | Health check before pipeline starts | +| `utils/progress-tracker.ts` | `[current/total]` formatting for console output | +| `utils/pipeline-timer.ts` | Per-word and global timing + LLM token metrics | -The enrich stage is designed to run overnight, one model at a time. Each model processes every entry and writes results to `pipeline.db` atomically per record. +### Scale Target -Only fully resolved records reach the production database. Records where LLMs could not reach a majority vote are handled automatically by the tiebreaker stage before syncing. +| Metric | Target | +| --------------- | ------------------------------------------------------------ | +| Words | 100,000+ | +| Languages | English (source), German, Italian, Spanish, French (targets) | +| Parts of speech | Nouns, verbs, adjectives, adverbs | +| Output | One `.json` file per word, ~2-5KB each | -## pipeline.db +--- -All pipeline state is stored in `pipeline.db` — a SQLite database in `data-pipeline/db/`. It is created automatically on first run and is not committed to git. +## 2. Problem & Context -The database serves three purposes: +### Why Build This? -- **Resumability** — every record is written atomically with a status. Interrupted overnight runs resume from the last pending record without losing work. -- **Vote tracking** — all model votes for CEFR levels and generated content are stored per model per record, giving full auditability of how every decision was reached. -- **Resolved output** — the final resolved records live here and are read by the sync script to seed the production database. +Existing multilingual dictionaries and translation APIs provide raw word-to-word mappings. They do not provide the structured, pedagogical data needed for a vocabulary trainer: -The schema is defined in `data-pipeline/db/schema.sql`. Never edit `pipeline.db` directly — all writes go through the pipeline scripts. +| What Exists | What is Missing | Why It Matters | +| ------------------------ | -------------------------------------- | ----------------------------------------------------------------- | +| Word + translation | **Student-friendly definition** | Learners need explanations, not just equivalents | +| Static difficulty labels | **CEFR-calibrated difficulty** | "Easy" vs "hard" must map to learner proficiency levels | +| Isolated translations | **Natural example sentences** | Context is how vocabulary is actually acquired | +| Raw gender data | **Gender integrated with translation** | Flashcards must show gender immediately, not as a separate lookup | +| Bulk word lists | **Per-word structured JSON** | The trainer consumes one file per word for fast random access | -On first run the orchestrator initialises `pipeline.db` automatically and imports the stage 1 output into the base tables. This happens once — subsequent runs skip the import if the base tables are already populated. +### The Target User -## Common commands +A language learner using the Lila vocabulary trainer. They see a word, its definition, an example sentence, and translations with gender - all calibrated to their CEFR level (A1-C2). -### Starting llama.cpp +### Why Not Use Existing Dictionaries? + +- **Wiktionary**: Rich data, but unstructured, inconsistent formatting, no CEFR levels, no student-friendly definitions +- **Kaikki (Wiktionary dump)**: Structured JSON, excellent for gender/translation lookup, but definitions are often technical, no difficulty classification, no example curation +- **Google Translate / DeepL**: No definitions, no examples, no difficulty, no structured output +- **Existing language learning apps**: Closed data, no export, no control over content + +The LLM fills the gap: it generates **pedagogical content** (student-friendly definitions, natural examples, difficulty classification) that no existing database provides at scale. + +### Language Direction + +The pipeline is **direction-agnostic**. A wordlist is defined by: + +- **Source language**: the language of the input words +- **Target languages**: the languages to translate into + +Current focus: **English -> German/Italian/Spanish/French** + +Planned directions include **German -> French**, **Italian -> Spanish**, etc. The LLM prompt and output schema support any combination - the only change is the source wordlist and the target languages specified in the prompt. + +### Why 100,000+ Words? + +- **Coverage**: A learner needs ~10,000 words for B2 proficiency. The pipeline targets 100k to cover multiple languages, POS categories, and difficulty levels with room for curation. +- **Languages**: English (source) -> German, Italian, Spanish, French (targets). +- **Parts of speech**: Nouns (current), verbs, adjectives, adverbs. Each POS has different enrichment needs (verb conjugations, adjective agreement, etc.). + +### The Data Flow + +``` +Source files LLM enrichment Final JSON +(one word per line) (definitions, (one per word, + examples, self-contained) +english/nouns difficulty, +english/verbs translations) time.json +italian/nouns year.json +... people.json +``` + +### The Quality Challenge + +Generating 100,000 entries with an LLM introduces risks: + +| Risk | Mitigation | +| ------------------------------ | -------------------------------------------------------------- | +| Hallucinated definitions | Low temperature (0.1), strict system prompt, schema validation | +| Incorrect grammatical gender | Under evaluation: larger models or external lookup | +| Inconsistent difficulty levels | Explicit CEFR mapping in prompt, spot-checking | +| JSON parse failures | Retry logic, schema validation, cleanup on failure | +| Model drift (online APIs) | Version pinning, local fallback | + +### Why TypeScript + Node? + +- **Familiarity**: Existing project uses TypeScript (frontend in TanStack Router + React) +- **Ecosystem**: `readline` for streaming files, `fs` for JSON I/O, native `fetch` for HTTP +- **Portability**: Runs on the same Debian laptop as the llama.cpp server +- **No build complexity**: `tsx` for direct execution, no bundler needed + +### Why llama.cpp? + +- **GGUF format**: Single-file models, easy to swap, quantize, and version +- **OpenAI-compatible API**: `/v1/chat/completions` means the same code works for local and online models +- **No dependencies**: Self-contained binary, runs on old hardware (tested on GTX 950M) +- **Privacy**: Local inference means no data leaves the machine + +--- + +## 3. Architecture Overview + +### Pipeline Flow + +``` +Scan sources -> Check LLM -> Loop words -> Skip if exists -> Create skeleton + -> Call LLM -> Parse JSON -> Merge -> Write atomically -> Verify schema +``` + +### Resumability + +- **Skip existing**: Checks if `{word}.json` exists with non-empty `senses` +- **Atomic writes**: `.tmp` -> rename, no partial files on crash +- **Cleanup on failure**: Deletes incomplete file, continues to next word + +### Directory Structure + +``` +data-pipeline/ +|-- pipeline.ts # Entry point / orchestrator +|-- config/ +| |-- llm.ts # API URL, model params +| |-- prompt.ts # System prompt +|-- utils/ # See source files (provided separately) +|-- source-data/ +| |-- {language}/ +| |-- {pos} # One word per line, no extension +|-- worddata/ + |-- {language}/ + |-- {pos}/ + |-- {word}.json # One self-contained file per word +``` + +### Output Schema + +Each `.json` file contains: `word`, `language`, `pos`, `senses[]` (each with `sense`, `example`, `difficulty_level`, `translations` per target language), `enrichedAt`, `model`. + +Full TypeScript interfaces: `utils/merge-enriched-data.ts`. + +### Error Handling + +| Failure | Behavior | +| ----------------------- | ------------------------------ | +| LLM server offline | Hard fail at startup | +| LLM returns bad JSON | Log, delete skeleton, continue | +| Schema validation fails | Log warnings, keep file | +| Individual word fails | Does not stop pipeline | + +### Metrics + +Per-run: words processed/skipped/failed, duration, throughput, LLM token counts and speeds. See `utils/pipeline-timer.ts`. + +--- + +## 4. Current Implementation + +### Tech Stack + +| Layer | Choice | Why | +| ----------- | ----------------------------- | ------------------------------------------ | +| Runtime | Node.js + `tsx` | Direct TypeScript execution, no build step | +| HTTP client | Native `fetch` | Works for local llama.cpp and online APIs | +| File I/O | `fs` + `readline` | Streaming line reader for large wordlists | +| JSON | Native `JSON.parse/stringify` | Simple, no schema library needed | + +### Configuration + +| File | Purpose | +| ------------------ | -------------------------------------------------------------------------- | +| `config/llm.ts` | `LLM_API_URL`, `LLM_DEFAULT_PARAMS` (`temperature`, `top_p`, `max_tokens`) | +| `config/prompt.ts` | System prompt with CEFR mapping, required fields, example output | + +### Key Modules + +| File | Responsibility | +| ------------------------------- | ------------------------------------------------------------------------------------------ | +| `utils/enrich-word.ts` | Calls LLM, strips markdown, parses JSON array, builds `EnrichedSense[]` with composite IDs | +| `utils/merge-enriched-data.ts` | Merges skeleton `{word, language, pos}` with LLM senses, adds `enrichedAt` and `model` | +| `utils/verify-enriched-file.ts` | Schema validation: required fields, array lengths, gender enum, translation structure | +| `utils/pipeline-timer.ts` | Tracks per-word and global metrics (duration, tokens, throughput) | + +### Current Model + +| Property | Value | +| ------------ | ---------------------------------------- | +| Model | `qwen2.5-1.5b-instruct-q4_k_m.gguf` | +| Size | ~1.0GB | +| Quantization | Q4_K_M | +| Server | llama.cpp (`llama-server`) | +| API | OpenAI-compatible `/v1/chat/completions` | + +### llama-server Flags: History & Rationale + +The server flags evolved through trial and error on the target hardware (Intel i7-6500U, GTX 950M 4GB, 8GB RAM). Below is what was tried, what failed, and why the current flags were chosen. + +#### Hardware Constraints + +| Component | Spec | Implication | +| --------- | ----------------------------------------- | --------------------------------------------------------------------------- | +| CPU | i7-6500U (2 physical cores, 4 threads HT) | `-t 2` matches physical cores; HT hurts more than helps | +| GPU | GTX 950M (Maxwell, 2015) | 32 GB/s memory bandwidth, 4GB VRAM - bandwidth-starved, not compute-starved | +| RAM | 8GB (3.95GB usable) | `--mlock` pins model in RAM; system must not swap | + +#### Flag Evolution + +| Flag | Value Tried | Result | Why | +| ----------------- | ----------------------------------- | ------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------- | +| `-m` | `qwen3.5-4b-q4_k_m.gguf` | Works, ~47s/word | Baseline. Correct genders. 2.6GB file, tight on VRAM. | +| `-m` | `Ministral-3b-instruct.Q4_K_M.gguf` | **Broken** | Tokenizer mismatch (Tekken). Outputs gibberish regardless of template. See [Known Issues](#12-known-issues--dev-notes). | +| `-m` | `qwen2.5-1.5b-instruct-q4_k_m.gguf` | Works, ~8s/word | Current. Fast but gender accuracy degraded. | +| `-ngl` | `999` | Keeps | Offload all layers to GPU. Required for any speed. | +| `-c` | `4096` | Wasteful | 4K context for 300-token dictionary entries wastes ~400MB VRAM. | +| `-c` | `2048` | Current | Sufficient headroom. Frees VRAM for KV cache. | +| `-b` / `-ub` | `512` | Keeps | Sweet spot for Maxwell. Larger batches (1024+) add overhead on old GPUs. | +| `-b` / `-ub` | `2048` | Slower on 950M | Tested briefly. No improvement, possibly worse due to memory pressure. | +| `-t` | `4` | Slower | Hyperthreading cores hurt llama.cpp performance. | +| `-t` | `2` | Current | Matches 2 physical cores. | +| `--threads-batch` | (default) | Risky | Defaults to same as `-t`, but explicit is safer. | +| `--threads-batch` | `2` | Current | Explicit match to `-t`. | +| `--flash-attn` | (omitted) | Correct | On Maxwell (compute 5.0), Flash Attention adds overhead. Not used. | +| `--flash-attn` | (tested) | No gain | Briefly tried with Qwen3.5-4B. No speedup, possibly regression. | +| `--mlock` | Keeps | Pins model weights in RAM. Prevents OS swapping on memory pressure. | +| `--prio` | `2` | Keeps | Raises process priority. Marginal on this hardware, harmless. | +| `--reasoning` | `off` | (Qwen3.5 only) | Qwen3.5 has reasoning mode. Disabling it speeds up non-reasoning tasks. Irrelevant for Qwen2.5. | + +#### Current Command ```bash -cd ~/Downloads/llama.cpp ./build/bin/llama-server \ - --model models/qwen3.5-4b-q4_k_m.gguf \ - --port 8080 \ - --ctx-size 4096 \ - --n-gpu-layers 999 \ + -m models/qwen2.5-1.5b-instruct-q4_k_m.gguf \ + -ngl 999 \ + -c 2048 \ + -b 512 \ + -ub 512 \ + -t 2 \ + --threads-batch 2 \ --host 127.0.0.1 \ - --chat-template-kwargs '{"enable_thinking":false}' \ - --reasoning-budget 0 + --port 8080 \ + --mlock \ + --prio 2 ``` -Verify the server is running: +#### What Was Not Tried (And Why) -```bash -curl http://127.0.0.1:8080/health +| Flag | Reason Skipped | +| --------------------------- | ---------------------------------------------------------------------------------------------------------- | +| `-fa` / `--flash-attn` | Maxwell architecture lacks efficient FA kernels. Benchmarks show regression or no gain on pre-Ampere GPUs. | +| `--no-mmap` | `--mlock` achieves the same (pin in RAM) without the I/O overhead of disabling mmap. | +| `-np` / `--parallel` | Not needed. Single sequential pipeline, no concurrent requests. | +| `--cont-batching` | Default in recent llama.cpp. No need to toggle. | +| `--defrag-thold` | KV cache defragmentation. Only relevant for very long contexts or heavy reuse. Not needed for 2048 ctx. | +| `-ot` / `--override-tensor` | Expert-level. No tensor-specific issues observed. | + +#### Future Flag Experiments + +| Experiment | Expected Outcome | +| ---------------- | -------------------------------------------------------------------------- | +| `-c 1024` | Further VRAM savings. Risk: insufficient for batching larger prompt sizes. | +| `-b 256 -ub 256` | Test if smaller batches reduce latency on bandwidth-starved Maxwell. | +| `--metrics` | Enable Prometheus endpoint for automated performance tracking. | + +### Performance Baseline + +| Metric | Qwen3.5-4B | Qwen2.5-1.5B | +| --------------------- | ---------- | ------------ | +| Time/word | ~47s | ~8s | +| Completion tok/s | ~6.4 | ~18.8 | +| Prompt tok/s | ~81 | ~549 | +| Avg completion tokens | ~274 | ~132 | +| Avg prompt tokens | ~327 | ~308 | + +### Known Limitations (Current) + +- **Gender accuracy**: Qwen2.5-1.5B systematically defaults to `neuter` for Romance languages. Under evaluation whether larger models fix this. +- **No batching**: One word = one LLM call. System prompt re-processed every time. +- **No retry logic**: LLM parse failures are logged and skipped, not retried. +- **Single POS**: Only nouns tested. Verbs/adjectives/adverbs need prompt adjustments. + +--- + +## 5. The LLM Layer + +### 5.1 Local Model Evaluation + +Models are evaluated on three criteria in order of priority: **quality** (definitions, examples, translations, gender accuracy), **speed** (completion tokens/sec), **VRAM fit** (must run on GTX 950M 4GB). + +| Model | Size | VRAM | Speed | Quality | Status | +| ----------------------- | ------ | ----- | ------------------- | --------------------------- | -------------------------------- | +| **Qwen3.5-4B Q4_K_M** | 2.6GB | Tight | ~6.4 tok/s | Baseline (assumed good) | Baseline - too slow | +| **Qwen2.5-1.5B Q4_K_M** | 1.0GB | Easy | ~18.8 tok/s | Gender systematically wrong | Current - fast, needs validation | +| **Ministral-3B Q4_K_M** | 1.9GB | Fits | - | Broken (tokenizer) | Abandoned | +| **Qwen2.5-3B Q4_K_M** | 1.9GB | Fits | ~12-15 tok/s (est.) | Unknown | Pending download | +| **Gemma 4 E2B Q4_K_M** | 3.46GB | No | - | - | Too large for 4GB VRAM | +| **Gemma 4 E2B IQ2_M** | 2.62GB | Fits | ~8-12 tok/s (est.) | "Low quality" per Google | Not worth it | + +#### Qwen2.5-1.5B Test Results (3 words) + +| Word | Definition | Example | Gender (de/it/es/fr) | Verdict | +| ------ | -------------------------------------------- | ---------------------------------- | --------------------------------- | --------- | +| time | "A period of duration..." | "The meeting was scheduled..." | neuter/neuter/neuter/neuter | All wrong | +| year | "A period of time consisting of 365 days..." | "The year 2023 is a leap year." | neuter/neuter/neuter/neuter | All wrong | +| people | "Individuals who are part of a group." | "The people gathered at the park." | neuter/feminine/feminine/feminine | Mixed | + +**Pattern:** Defaults to `neuter` when uncertain. Only correct when obvious (feminine endings in Romance languages). German "Jahr" is genuinely neuter - only correct by accident. + +#### Pending Tests + +- **Qwen2.5-3B**: Same architecture, 2x params. If gender fixes, it was a size problem. +- **20-word torture suite**: concrete, abstract, polysemous, technical, false friends. Will run on all candidate models. + +### 5.2 Online API Options + +Evaluated as fallbacks if local models fail quality or speed targets. All support OpenAI-compatible API. + +| Provider | Model | Input $/1M | Output $/1M | Free Tier | Rate Limit | Est. Cost (100k words) | Est. Time | +| ------------------- | -------------------- | ---------- | ----------- | ------------- | ---------------- | ---------------------- | ------------------- | +| **DeepSeek** | V4 Flash | $0.14 | $0.28 | 5M tokens | None | **$0-15** | ~1-2 days | +| **Gemini** | 2.5 Flash-Lite | $0.10 | $0.40 | 1,500 req/day | 1M TPM | **$0** (free tier) | ~1.5 days (batched) | +| **Qwen/Alibaba** | Qwen-Turbo | $0.05 | $0.20 | Unknown | 600 RPM | **~$11** | ~1-2 days | +| **Groq** | Llama 3.1 8B Instant | $0.05 | $0.08 | Yes | High | **~$7** | **~3-4 hours** | +| **OpenRouter free** | Various | $0 | $0 | 200 req/day | 20 RPM | **$0** | ~10 days | +| **OpenRouter paid** | DeepSeek V4 Flash | $0.14 | $0.28 | - | Same as provider | **~$16** (+5.5% fee) | ~2-3 days | + +**Notes:** + +- Costs assume ~550 tokens/word (300 prompt + 250 completion). +- Gemini free tier: 1,500 requests/day x 50 words/batch = 75k words/day. +- Groq: 500+ tok/s inference speed. Fastest option if cost is acceptable. +- DeepSeek: 5M free tokens ~ 9,000 words. Remainder at $0.14/$0.28 per million. + +### 5.3 Model Selection Criteria + +Decision flow for 100,000 words: + +``` +Start + | + v +Run 20-word torture suite +on Qwen2.5-3B (local) + | + |-- Quality good? -----> Use Qwen2.5-3B locally + | (gender correct) ~20 days, $0 + | + |-- Quality meh? -------> Test Gemini 2.5 Flash-Lite (free) + | + |-- Quality good? --> Batch 50, free tier + | ~1.5 days, $0 + | + |-- Quality meh? ---> Test Groq or DeepSeek paid + | + |-- Speed priority? --> Groq + | ~$7, 3-4 hours + | + |-- Cost priority? ---> DeepSeek + ~$15, 1-2 days ``` -### Running the pipeline +**Quality gates:** -```bash -pnpm --filter @lila/pipeline pipeline:run -``` +- > =90% gender accuracy (de/it/es/fr) +- 100% JSON parse rate +- No hallucinated definitions on polysemous words +- Natural, contextually appropriate example sentences +- Sensible difficulty classification (CEFR mapping) -The pipeline auto-generates a run name from the date and a counter. It picks up where it left off — completed stages are skipped automatically. +--- -### Stage 1 — Extract +## 6. The Gender Problem & Kaikki Integration -```bash -pnpm --filter @lila/pipeline extract -``` +### The Problem -Runs in sample mode (500 entries per language) by default. Remove the hardcoded limit in `stage-1-extract/scripts/extract.ts` for a full run. - -### Stage 2 — Reverse link sync - -```bash -pnpm --filter @lila/pipeline reverse-link -``` - -### Initialising and importing the database - -```bash -# Initialise pipeline.db from schema -pnpm --filter @lila/pipeline db:init - -# Import stage 1 output into pipeline.db -pnpm --filter @lila/pipeline db:import -``` - -### Resetting the database - -```bash -# Full reset — delete and reinitialise -rm data-pipeline/db/pipeline.db -pnpm --filter @lila/pipeline db:init -pnpm --filter @lila/pipeline db:import -pnpm --filter @lila/pipeline reverse-link -``` - -### Resetting enrich stage progress - -```bash -# Reset round 1 only -pnpm --filter @lila/pipeline db:reset round1 - -# Reset all stages except reverse link -pnpm --filter @lila/pipeline db:reset all -``` - -### Checking pipeline progress - -```bash -node -e " -const Database = require('better-sqlite3'); -const db = new Database('data-pipeline/db/pipeline.db', { readonly: true }); -const total = db.prepare('SELECT COUNT(*) as c FROM entries WHERE language = \\'en\\'').get().c; -const complete = db.prepare(\"SELECT COUNT(*) as c FROM run_status WHERE stage = 'round1' AND status = 'complete'\").get().c; -const needsReview = db.prepare(\"SELECT COUNT(*) as c FROM run_status WHERE stage = 'round1' AND status = 'needs_review'\").get().c; -console.log('Total English entries:', total); -console.log('Round 1 complete:', complete); -console.log('Needs review:', needsReview); -console.log('Pending:', total - complete - needsReview); -db.close(); -" -``` - -## Data source - -### Kaikki (Wiktionary) - -The pipeline uses pre-extracted Wiktionary data from [kaikki.org](https://kaikki.org), built with the [wiktextract](https://github.com/tatuylonen/wiktextract) tool. This data is updated weekly from the English Wiktionary dump and is freely available under the same license as Wiktionary (CC-BY-SA). - -**Why Kaikki instead of OMW:** -Kaikki is structured per word sense. Each headword has multiple senses, and translations are linked to a specific sense rather than a general concept. This prevents the sense disambiguation problems found in OMW, where a single concept entry could contain translations from entirely different meanings of a word. - -Each Kaikki entry provides: - -- A headword in the entry language -- One or more senses, each with a gloss and examples -- Per-sense translations to other languages with sense hints -- IPA pronunciations and audio file references (deferred — see **Further extensions**) -- Inflected forms (deferred — see **Further extensions**) - -The pipeline uses the English Wiktionary edition (`enwiktionary`), which contains entries for all five supported languages with glosses in English. - -### CEFR levels - -CEFR levels are assigned entirely by LLM majority vote. Each model receives the headword, gloss, and an example sentence and votes on the appropriate level (A1–C2). There are no curated source files — the LLMs are the sole source of CEFR annotations. - -If no majority is reached after all model runs, the entry is handled automatically by the tiebreaker stage. - -## Setup - -### Kaikki data files - -Download the pre-extracted Kaikki JSONL files for each language. These are large files — download them to `stage-1-extract/sources/` which is not committed to git. - -```bash -mkdir -p stage-1-extract/sources -cd stage-1-extract/sources - -# English entries (contains translations to all other languages) -wget https://kaikki.org/dictionary/English/kaikki.org-dictionary-English.jsonl.gz - -# Per-language files (for entries written in those languages) -wget https://kaikki.org/dictionary/German/kaikki.org-dictionary-German.jsonl.gz -wget https://kaikki.org/dictionary/Italian/kaikki.org-dictionary-Italian.jsonl.gz -wget https://kaikki.org/dictionary/French/kaikki.org-dictionary-French.jsonl.gz -wget https://kaikki.org/dictionary/Spanish/kaikki.org-dictionary-Spanish.jsonl.gz - -# Decompress -gunzip *.gz -``` - -### LLM setup - -See `llm-setup.md`. - -## Pipeline stages - -| Stage | What it does | -| --------------- | ------------------------------------------------------------------------ | -| 1. Extract | Parses Kaikki JSONL, imports entries into `pipeline.db` | -| 2. Reverse link | Inserts missing reverse translations between language pairs | -| 3. Enrich | LLMs fill translation gaps, improve glosses/examples, assign CEFR levels | -| 4. Merge | Resolves LLM votes into final values | -| 4b. Tiebreak | Runs unused models on flagged entries until majority is reached | -| 5. Compare / QA | Generates `COVERAGE.md` with detailed quality report | -| 6. Sync | Upserts resolved records into production PostgreSQL | - -### 1. Extract - -Parses the Kaikki JSONL files for all five languages and imports them into the base tables of `pipeline.db`. Filters to the four supported parts of speech: noun, verb, adjective, adverb. Each Kaikki sense becomes one row in `vocabulary_entries`. Translations are stored in `entry_translations` with their sense hints. - -**Input:** `stage-1-extract/sources/*.jsonl` -**Output:** `pipeline.db` — `vocabulary_entries` and `entry_translations` tables populated - -```bash -pnpm --filter @lila/pipeline extract -``` - -Add `--sample 100` to import only 100 entries per language for inspection before running the full import. - -Each entry in `pipeline.db` looks like this: +Grammatical gender is embedded in the `translations` object of each sense: ```json -{ - "headword": "thrill", - "language": "en", - "pos": "verb", - "sense_index": 0, - "gloss": "To suddenly excite someone, or to give them great pleasure.", - "examples": ["The movie thrilled the audience."], - "translations": [ - { "language": "de", "word": "begeistern", "sense_hint": "suddenly excite" }, - { - "language": "fr", - "word": "enthousiasmer", - "sense_hint": "suddenly excite" - }, - { "language": "it", "word": "entusiasmare" }, - { "language": "es", "word": "emocionar" } - ] +"translations": { + "de": [{"word": "Haus", "gender": "neuter"}], + "it": [{"word": "casa", "gender": "feminine"}], + "es": [{"word": "casa", "gender": "feminine"}], + "fr": [{"word": "maison", "gender": "feminine"}] } ``` -> **Note:** Stage 1 is a manual prerequisite. It is not run by the pipeline orchestrator (`pipeline.ts`). Run it once before running the orchestrator for the first time, and re-run it manually if the Kaikki source files are updated. +Early testing with **Qwen2.5-1.5B** showed systematic failure: the model defaults to `neuter` for any translation where it is uncertain. This is particularly broken for Romance languages (Italian, Spanish, French), which do not have a neuter grammatical gender at all - only masculine and feminine. -### 2. Reverse link sync +Whether this is a **model size issue** (1.5B too small to retain gender facts) or a **training data gap** (Qwen2.5 family lacks gender-annotated multilingual data) is unresolved. Pending the Qwen2.5-3B evaluation. -A pure script stage — no LLMs. For each translation pair in `entry_translations`, checks whether the reverse link exists. If English _thrill → begeistern_ exists and the German entry _begeistern_ exists in `vocabulary_entries` but lacks the English back-link, it is inserted automatically. +### Two Approaches Under Consideration -This runs before the enrich stage so that LLMs only generate translations that are genuinely missing — not translations that would be found by a simple reverse lookup. +| Approach | How It Works | Pros | Cons | +| -------------------------- | ---------------------------------------------------------------------------------------------------- | ------------------------------------------------------- | ----------------------------------------------------------------------------------------------- | +| **A. LLM-only** | Trust the model to generate correct gender | Zero additional complexity | Unreliable at small sizes; hallucination risk | +| **B. LLM + Kaikki lookup** | LLM generates word + translation string; post-processing step looks up gender from Kaikki JSONL dump | 100% deterministic; decouples gender from model quality | Adds pipeline stage; requires Kaikki data for each target language; must handle missing entries | -**Input:** `pipeline.db` — populated `vocabulary_entries` and `entry_translations` -**Output:** `pipeline.db` — missing reverse links inserted into `entry_translations` +### Kaikki Data + +Kaikki provides processed Wiktionary dumps as JSONL files, one per language. Each line is a lexical entry with structured data including gender. + +| Language | Kaikki File | Coverage | +| -------- | ------------------------------------- | -------- | +| German | `kaikki.org-dictionary-German.jsonl` | High | +| Italian | `kaikki.org-dictionary-Italian.jsonl` | High | +| Spanish | `kaikki.org-dictionary-Spanish.jsonl` | High | +| French | `kaikki.org-dictionary-French.jsonl` | High | + +Lookup logic: match on `word` (the translated string) -> extract `gender` field -> map to `"masculine" | "feminine" | "neuter" | null`. + +### Decision Pending + +- If **Qwen2.5-3B** or an online model produces >=90% accurate gender: **Approach A**, no Kaikki needed. +- If all tested models fail gender: **Approach B**, implement Kaikki lookup as a post-processing step after LLM enrichment. + +No implementation work started until the 20-word torture suite resolves this. + +--- + +## 7. Batching Strategy + +### Why Batching is Necessary + +At 1 word per call, 100,000 words = 100,000 LLM requests. Each call re-processes the ~300-token system prompt. Batching amortizes this cost. + +### Configurable Batch Size + +Single config point controls batch size everywhere: + +```typescript +// config/batch.ts (planned) +export const BATCH_CONFIG = { + size: 5, // Change to 15 or 50 to test + get maxTokens() { + return Math.ceil(this.size * 250 * 1.2); // 250 tok/word + 20% buffer + }, +} as const; +``` + +### Prompt Structure + +**Single word:** + +``` +Word: house +``` + +**Batch of 5:** + +``` +Words: ["house", "car", "tree", "water", "book"] +``` + +LLM returns a JSON object with word keys: + +```json +{ + "house": [ { "sense": "...", "example": "...", ... } ], + "car": [ { ... } ], + "tree": [ { ... } ], + "water": [ { ... } ], + "book": [ { ... } ] +} +``` + +### Retry Strategy + +If a batch fails (bad JSON, missing key, etc.): + +``` +Batch of 50 fails + | + v +Retry as 2 batches of 25 + | + v +If a 25 fails, retry as 5 batches of 5 + | + v +If a 5 fails, retry as individual words (fallback) +``` + +This gives resilience without losing the speed benefit of large batches. + +### Expected Impact by Environment + +| Environment | Batch Size | Expected Speedup | Notes | +| ------------------ | ---------- | ---------------- | -------------------------------------------------------- | +| Local GTX 950M | 3 | ~1.05-1.15x | Memory bandwidth limited. KV cache pressure on 4GB VRAM. | +| Local GTX 950M | 5 | ~1.10-1.20x | Sweet spot for this hardware. | +| Local GTX 950M | 15 | Risky | May OOM. Test carefully. | +| Local GTX 950M | 50 | Unlikely | VRAM insufficient. | +| Cloud API (Gemini) | 50 | 5x fewer calls | Unlocks free tier viability. | +| Cloud API (Groq) | 50 | 5x fewer calls | Minimal gain - already fast. | + +### Batching is Non-Negotiable + +Regardless of local vs cloud, batching is required for 100k words: + +- **Local**: Better GPU utilization, amortized prompt processing +- **Cloud**: Slams into rate limits slower, unlocks free tiers, some APIs offer 50% batch discounts + +--- + +## 8. Hardware Constraints + +### Current Machine + +| Component | Spec | +| --------- | ------------------------------------------------------------------- | +| OS | Debian GNU/Linux 13 (trixie) x86_64 | +| CPU | Intel Core i7-6500U (2 physical cores, 4 threads via HT) @ 3.10 GHz | +| GPU | NVIDIA GeForce GTX 950M (Maxwell, 2015) | +| GPU VRAM | 4GB | +| RAM | 8GB (3.95GB usable at idle) | +| Disk | 102GB ext4 (~63GB used) | + +### What Fits in 4GB VRAM + +| Model | File Size | KV Cache (2048 ctx) | Total VRAM | Fits? | +| ------------------- | --------- | ------------------- | ---------- | ------------------ | +| Qwen2.5-1.5B Q4_K_M | ~1.0GB | ~0.5GB | ~1.5GB | Yes | +| Qwen2.5-3B Q4_K_M | ~1.9GB | ~0.8GB | ~2.7GB | Yes | +| Ministral-3B Q4_K_M | ~1.9GB | ~0.8GB | ~2.7GB | Yes (but broken) | +| Qwen3.5-4B Q4_K_M | 2.6GB | ~1.0GB | ~3.6GB | Tight | +| Gemma 4 E2B Q4_K_M | 3.46GB | ~1.2GB | ~4.7GB | No | +| Gemma 4 E2B IQ2_M | 2.62GB | ~1.0GB | ~3.6GB | Maybe, low quality | + +### GPU Rental Alternatives + +If local hardware becomes the bottleneck: + +| Provider | GPU | VRAM | Price/Hour | Time for 100k Words | Total Cost | +| -------- | -------- | ---- | ----------- | ------------------- | ---------- | +| Vast.ai | RTX 4090 | 24GB | ~$0.30-0.60 | ~6-8 hours | **~$2-5** | +| RunPod | RTX 4090 | 24GB | ~$0.50-0.80 | ~6-8 hours | **~$4-6** | +| Vast.ai | RTX 3090 | 24GB | ~$0.20-0.40 | ~8-10 hours | **~$2-4** | + +With an RTX 4090, Qwen2.5-1.5B runs at ~100-150 tok/s. 100k words in under a day. + +--- + +## 9. Testing & Quality Assurance + +### 20-Word Torture Suite + +Planned test set covering edge cases: + +| Category | Words | Why | +| -------------- | ------------------------------------------------------ | ----------------------------------- | +| Easy concrete | `house`, `water`, `book` | Baseline | +| Easy abstract | `time`, `love`, `hope` | Abstract nouns harder to define | +| Polysemous | `bank`, `run`, `light` | Multiple senses test disambiguation | +| Hard/technical | `democracy`, `photosynthesis`, `entropy` | Complex definitions | +| False friends | `actual` (en/es), `sensible` (en/fr), `fabric` (en/de) | Cross-lingual traps | + +### Evaluation Criteria + +For each word and each candidate model: + +| Criterion | Pass Threshold | +| ----------------------------- | ------------------------------------------------- | +| Definition accuracy | Factually correct, max 15 words, student-friendly | +| Example quality | Natural sentence, word used correctly in context | +| Translation correctness | Correct word sense match | +| Gender accuracy (de/it/es/fr) | >=90% correct | +| Difficulty classification | Sensible per CEFR mapping | +| JSON reliability | 100% parse rate, valid schema | + +### Verification + +`verify-enriched-file.ts` checks: + +- Required top-level fields: `word`, `language`, `pos`, `senses` +- Each sense: `sense` (string), `example` (string), `difficulty_level` in {easy, medium, hard} +- Each translation: `word` (string), `gender` in {masculine, feminine, neuter, null} + +--- + +## 10. Future Extensions & Roadmap + +### Near-Term (Next 2-4 Weeks) + +| Item | Status | Notes | +| --------------------- | ----------- | ------------------------------------------- | +| Configurable batching | In progress | Single `BATCH_CONFIG.size` value | +| 20-word torture suite | Pending | Decides gender approach and model selection | +| Qwen2.5-3B evaluation | Pending | Download and test | +| Online API testing | Pending | Gemini free tier, DeepSeek, Groq | +| Retry logic | Pending | Exponential backoff on LLM failures | + +### Medium-Term (1-3 Months) + +| Item | Notes | +| ---------------------------- | ------------------------------------------------------ | +| Multi-POS support | Verbs, adjectives, adverbs need prompt variants | +| Multi-language source | German -> French, Italian -> Spanish, etc. | +| Parallel wordlist processing | Run `english/nouns` and `english/verbs` simultaneously | +| Incremental enrichment | Only process new/changed words in a wordlist | +| Model auto-switching | Fallback to online API if local server fails | + +### Long-Term (3-6 Months) + +| Item | Notes | +| ------------------------ | ---------------------------------------------------------------- | +| Batch API discounts | Gemini, Qwen, Azure offer 50% off for 24h SLA | +| GPU rental integration | Script to spin up Vast.ai/RunPod, run pipeline, download results | +| Quality regression tests | Run torture suite on every model change | +| Community contributions | Open-source the pipeline for other language learners | + +--- + +## 11. Decisions Log + +| Date | Decision | Context | Rationale | +| ---------- | ----------------------------- | ---------------------------------------------------- | --------------------------------------------------------------------------- | +| 2026-01-04 | TanStack Router for frontend | Previous project used React Router | Simpler, type-safe routing for the trainer app | +| 2026-01-04 | Vite dev server (no Nginx) | Docker setup for glossa-web | Nginx unnecessary for dev; Vite handles HMR and proxying | +| 2026-01-17 | Backend answer verification | Security vulnerability: correctAnswer exposed in API | Moved verification to server-side, shared schemas | +| 2026-03-26 | Multi-stage Docker builds | glossa-api and glossa-web containers | Smaller images, faster deploys | +| 2026-06-16 | llama.cpp for local LLM | Need local inference on old laptop | GGUF format, OpenAI-compatible API, no dependencies | +| 2026-06-16 | Q4_K_M quantization | Balance size vs quality | Q4_K_M is the community standard for 4-bit inference | +| 2026-06-16 | `-c 2048` context | Default was 4096 | Dictionary entries need ~500 tokens max; frees VRAM | +| 2026-06-16 | `-t 2` physical cores | Default was 4 (HT threads) | Hyperthreading hurts llama.cpp performance | +| 2026-06-17 | Qwen2.5-1.5B as current model | Qwen3.5-4B too slow (47s/word) | 6x speedup (8s/word), quality under evaluation | +| 2026-06-17 | Skip Gemma 4 | E2B Q4_K_M is 3.46GB | Does not fit in 4GB VRAM; lower quants sacrifice quality | +| 2026-06-17 | Skip Ministral-3B | Tokenizer mismatch (Tekken) | Outputs gibberish regardless of template; not fixable without re-conversion | + +--- + +## 12. Known Issues & Dev Notes + +### glossa-web (Frontend) + +| Issue | Details | +| ------------------------ | ----------------------------------------------------------------------------------------- | +| No healthcheck | Vite dev server has no health endpoint. Docker `HEALTHCHECK` cannot verify running state. | +| Valkey memory overcommit | Harmless warning in dev: `vm.overcommit_memory = 1` recommended before production. | + +### Data Pipeline + +| Issue | Details | Severity | +| --------------------------------- | ------------------------------------------------------------------------- | ------------------- | +| Ministral-3B tokenizer mismatch | Tekken tokenizer not properly converted to GGUF. Model outputs gibberish. | Blocker - abandoned | +| Qwen2.5-1.5B gender hallucination | Systematic `neuter` default for Romance languages. | Under evaluation | +| No batching | 1 word = 1 call. System prompt re-processed every time. | In progress | +| No retry logic | LLM parse failures are logged and skipped. | Planned | +| Single POS tested | Only nouns validated. Verbs/adjectives need prompt changes. | Known limitation | + +### Hardware + +| Issue | Details | +| --------------------- | ----------------------------------------------------- | +| GTX 950M VRAM ceiling | 4GB hard limit. Models >3GB risk OOM. | +| Maxwell GPU aging | No Flash Attention support, bandwidth-starved. | +| Laptop thermals | Cannot run 24/7 for weeks. Batch processing required. | + +--- + +## 13. How to Run + +### Prerequisites + +- Node.js + npm +- `tsx` installed globally: `npm install -g tsx` +- llama.cpp built from source +- GGUF model downloaded to `~/Downloads/llama.cpp/models/` + +### Start the LLM Server ```bash -pnpm --filter @lila/pipeline reverse-link +cd ~/Downloads/llama.cpp + +./build/bin/llama-server \ + -m models/qwen2.5-1.5b-instruct-q4_k_m.gguf \ + -ngl 999 \ + -c 2048 \ + -b 512 \ + -ub 512 \ + -t 2 \ + --threads-batch 2 \ + --host 127.0.0.1 \ + --port 8080 \ + --mlock \ + --prio 2 ``` -### 3. Enrich - -> **Note:** Before running this stage, ensure the llama.cpp server is running -> locally. The orchestrator checks for a running server at -> `http://127.0.0.1:8080/health` and exits with instructions if it is not -> reachable. See `llm-setup.md` for setup instructions. - -The enrich stage runs in four ordered sub-stages per entry, designed to build context progressively. All output is written to `pipeline.db` atomically per sub-stage — runs are fully resumable if interrupted. Each model is run once — one model produces one vote per sub-stage. - -**Sub-stage order:** - -1. **`round1_gloss`** — the LLM reviews the existing gloss. If it is clear and learner-friendly, it confirms it. If not, it generates a better one. - -2. **`round1_example`** — the LLM reviews the existing examples. If they are natural and suitable, it confirms them. If not, it generates one better example sentence in the entry language. - -3. **`round1_translations`** — using the verified gloss as context, the LLM reviews each existing translation. Valid translations are confirmed. Invalid ones (wrong language, suffixes, garbled text, wrong sense) are explicitly rejected. Missing languages get a generated translation. - -4. **`round1_cefr`** — using only the validated translations from the previous sub-stage, the LLM votes on the CEFR level for the headword and for each confirmed translation. Rejected translations never reach this sub-stage. - -This ordering ensures the CEFR voting sub-stage only sees clean, verified data. - -All output is written to `pipeline.db` atomically per sub-stage per entry. Interrupted runs resume from the last incomplete sub-stage without losing work. Each model is run once — one model, one vote per sub-stage. - -**Input:** `pipeline.db` — entries after reverse link sync -**Output:** `pipeline.db` — gloss votes, example votes, translation votes, CEFR votes per entry per model - -> **Note:** The tiebreaker is not a standalone script. It runs automatically > as part of the pipeline orchestrator after merge completes. - -### 4. Merge - -Reads all LLM votes from `pipeline.db` and resolves the final value for every field. Writes resolved entries back to `pipeline.db`. - -**Merge rules:** - -- Kaikki source data wins automatically and is never overridden by LLM output -- For CEFR levels: the level with the most votes wins. If no majority is reached, the entry is flagged for the tiebreaker -- For LLM-generated text fields: the candidate with the most votes wins. If no majority is reached, the tiebreaker runs - -**Difficulty mapping:** - -| CEFR | Difficulty | -| ------ | ------------ | -| A1, A2 | easy | -| B1, B2 | intermediate | -| C1, C2 | hard | - -**Input:** `pipeline.db` — LLM votes -**Output:** `pipeline.db` — entries updated with resolved values or flagged status - -### 4b. Tiebreak - -Runs automatically after merge if any entries remain flagged. The script queries `pipeline.db` for flagged entries, identifies which configured models have not yet voted on each entry, and runs those models on the flagged subset only. Merge is re-run after each tiebreaker pass. This repeats until all flagged entries are resolved or no unused models remain. - -If unused models are exhausted and flagged entries remain, the script logs a detailed report showing the exact vote split for each unresolved entry and lists available models from OpenRouter that have not been used. Syncing is blocked until all entries are resolved. To continue, add one or more models to the config and re-run the pipeline — the tiebreaker will pick up automatically. - -> **Note:** The tiebreaker is not a standalone script. It runs automatically as part of the pipeline orchestrator after merge completes. - -### 5. Compare / QA - -Read-only. Generates `COVERAGE.md` with a full breakdown of pipeline output quality per language. Run this after merge to verify output before syncing to the database. - -**Input:** `pipeline.db` — entries with status `final` -**Output:** `COVERAGE.md` - -`COVERAGE.md` reports the following per language: - -- Total entries extracted -- POS breakdown — entry counts for noun, verb, adjective, adverb -- Translation coverage — how many entries have translations in each other language -- CEFR coverage — how many entries have a resolved CEFR level, broken down by level -- Difficulty breakdown — entry counts for easy, intermediate, hard -- Gloss coverage — how many entries have a gloss, broken down by source (Kaikki vs LLM-generated) -- Example coverage — same breakdown as glosses -- LLM model contribution — how many CEFR votes and text candidates each anonymised model contributed - -## Sync - -The sync script transfers all entries with status `final` in `pipeline.db` to the production PostgreSQL database. It is upsert-based and never wipes existing data. For each entry it checks whether a matching record already exists in the target database: - -- **Missing** → insert -- **Present but changed** → update -- **Present and unchanged** → skip - -Run this after all entries are resolved and Compare / QA has been reviewed. +### Run the Pipeline ```bash -pnpm --filter @lila/pipeline sync +cd /path/to/data-pipeline +npx tsx pipeline.ts ``` -The sync script requires a connection string to the target database. Set `DATABASE_URL` in your `.env` file before running. - -## Reports - -The pipeline generates a report at the end of every run. Reports are written to `data-pipeline/reports/` as a JSON file and a markdown file with the same name. The markdown is generated from the JSON and contains identical data. +### Expected Output ``` -data-pipeline/reports/ - 2026-05-03_run-1.json - 2026-05-03_run-1.md +Starting data pipeline... + + step 1: scanning the source files... +Scan complete! Found 1 wordlist(s): + - ENGLISH (nouns) + +... + +Pipeline Summary + Duration: 23.0s + Processed: 3 + Skipped: 0 + Failed: 0 + Total: 3 + Throughput: 0.13 words/sec + +LLM Metrics + Calls: 3 + Avg prompt tokens: 308 + Avg completion tokens: 132 + Avg prompt speed: 549.4 tok/s + Avg completion speed: 18.8 tok/s + +Global data pipeline run completed successfully. ``` -The run name is auto-generated from the date and a counter. Reports are not committed to git. +### Environment Variables (Online Mode) -**Nightly report** contains: +```bash +export DEEPSEEK_API_KEY="sk-..." +export GEMINI_API_KEY="..." +export GROQ_API_KEY="..." +``` -- Entries processed this run vs total -- Entries remaining per stage -- Average processing speed and estimated nights remaining -- `needs_review` count — entries that failed structural validation -- Per-model progress breakdown +Then update `config/llm.ts` to point to the online API URL. -**Final report** (generated when all entries are processed) additionally contains: +--- -- Full vote breakdown per model -- Flagged entries with exact vote splits -- Available unused models from OpenRouter for tiebreaking -- Per-model quality metrics — CEFR agreement rate, field coverage, JSON parse rate +## 14. Roadmap -## Adding a new language +### Phase 1: Batching (Current) -1. Add the language code to `SUPPORTED_LANGUAGE_CODES` in `packages/shared/src/constants.ts` -2. Build shared: `pnpm --filter @lila/shared build` -3. Generate and run a DB migration: `pnpm --filter @lila/db generate` then `pnpm --filter @lila/db migrate` -4. Download the Kaikki JSONL file for the language from kaikki.org -5. Re-run the full pipeline +| Task | Status | Notes | +| ------------------------------------- | ----------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Implement configurable batch size | In progress | Single `BATCH_CONFIG.size` value. Prompt formatting: single word -> word array. Response parsing: keyed JSON object. Retry: 50 -> 25 -> 5 -> 1. | +| Verify batching doesn't break quality | Pending | Run 20-word torture suite on Qwen2.5-1.5B with batch sizes 1, 5, 15. Compare output. | +| Measure speedup vs batch size | Pending | Track throughput at 1, 5, 15 on local hardware. | -## Constants and constraints +**Goal:** Unlock 5-15x speedup on local, unlock free online API tiers. -These values are defined in `packages/shared/src/constants.ts` and enforced by database check constraints. The pipeline filters out any entries that violate them. +--- -| Constant | Values | -| --------------- | ------------------------------------- | -| Languages | `en`, `it`, `de`, `es`, `fr` | -| Parts of speech | `noun`, `verb`, `adjective`, `adverb` | -| CEFR levels | `A1`, `A2`, `B1`, `B2`, `C1`, `C2` | -| Difficulty | `easy`, `intermediate`, `hard` | +### Phase 2: Model Selection -Adding a new value to any of these requires a constants update and a database migration before re-running the pipeline. See **Adding a new language** for the full steps — the same process applies for new parts of speech. +| Task | Status | Notes | +| ------------------------------------------------ | ------- | ------------------------------------------------------------------------------------------- | +| Download Qwen2.5-3B Q4_K_M | Pending | ~1.9GB, fits in 4GB VRAM. | +| Run torture suite on Qwen2.5-3B (batched) | Pending | Check if gender accuracy improves with 2x params. | +| Test Gemini 2.5 Flash-Lite free tier (batched) | Pending | 1,500 req/day x 50 words = 75k words/day. Zero cost. | +| Test DeepSeek V4 Flash free tier (batched) | Pending | 5M tokens free. ~9k words. | +| Test Groq Llama 3.1 8B (batched, paid if needed) | Pending | Fastest option. ~$7 for 100k words. | +| Decide: local vs online, which model | Pending | Criteria: quality >= 90% gender, 100% JSON, sensible definitions. Then speed, then cost. | -## Further extensions +**Goal:** Pick the model and provider for the 100k word run. -These are not part of the current pipeline but are worth considering as the dataset matures: +--- -- **IPA pronunciations** — Kaikki includes IPA transcriptions for most entries. Could be extracted and stored in a `entry_pronunciations` table and displayed in the quiz UI. -- **Audio files** — kaikki.org provides bulk audio file downloads (~20GB) for pronunciations. Could be stored as static files and served alongside the quiz UI. -- **Inflected forms** — Kaikki provides conjugation and declension tables in a `forms` array. Useful for a future grammar-focused quiz mode. -- **Grammatical gender** — Kaikki includes grammatical gender for nouns. Could be stored per entry and used as an additional quiz mechanic. -- **Frequency data** — Word frequency rankings per language from sources like the Google Ngram dataset. Useful for smarter difficulty calibration beyond CEFR levels alone. -- **Additional languages** — The pipeline is language-agnostic. Adding a new language requires downloading its Kaikki JSONL file, a constants update, and a database migration. See **Adding a new language**. +### Phase 3: Scale -## Roadmap +| Task | Status | Notes | +| ------------------------- | ------- | -------------------------------------------------------------------------------------------------------------------------- | +| Run 100k word pipeline | Pending | Estimated time depends on Phase 2 decision: ~10 days (local 1.5B) to ~1.5 days (Gemini batched free) to ~3-4 hours (Groq). | +| Spot-check output quality | Pending | Random sample of 100 entries. | +| Fix gender if needed | Pending | Kaikki lookup post-processing if LLM gender remains unreliable. | +| Handle failures & retries | Pending | Exponential backoff, split-and-retry for batch failures. | -**Current state:** Stage 1 extraction and stage 2 reverse link sync complete and verified on sample data. Stage 3 enrich script written and tested — redesigning to sub-stage architecture for better data quality. llama.cpp running with Qwen3.5-4B. +**Goal:** Complete 100k word dataset. -**Next action:** Rewrite enrich script for sub-stage design. +--- -| Stage | Status | -| --------------- | -------------- | -| 1. Extract | 🔲 not started | -| 2. Reverse link | 🔲 not started | -| 3. Enrich | 🔲 not started | -| 4. Merge | 🔲 not started | -| 4b. Tiebreak | 🔲 not started | -| 5. Compare / QA | 🔲 not started | -| 6. Sync | 🔲 not started | +### Phase 4: Extend -### Stage 1 — Extract `🔄 in progress` +| Task | Status | Notes | +| ---------------------------- | ------- | ------------------------------------------------------------------------------------------------- | +| Multi-POS support | Pending | Verbs, adjectives, adverbs. Each needs prompt variants (conjugations, agreement, etc.). | +| Multi-language source | Pending | German -> French, Italian -> Spanish, etc. Schema already supports any source/target combo. | +| Parallel wordlist processing | Pending | Run `english/nouns` and `english/verbs` simultaneously. | +| Incremental enrichment | Pending | Only process new/changed words in a wordlist. | +| GPU rental integration | Pending | Script to spin up Vast.ai/RunPod, run pipeline, download results. | +| Quality regression tests | Pending | Run torture suite on every model change. | -- [x] Download Kaikki JSONL files for all 5 languages -- [x] Write extraction script -- [x] Write stage 1 validation tests -- [x] Write db schema, init, and import scripts -- [x] Write db import validation tests -- [x] Run sample extraction → `stage-1-extract/output/{lang}.json` -- [ ] Remove sample limit and run full extraction -- [ ] Re-run full import → `pipeline.db` +**Goal:** Generalize pipeline for any language direction and POS. -### Stage 2 — Reverse link sync `🔄 in progress` +--- -- [x] Write reverse link sync script -- [x] Run reverse link sync on sample data → 141 links inserted -- [ ] Run reverse link sync on full data after full extraction +### Backlog (Unscheduled) -### Stage 3 — Enrich `🔄 in progress` - -**Next action:** Rewrite enrich script for sub-stage design. - -- [x] Write initial enrich script (single-prompt design) -- [x] Install llama.cpp and verify server -- [x] Smoke test with sample entries -- [ ] Rewrite enrich script for sub-stage design (round1_gloss, round1_example, round1_translations, round1_cefr) -- [ ] Write tests for enrich sub-stages -- [ ] Run full sample, collect metrics -- [ ] Compare providers (local vs OpenRouter free models) -- [ ] Production run — all entries, all models - -### Stage 4 — Merge `🔲 not started` - -- [ ] Write merge script -- [ ] Write tests -- [ ] Run merge → `pipeline.db` -- [ ] Confirm tiebreaker resolves all flagged entries - -### Stage 4b — Tiebreak `🔲 not started` - -- [ ] Write tiebreak logic -- [ ] Run tiebreaker for all flagged entries -- [ ] Confirm no flagged entries remain before syncing - -### Stage 5 — Compare / QA `🔲 not started` - -- [ ] Write compare script -- [ ] Write tests -- [ ] Run compare → `COVERAGE.md` -- [ ] Review output quality before syncing - -### Stage 6 — Sync `🔲 not started` - -- [ ] Write sync script -- [ ] Write tests -- [ ] Configure `DATABASE_URL` in `.env` -- [ ] Run sync → production PostgreSQL -- [ ] Verify seeded data in production - -### Utilities - -**`sample/`** — Runs the pipeline against a small sample to produce human-readable output for a quick sanity check before committing to a full run. Run this after any script change before running the full pipeline. +| Task | Context | +| -------------------------------- | ------------------------------------------------------------------------------------------- | +| Batch API discounts | Gemini, Qwen, Azure offer 50% off for 24h SLA. Relevant if running recurring large batches. | +| Model auto-switching | Fallback to online API if local server fails mid-run. | +| Community open-source | Clean up, document, publish for other language learners. | +| Prometheus metrics | `--metrics` flag on llama-server for automated performance tracking. | +| `-c 1024` / `-b 256` experiments | Further VRAM optimization on GTX 950M. Low priority if moving to cloud. | diff --git a/documentation/pipeline/ENGLISH_NOUNS.md b/documentation/pipeline/ENGLISH_NOUNS.md index 72fa88a..6c23891 100644 --- a/documentation/pipeline/ENGLISH_NOUNS.md +++ b/documentation/pipeline/ENGLISH_NOUNS.md @@ -76,7 +76,7 @@ example output: "headword": "house", "language": "en", "pos": "noun", - "glosses": ["A building for human habitation."], + "sense": ["A building for human habitation."], "examples": ["They bought a house in the city."], "translations": { "de": [{ "word": "Haus", "gender": "neuter" }], @@ -90,7 +90,7 @@ example output: "headword": "house", "language": "en", "pos": "noun", - "glosses": ["A noble family or lineage."], + "sense": ["A noble family or lineage."], "examples": ["The House of Tudor ruled England."], "translations": { "de": [ @@ -104,7 +104,7 @@ example output: "headword": "bank", "language": "en", "pos": "noun", - "glosses": ["An institution where one can place and borrow money."], + "sense": ["An institution where one can place and borrow money."], "examples": ["She deposited her paycheck at the bank."], "translations": { "de": [{ "word": "Bank", "gender": "feminine" }], @@ -118,7 +118,7 @@ example output: "headword": "bank", "language": "en", "pos": "noun", - "glosses": ["The land alongside a river or lake."], + "sense": ["The land alongside a river or lake."], "examples": ["They picnicked on the bank of the river."], "translations": { "de": [{ "word": "Ufer", "gender": "neuter" }], @@ -132,7 +132,7 @@ example output: "headword": "bank", "language": "en", "pos": "noun", - "glosses": ["A collection or store of something held in reserve."], + "sense": ["A collection or store of something held in reserve."], "examples": ["The hospital keeps a blood bank."], "translations": { "de": [{ "word": "Bank", "gender": "feminine" }] @@ -336,7 +336,7 @@ read to decide, not to archive. Notes: - the miss list needs to get verified/re-worked later on -- the kaikki data contains ipas and links to audio files, add them later if needed, they are not needed now" +- the kaikki data contains ipas and links to audio files, add them later if needed, they are not needed now - use online llms to set the difficulty(cefr) of the not shipped words - add plurals from kaikki/wiktionary - postgres sync to prod db