From 2a6c56ed2330593eb2d37e9faa6c7fbba214f93d Mon Sep 17 00:00:00 2001 From: lila Date: Mon, 6 Jul 2026 14:27:47 +0200 Subject: [PATCH] bugfixing #1 --- data-pipeline/config/batch.ts | 2 - data-pipeline/config/constants.ts | 14 + data-pipeline/pipeline.ts | 25 +- data-pipeline/utils/check-llm-server.ts | 8 + data-pipeline/utils/create-base-json.ts | 16 +- data-pipeline/utils/enrich-word.ts | 17 +- data-pipeline/utils/llm-adapters/gemini.ts | 15 +- .../utils/llm-adapters/openai-compatible.ts | 29 +- data-pipeline/utils/llm-adapters/types.ts | 2 - data-pipeline/utils/merge-enriched-data.ts | 37 +- data-pipeline/utils/pipeline-timer.ts | 2 - documentation/DATA_PIPELINE.md | 402 +++++++++++------- 12 files changed, 300 insertions(+), 269 deletions(-) create mode 100644 data-pipeline/config/constants.ts diff --git a/data-pipeline/config/batch.ts b/data-pipeline/config/batch.ts index 289fe55..e05f603 100644 --- a/data-pipeline/config/batch.ts +++ b/data-pipeline/config/batch.ts @@ -1,3 +1 @@ export const BATCH_CONFIG = { size: 4, maxRetries: 3 } as const; - -//1, 2, 5, 10, 20 diff --git a/data-pipeline/config/constants.ts b/data-pipeline/config/constants.ts new file mode 100644 index 0000000..98731f2 --- /dev/null +++ b/data-pipeline/config/constants.ts @@ -0,0 +1,14 @@ +export const LANG_MAP: Record = { + english: "en", + italian: "it", + german: "de", + french: "fr", + spanish: "es", +}; + +export const POS_MAP: Record = { + nouns: "noun", + verbs: "verb", + adverbs: "adverb", + adjectives: "adjective", +}; diff --git a/data-pipeline/pipeline.ts b/data-pipeline/pipeline.ts index 863c4af..f660478 100644 --- a/data-pipeline/pipeline.ts +++ b/data-pipeline/pipeline.ts @@ -27,7 +27,7 @@ async function main() { console.log("\n step 2: creating necessary output folders..."); ensureOutputFolders(wordlists); - // step 3: check to verify the AI engine is ready before touching anything + // step 3: check to verify the local AI engine is ready before touching anything console.log("\n step 3: verifying local AI engine status..."); await checkLlmServer(); @@ -99,12 +99,7 @@ async function main() { ); const targetFilePath = getWordFilePath(word, wordlist.outputDir); - const enrichedData = mergeEnrichedData( - word, - wordlist.language, - wordlist.pos, - senses, - ); + const enrichedData = mergeEnrichedData(word, senses); writeJsonFile(targetFilePath, enrichedData); @@ -118,17 +113,19 @@ async function main() { } timer.recordProcessed({ - promptTokens: result.metrics.promptTokens / batch.length, - completionTokens: result.metrics.completionTokens / batch.length, - totalTokens: result.metrics.totalTokens / batch.length, - promptTimeMs: result.metrics.promptTimeMs / batch.length, - completionTimeMs: result.metrics.completionTimeMs / batch.length, - promptTokensPerSecond: result.metrics.promptTokensPerSecond, - completionTokensPerSecond: result.metrics.completionTokensPerSecond, + promptTokens: result.metrics.promptTokens, + completionTokens: result.metrics.completionTokens, + totalTokens: result.metrics.totalTokens, + promptTimeMs: result.metrics.promptTimeMs, + completionTimeMs: result.metrics.completionTimeMs, }); } console.log(` ${timer.getWordTiming()}`); + // Show ETA every 5 batches or on the last batch + if (batchNum % 5 === 0 || batchNum === totalBatches) { + console.log(` 📊 ${timer.getEta(unprocessedWords.length)}`); + } } catch (error: unknown) { const errorMessage = error instanceof Error ? error.message : String(error); diff --git a/data-pipeline/utils/check-llm-server.ts b/data-pipeline/utils/check-llm-server.ts index b4127a0..f2de396 100644 --- a/data-pipeline/utils/check-llm-server.ts +++ b/data-pipeline/utils/check-llm-server.ts @@ -1,10 +1,18 @@ +import { LLM_CONFIG } from "../config/llm.js"; + /** * Pings the local llama.cpp server to ensure it's up, running, and has a model loaded. * If the server is offline or still loading, it terminates the pipeline gracefully. + * Skipped entirely when using a cloud provider. */ export async function checkLlmServer( url = "http://127.0.0.1:8080/health", ): Promise { + if (LLM_CONFIG.provider !== "local") { + console.log("🌐 Using cloud provider — skipping local health check."); + return; + } + try { const response = await fetch(url); diff --git a/data-pipeline/utils/create-base-json.ts b/data-pipeline/utils/create-base-json.ts index 2d21ecc..6d7d057 100644 --- a/data-pipeline/utils/create-base-json.ts +++ b/data-pipeline/utils/create-base-json.ts @@ -1,20 +1,6 @@ import fs from "fs"; import path from "path"; - -const LANG_MAP: Record = { - english: "en", - italian: "it", - german: "de", - french: "fr", - spanish: "es", -}; - -const POS_MAP: Record = { - nouns: "noun", - verbs: "verb", - adverbs: "adverb", - adjectives: "adjective", -}; +import { LANG_MAP, POS_MAP } from "../config/constants.js"; /** * Creates the base JSON file with word, language, and pos. diff --git a/data-pipeline/utils/enrich-word.ts b/data-pipeline/utils/enrich-word.ts index d970ee3..36b6b25 100644 --- a/data-pipeline/utils/enrich-word.ts +++ b/data-pipeline/utils/enrich-word.ts @@ -1,4 +1,3 @@ -// utils/enrich-word.ts import { ENRICHMENT_SYSTEM_PROMPT } from "../config/prompt.js"; import { createAdapter } from "./llm-adapters/factory.js"; import { BATCH_CONFIG } from "../config/batch.js"; @@ -26,8 +25,6 @@ interface LlmResponse { totalTokens: number; promptTimeMs: number; completionTimeMs: number; - promptTokensPerSecond: number; - completionTokensPerSecond: number; } export interface EnrichmentResult { @@ -38,13 +35,11 @@ export interface EnrichmentResult { totalTokens: number; promptTimeMs: number; completionTimeMs: number; - promptTokensPerSecond: number; - completionTokensPerSecond: number; }; } /** - * Calls the local LLM with the enrichment prompt. + * Calls the LLM with the enrichment prompt. * Returns the response content and timing metrics. */ async function callLlm(words: string[]): Promise { @@ -157,8 +152,6 @@ export async function enrichWord( totalTokens: llmResponse.totalTokens, promptTimeMs: llmResponse.promptTimeMs, completionTimeMs: llmResponse.completionTimeMs, - promptTokensPerSecond: llmResponse.promptTokensPerSecond, - completionTokensPerSecond: llmResponse.completionTokensPerSecond, }, }; } @@ -225,14 +218,6 @@ export async function enrichWordWithRetry( completionTimeMs: leftResult.metrics.completionTimeMs + rightResult.metrics.completionTimeMs, - promptTokensPerSecond: - (leftResult.metrics.promptTokensPerSecond + - rightResult.metrics.promptTokensPerSecond) / - 2, - completionTokensPerSecond: - (leftResult.metrics.completionTokensPerSecond + - rightResult.metrics.completionTokensPerSecond) / - 2, }; return { results: merged, metrics: mergedMetrics }; diff --git a/data-pipeline/utils/llm-adapters/gemini.ts b/data-pipeline/utils/llm-adapters/gemini.ts index 6fdb2de..3929b4d 100644 --- a/data-pipeline/utils/llm-adapters/gemini.ts +++ b/data-pipeline/utils/llm-adapters/gemini.ts @@ -28,8 +28,6 @@ export class GeminiAdapter implements LlmAdapter { totalTokens: number; promptTimeMs: number; completionTimeMs: number; - promptTokensPerSecond: number; - completionTokensPerSecond: number; }> { const url = `https://generativelanguage.googleapis.com/v1beta/models/${this.model}:generateContent?key=${this.apiKey}`; @@ -72,21 +70,14 @@ export class GeminiAdapter implements LlmAdapter { const promptTokens = json.usageMetadata.promptTokenCount; const completionTokens = json.usageMetadata.candidatesTokenCount; - const totalTokens = json.usageMetadata.totalTokenCount; - - // Gemini doesn't provide timing breakdown, so we estimate - const promptTimeMs = totalTimeMs * 0.3; // rough estimate - const completionTimeMs = totalTimeMs * 0.7; // rough estimate return { content, promptTokens, completionTokens, - totalTokens, - promptTimeMs, - completionTimeMs, - promptTokensPerSecond: promptTokens / (promptTimeMs / 1000), - completionTokensPerSecond: completionTokens / (completionTimeMs / 1000), + totalTokens: json.usageMetadata.totalTokenCount, + promptTimeMs: totalTimeMs * 0.3, + completionTimeMs: totalTimeMs * 0.7, }; } } diff --git a/data-pipeline/utils/llm-adapters/openai-compatible.ts b/data-pipeline/utils/llm-adapters/openai-compatible.ts index f6e50a6..14f0859 100644 --- a/data-pipeline/utils/llm-adapters/openai-compatible.ts +++ b/data-pipeline/utils/llm-adapters/openai-compatible.ts @@ -7,12 +7,7 @@ interface OpenAiResponse { completion_tokens: number; total_tokens: number; }; - timings: { - prompt_ms: number; - predicted_ms: number; - prompt_per_second: number; - predicted_per_second: number; - }; + timings?: { prompt_ms: number; predicted_ms: number }; } export class OpenAiCompatibleAdapter implements LlmAdapter { @@ -36,8 +31,6 @@ export class OpenAiCompatibleAdapter implements LlmAdapter { totalTokens: number; promptTimeMs: number; completionTimeMs: number; - promptTokensPerSecond: number; - completionTokensPerSecond: number; }> { const payload: Record = { messages: [ @@ -61,12 +54,16 @@ export class OpenAiCompatibleAdapter implements LlmAdapter { headers["Authorization"] = `Bearer ${this.apiKey}`; } + const startTime = Date.now(); + const response = await fetch(this.url, { method: "POST", headers, body: JSON.stringify(payload), }); + const totalTimeMs = Date.now() - startTime; + if (!response.ok) { throw new Error(`LLM server responded with status: ${response.status}`); } @@ -78,15 +75,19 @@ export class OpenAiCompatibleAdapter implements LlmAdapter { throw new Error("LLM response content is empty"); } + const promptTokens = json.usage.prompt_tokens; + const completionTokens = json.usage.completion_tokens; + + const promptTimeMs = json.timings?.prompt_ms ?? totalTimeMs * 0.3; + const completionTimeMs = json.timings?.predicted_ms ?? totalTimeMs * 0.7; + return { content, - promptTokens: json.usage.prompt_tokens, - completionTokens: json.usage.completion_tokens, + promptTokens, + completionTokens, totalTokens: json.usage.total_tokens, - promptTimeMs: json.timings.prompt_ms, - completionTimeMs: json.timings.predicted_ms, - promptTokensPerSecond: json.timings.prompt_per_second, - completionTokensPerSecond: json.timings.predicted_per_second, + promptTimeMs, + completionTimeMs, }; } } diff --git a/data-pipeline/utils/llm-adapters/types.ts b/data-pipeline/utils/llm-adapters/types.ts index c0c5b70..ae38a8b 100644 --- a/data-pipeline/utils/llm-adapters/types.ts +++ b/data-pipeline/utils/llm-adapters/types.ts @@ -9,7 +9,5 @@ export interface LlmAdapter { totalTokens: number; promptTimeMs: number; completionTimeMs: number; - promptTokensPerSecond: number; - completionTokensPerSecond: number; }>; } diff --git a/data-pipeline/utils/merge-enriched-data.ts b/data-pipeline/utils/merge-enriched-data.ts index 85035e3..ad722a4 100644 --- a/data-pipeline/utils/merge-enriched-data.ts +++ b/data-pipeline/utils/merge-enriched-data.ts @@ -1,3 +1,5 @@ +import { LLM_CONFIG } from "../config/llm.js"; + export type Language = "en" | "de" | "it" | "es" | "fr"; export type Pos = "noun" | "verb" | "adjective" | "adverb"; export type Gender = "masculine" | "feminine" | "neuter" | null; @@ -24,46 +26,19 @@ export interface EnrichedSense { }; } -const LANG_MAP: Record = { - english: "en", - italian: "it", - german: "de", - french: "fr", - spanish: "es", -}; - -const POS_MAP: Record = { - nouns: "noun", - verbs: "verb", - adverbs: "adverb", - adjectives: "adjective", -}; - /** * Merges skeleton data with enriched LLM senses into the final pipeline output. */ export function mergeEnrichedData( word: string, - rawLanguage: string, - rawPos: string, senses: EnrichedSense[], ): Record { - const language = LANG_MAP[rawLanguage] || (rawLanguage as Language); - const pos = POS_MAP[rawPos] || (rawPos as Pos); - - const fixedSenses = senses.map((sense, index) => ({ - ...sense, - id: `${word}:${language}:${pos}:${index}`, - language, - pos, - })); - return { word, - language, - pos, - senses: fixedSenses, + language: senses[0]?.language ?? "en", + pos: senses[0]?.pos ?? "noun", + senses, enrichedAt: new Date().toISOString(), - model: "qwen3.5-4b-q4_k_m", + model: LLM_CONFIG.model ?? "unknown", }; } diff --git a/data-pipeline/utils/pipeline-timer.ts b/data-pipeline/utils/pipeline-timer.ts index 56683e9..1eecfb0 100644 --- a/data-pipeline/utils/pipeline-timer.ts +++ b/data-pipeline/utils/pipeline-timer.ts @@ -4,8 +4,6 @@ interface LlmMetrics { totalTokens: number; promptTimeMs: number; completionTimeMs: number; - promptTokensPerSecond: number; - completionTokensPerSecond: number; } interface PipelineMetrics { diff --git a/documentation/DATA_PIPELINE.md b/documentation/DATA_PIPELINE.md index f03f6ba..9271551 100644 --- a/documentation/DATA_PIPELINE.md +++ b/documentation/DATA_PIPELINE.md @@ -1,7 +1,7 @@ # Lila Data Pipeline — Technical Documentation > Multilingual dictionary enrichment pipeline for the Lila vocabulary trainer. -> Last updated: 2026-06-17 +> Last updated: 2026-07-06 --- @@ -29,15 +29,16 @@ ## Quick Reference -| What | Where | -| ----------------- | ----------------------------------- | -| Entry point | `pipeline.ts` | -| LLM config | `config/llm.ts` | -| System prompt | `config/prompt.ts` | -| Output schema | `utils/merge-enriched-data.ts` | -| Batch size config | `config/batch.ts` _(planned)_ | -| Current model | `qwen2.5-1.5b-instruct-q4_k_m.gguf` | -| Target scale | 100,000+ words | +| What | Where | +| ------------- | ----------------------------------- | +| Entry point | `pipeline.ts` | +| LLM config | `config/llm.ts` | +| System prompt | `config/prompt.ts` | +| Batch config | `config/batch.ts` | +| Output schema | `utils/merge-enriched-data.ts` | +| LLM adapters | `utils/llm-adapters/` | +| Current model | `qwen2.5-1.5b-instruct-q4_k_m.gguf` | +| Target scale | 100,000+ words | --- @@ -50,7 +51,7 @@ The Lila Data Pipeline is a TypeScript-based batch processing system that enrich - A **CEFR-based difficulty level** (`easy` / `medium` / `hard`) - **Translations** into German, Italian, Spanish, and French, each with grammatical **gender** -The pipeline is designed to scale to **100,000+ words** across multiple languages and parts of speech (nouns, verbs, adjectives, adverbs). It is currently in active development: the core architecture is stable, the LLM integration layer is being evaluated across local and online providers, and a configurable batching system is planned to unlock throughput at scale. +The pipeline is designed to scale to **100,000+ words** across multiple languages and parts of speech (nouns, verbs, adjectives, adverbs). It is currently in active development: the core architecture is stable, the LLM integration layer supports both local and cloud providers via a pluggable adapter pattern, and a configurable batching system with retry/split logic is fully implemented. ### Key Design Principles @@ -59,7 +60,8 @@ The pipeline is designed to scale to **100,000+ words** across multiple language | **Quality first** | Definitions, examples, translations, and gender must be accurate. Speed and cost are secondary. | | **Local-first, cloud-fallback** | Local LLMs (llama.cpp) are the default for cost control and data privacy. Online APIs are evaluated as alternatives for speed. | | **Resumable & idempotent** | Each word writes to its own JSON file. The pipeline skips already-processed words on restart. | -| **Configurable batching** | Batch size (1, 5, 15, 50, etc.) is a single config value. The pipeline adapts without code changes. | +| **Configurable batching** | Batch size is a single config value (`config/batch.ts`). The pipeline adapts without code changes. | +| **Provider-agnostic** | LLM adapters abstract local, OpenRouter, DeepSeek, and Gemini behind a single interface. | ### Open Question: Gender Accuracy @@ -73,14 +75,15 @@ Grammatical gender is currently generated by the LLM as part of the translation No decision made. Gender handling will be determined by the 20-word quality torture suite. -### Current Status (2026-06-17) +### Current Status (2026-07-06) - Core pipeline: scanning, enrichment, merging, verification, writing - Local LLM integration via llama.cpp server (OpenAI-compatible API) +- **Cloud provider adapters**: Gemini, DeepSeek, OpenRouter via `utils/llm-adapters/` +- **Batching with retry/split**: configurable batch size, exponential split-on-failure (4 → 2 → 1) - Schema validation for generated JSON - Progress tracking and timing metrics - **In progress:** Evaluating local models (Qwen2.5-1.5B tested; Qwen2.5-3B download pending) -- **In progress:** Designing configurable batching system - **Pending:** 20-word quality torture suite (will decide gender approach) - **Pending:** Online API evaluation (Gemini free tier, DeepSeek, Groq) @@ -94,17 +97,30 @@ source wordlists -> llama.cpp server (LLM) -> merge senses -> verify schema -> w ### Files at a Glance -| File | Purpose | -| ------------------------------- | ---------------------------------------------------------------- | -| `pipeline.ts` | Orchestrator. Scans sources, loops words, coordinates all stages | -| `config/llm.ts` | API URL, default parameters (`temperature`, `max_tokens`, etc.) | -| `config/prompt.ts` | System prompt sent to the LLM | -| `utils/enrich-word.ts` | Calls LLM, parses response, builds `EnrichedSense[]` | -| `utils/merge-enriched-data.ts` | Merges skeleton + enriched senses into final JSON | -| `utils/verify-enriched-file.ts` | Schema validation (required fields, types, gender enum) | -| `utils/check-llm-server.ts` | Health check before pipeline starts | -| `utils/progress-tracker.ts` | `[current/total]` formatting for console output | -| `utils/pipeline-timer.ts` | Per-word and global timing + LLM token metrics | +| File | Purpose | +| ----------------------------------------- | ----------------------------------------------------------------------------------- | +| `pipeline.ts` | Orchestrator. Scans sources, loops words, coordinates all stages | +| `config/llm.ts` | Provider selection, API URL, model name | +| `config/prompt.ts` | System prompt sent to the LLM | +| `config/batch.ts` | Batch size and max retry count | +| `utils/enrich-word.ts` | Calls LLM via adapter, parses response, builds `EnrichedSense[]`, retry/split logic | +| `utils/merge-enriched-data.ts` | Merges skeleton + enriched senses into final JSON; defines TypeScript schema | +| `utils/verify-enriched-file.ts` | Schema validation (required fields, types, gender enum) | +| `utils/check-llm-server.ts` | Health check before pipeline starts | +| `utils/scanning-source-files.ts` | Discovers wordlists from `source-data/` directory | +| `utils/create-base-json.ts` | Writes skeleton `{word, language, pos}` files | +| `utils/write-json-file.ts` | Atomic `.tmp` → rename writes | +| `utils/check-if-json-exists.ts` | Resumability: checks if word already has enriched senses | +| `utils/create-line-reader.ts` | Streaming line reader for large wordlists | +| `utils/create-output-dirs.ts` | Creates `worddata/{language}/{pos}/` folders | +| `utils/delete-file.ts` | Cleanup helper for failed batches | +| `utils/get-word-file-path.ts` | Path construction helper | +| `utils/progress-tracker.ts` | `[current/total]` formatting for console output | +| `utils/pipeline-timer.ts` | Per-word and global timing + LLM token metrics | +| `utils/llm-adapters/factory.ts` | Creates the right adapter based on `LLM_CONFIG.provider` | +| `utils/llm-adapters/types.ts` | `LlmAdapter` interface | +| `utils/llm-adapters/openai-compatible.ts` | Local llama.cpp, OpenRouter, DeepSeek | +| `utils/llm-adapters/gemini.ts` | Google Gemini native API | ### Scale Target @@ -133,7 +149,7 @@ Existing multilingual dictionaries and translation APIs provide raw word-to-word ### The Target User -A language learner using the Lila vocabulary trainer. They see a word, its definition, an example sentence, and translations with gender - all calibrated to their CEFR level (A1-C2). +A language learner using the Lila vocabulary trainer. They see a word, its definition, an example sentence, and translations with gender — all calibrated to their CEFR level (A1-C2). ### Why Not Use Existing Dictionaries? @@ -153,7 +169,7 @@ The pipeline is **direction-agnostic**. A wordlist is defined by: Current focus: **English -> German/Italian/Spanish/French** -Planned directions include **German -> French**, **Italian -> Spanish**, etc. The LLM prompt and output schema support any combination - the only change is the source wordlist and the target languages specified in the prompt. +Planned directions include **German -> French**, **Italian -> Spanish**, etc. The LLM prompt and output schema support any combination — the only change is the source wordlist and the target languages specified in the prompt. ### Why 100,000+ Words? @@ -182,8 +198,9 @@ Generating 100,000 entries with an LLM introduces risks: | Hallucinated definitions | Low temperature (0.1), strict system prompt, schema validation | | Incorrect grammatical gender | Under evaluation: larger models or external lookup | | Inconsistent difficulty levels | Explicit CEFR mapping in prompt, spot-checking | -| JSON parse failures | Retry logic, schema validation, cleanup on failure | +| JSON parse failures | Retry + split logic, schema validation, cleanup on failure | | Model drift (online APIs) | Version pinning, local fallback | +| Provider downtime | Adapter pattern allows hot-swapping providers | ### Why TypeScript + Node? @@ -195,7 +212,7 @@ Generating 100,000 entries with an LLM introduces risks: ### Why llama.cpp? - **GGUF format**: Single-file models, easy to swap, quantize, and version -- **OpenAI-compatible API**: `/v1/chat/completions` means the same code works for local and online models +- **OpenAI-compatible API**: `/v1/chat/completions` means the same adapter code works for local and online models - **No dependencies**: Self-contained binary, runs on old hardware (tested on GTX 950M) - **Privacy**: Local inference means no data leaves the machine @@ -206,15 +223,16 @@ Generating 100,000 entries with an LLM introduces risks: ### Pipeline Flow ``` -Scan sources -> Check LLM -> Loop words -> Skip if exists -> Create skeleton - -> Call LLM -> Parse JSON -> Merge -> Write atomically -> Verify schema +Scan sources -> Check LLM -> Loop wordlists -> Stream words -> Skip processed + -> Create skeletons (batch) -> Call LLM -> Parse JSON -> Retry/split on failure + -> Merge -> Write atomically -> Verify schema -> Log metrics ``` ### Resumability -- **Skip existing**: Checks if `{word}.json` exists with non-empty `senses` -- **Atomic writes**: `.tmp` -> rename, no partial files on crash -- **Cleanup on failure**: Deletes incomplete file, continues to next word +- **Skip existing**: `check-if-json-exists.ts` checks if `{word}.json` exists with non-empty `senses` +- **Atomic writes**: `.tmp` -> rename in `write-json-file.ts`, no partial files on crash +- **Cleanup on failure**: Deletes skeleton files for failed batches, continues to next batch ### Directory Structure @@ -222,32 +240,54 @@ Scan sources -> Check LLM -> Loop words -> Skip if exists -> Create skeleton data-pipeline/ |-- pipeline.ts # Entry point / orchestrator |-- config/ -| |-- llm.ts # API URL, model params +| |-- llm.ts # Provider, API URL, model name | |-- prompt.ts # System prompt -|-- utils/ # See source files (provided separately) +| |-- batch.ts # Batch size and retry config +|-- utils/ +| |-- enrich-word.ts # LLM call, parse, retry/split +| |-- merge-enriched-data.ts # Schema types + merge logic +| |-- verify-enriched-file.ts # Schema validation +| |-- check-llm-server.ts # Health check +| |-- scanning-source-files.ts # Source discovery +| |-- create-base-json.ts # Skeleton writer +| |-- write-json-file.ts # Atomic JSON writer +| |-- check-if-json-exists.ts # Resumability check +| |-- create-line-reader.ts # Streaming file reader +| |-- create-output-dirs.ts # Directory creation +| |-- delete-file.ts # Cleanup helper +| |-- get-word-file-path.ts # Path helper +| |-- progress-tracker.ts # Console progress formatting +| |-- pipeline-timer.ts # Timing + token metrics +| |-- llm-adapters/ +| |-- factory.ts # Adapter selection +| |-- types.ts # LlmAdapter interface +| |-- openai-compatible.ts # Local, OpenRouter, DeepSeek +| |-- gemini.ts # Google Gemini |-- source-data/ | |-- {language}/ -| |-- {pos} # One word per line, no extension +| |-- {pos} # One word per line, no extension |-- worddata/ |-- {language}/ |-- {pos}/ - |-- {word}.json # One self-contained file per word + |-- {word}.json # One self-contained file per word +|-- kaikki-source-files/ # Wiktionary dumps for gender lookup (planned) ``` ### Output Schema -Each `.json` file contains: `word`, `language`, `pos`, `senses[]` (each with `sense`, `example`, `difficulty_level`, `translations` per target language), `enrichedAt`, `model`. +Each `.json` file contains: `word`, `language`, `pos`, `senses[]` (each with `id`, `sense`, `example`, `difficulty_level`, `translations` per target language), `enrichedAt`, `model`. Full TypeScript interfaces: `utils/merge-enriched-data.ts`. ### Error Handling -| Failure | Behavior | -| ----------------------- | ------------------------------ | -| LLM server offline | Hard fail at startup | -| LLM returns bad JSON | Log, delete skeleton, continue | -| Schema validation fails | Log warnings, keep file | -| Individual word fails | Does not stop pipeline | +| Failure | Behavior | +| ------------------------------ | ------------------------------------------------------- | +| LLM server offline | Hard fail at startup (`check-llm-server.ts`) | +| LLM returns bad JSON | Retry up to 3 times, then split batch. Log and continue | +| Schema validation fails | Log warnings, keep file | +| Individual batch fails | Does not stop pipeline; cleans up skeletons | +| Individual word fails (size 1) | Log and continue to next word | ### Metrics @@ -268,19 +308,23 @@ Per-run: words processed/skipped/failed, duration, throughput, LLM token counts ### Configuration -| File | Purpose | -| ------------------ | -------------------------------------------------------------------------- | -| `config/llm.ts` | `LLM_API_URL`, `LLM_DEFAULT_PARAMS` (`temperature`, `top_p`, `max_tokens`) | -| `config/prompt.ts` | System prompt with CEFR mapping, required fields, example output | +| File | Purpose | +| ------------------ | ---------------------------------------------------------------- | ------------ | ---------- | ------------------------- | +| `config/llm.ts` | `provider` (`local` | `openrouter` | `deepseek` | `gemini`), `url`, `model` | +| `config/prompt.ts` | System prompt with CEFR mapping, required fields, example output | +| `config/batch.ts` | `BATCH_CONFIG.size` (words per call), `maxRetries` | ### Key Modules -| File | Responsibility | -| ------------------------------- | ------------------------------------------------------------------------------------------ | -| `utils/enrich-word.ts` | Calls LLM, strips markdown, parses JSON array, builds `EnrichedSense[]` with composite IDs | -| `utils/merge-enriched-data.ts` | Merges skeleton `{word, language, pos}` with LLM senses, adds `enrichedAt` and `model` | -| `utils/verify-enriched-file.ts` | Schema validation: required fields, array lengths, gender enum, translation structure | -| `utils/pipeline-timer.ts` | Tracks per-word and global metrics (duration, tokens, throughput) | +| File | Responsibility | +| ----------------------------------------- | ------------------------------------------------------------------------------------------------------------------------- | +| `utils/enrich-word.ts` | Calls LLM via adapter, strips markdown, parses JSON array, builds `EnrichedSense[]` with composite IDs, retry/split logic | +| `utils/merge-enriched-data.ts` | Merges skeleton `{word, language, pos}` with LLM senses, adds `enrichedAt` and `model` | +| `utils/verify-enriched-file.ts` | Schema validation: required fields, array lengths, gender enum, translation structure | +| `utils/pipeline-timer.ts` | Tracks per-word and global metrics (duration, tokens, throughput) | +| `utils/llm-adapters/factory.ts` | Creates adapter based on `LLM_CONFIG.provider` | +| `utils/llm-adapters/openai-compatible.ts` | OpenAI chat completions API for local llama.cpp, OpenRouter, DeepSeek | +| `utils/llm-adapters/gemini.ts` | Google Gemini `generateContent` API | ### Current Model @@ -375,9 +419,9 @@ The server flags evolved through trial and error on the target hardware (Intel i ### Known Limitations (Current) - **Gender accuracy**: Qwen2.5-1.5B systematically defaults to `neuter` for Romance languages. Under evaluation whether larger models fix this. -- **No batching**: One word = one LLM call. System prompt re-processed every time. -- **No retry logic**: LLM parse failures are logged and skipped, not retried. - **Single POS**: Only nouns tested. Verbs/adjectives/adverbs need prompt adjustments. +- **Hardcoded model name**: `merge-enriched-data.ts` hardcodes `"qwen3.5-4b-q4_k_m"` regardless of actual model used. +- **Duplicated mappings**: `LANG_MAP`/`POS_MAP` exist in three separate files. --- @@ -413,7 +457,7 @@ Models are evaluated on three criteria in order of priority: **quality** (defini ### 5.2 Online API Options -Evaluated as fallbacks if local models fail quality or speed targets. All support OpenAI-compatible API. +Evaluated as fallbacks if local models fail quality or speed targets. All support OpenAI-compatible API (except Gemini, which has a native adapter). | Provider | Model | Input $/1M | Output $/1M | Free Tier | Rate Limit | Est. Cost (100k words) | Est. Time | | ------------------- | -------------------- | ---------- | ----------- | ------------- | ---------------- | ---------------------- | ------------------- | @@ -461,7 +505,7 @@ on Qwen2.5-3B (local) **Quality gates:** -- > =90% gender accuracy (de/it/es/fr) +- > = 90% gender accuracy (de/it/es/fr) - 100% JSON parse rate - No hallucinated definitions on polysemous words - Natural, contextually appropriate example sentences @@ -484,7 +528,7 @@ Grammatical gender is embedded in the `translations` object of each sense: } ``` -Early testing with **Qwen2.5-1.5B** showed systematic failure: the model defaults to `neuter` for any translation where it is uncertain. This is particularly broken for Romance languages (Italian, Spanish, French), which do not have a neuter grammatical gender at all - only masculine and feminine. +Early testing with **Qwen2.5-1.5B** showed systematic failure: the model defaults to `neuter` for any translation where it is uncertain. This is particularly broken for Romance languages (Italian, Spanish, French), which do not have a neuter grammatical gender at all — only masculine and feminine. Whether this is a **model size issue** (1.5B too small to retain gender facts) or a **training data gap** (Qwen2.5 family lacks gender-annotated multilingual data) is unresolved. Pending the Qwen2.5-3B evaluation. @@ -528,27 +572,24 @@ At 1 word per call, 100,000 words = 100,000 LLM requests. Each call re-processes Single config point controls batch size everywhere: ```typescript -// config/batch.ts (planned) -export const BATCH_CONFIG = { - size: 5, // Change to 15 or 50 to test - get maxTokens() { - return Math.ceil(this.size * 250 * 1.2); // 250 tok/word + 20% buffer - }, -} as const; +// config/batch.ts +export const BATCH_CONFIG = { size: 4, maxRetries: 3 } as const; ``` +Change `size` to 1, 2, 5, 10, 20, etc. The pipeline adapts without code changes. + ### Prompt Structure **Single word:** ``` -Word: house +["house"] ``` -**Batch of 5:** +**Batch of 4:** ``` -Words: ["house", "car", "tree", "water", "book"] +["house", "car", "tree", "water"] ``` LLM returns a JSON object with word keys: @@ -558,29 +599,28 @@ LLM returns a JSON object with word keys: "house": [ { "sense": "...", "example": "...", ... } ], "car": [ { ... } ], "tree": [ { ... } ], - "water": [ { ... } ], - "book": [ { ... } ] + "water": [ { ... } ] } ``` -### Retry Strategy +### Retry & Split Strategy If a batch fails (bad JSON, missing key, etc.): ``` -Batch of 50 fails +Batch of 4 fails (3 retries exhausted) | v -Retry as 2 batches of 25 +Split into 2 batches of 2 | v -If a 25 fails, retry as 5 batches of 5 +If a batch of 2 fails (3 retries), split into 2 batches of 1 | v -If a 5 fails, retry as individual words (fallback) +If a single word fails (3 retries), log and skip ``` -This gives resilience without losing the speed benefit of large batches. +This gives resilience without losing the speed benefit of large batches. The `maxRetries` config controls how many attempts are made before splitting. ### Expected Impact by Environment @@ -681,13 +721,14 @@ For each word and each candidate model: ### Near-Term (Next 2-4 Weeks) -| Item | Status | Notes | -| --------------------- | ----------- | ------------------------------------------- | -| Configurable batching | In progress | Single `BATCH_CONFIG.size` value | -| 20-word torture suite | Pending | Decides gender approach and model selection | -| Qwen2.5-3B evaluation | Pending | Download and test | -| Online API testing | Pending | Gemini free tier, DeepSeek, Groq | -| Retry logic | Pending | Exponential backoff on LLM failures | +| Item | Status | Notes | +| ------------------------ | ------------ | --------------------------------------------------------------- | +| Configurable batching | **Complete** | Single `BATCH_CONFIG.size` value, retry/split logic implemented | +| 20-word torture suite | Pending | Decides gender approach and model selection | +| Qwen2.5-3B evaluation | Pending | Download and test | +| Online API testing | Pending | Gemini free tier, DeepSeek, Groq | +| Fix hardcoded model name | Pending | `merge-enriched-data.ts` hardcodes `"qwen3.5-4b"` | +| Extract shared constants | Pending | `LANG_MAP`/`POS_MAP` duplicated in 3 files | ### Medium-Term (1-3 Months) @@ -697,7 +738,7 @@ For each word and each candidate model: | Multi-language source | German -> French, Italian -> Spanish, etc. | | Parallel wordlist processing | Run `english/nouns` and `english/verbs` simultaneously | | Incremental enrichment | Only process new/changed words in a wordlist | -| Model auto-switching | Fallback to online API if local server fails | +| Model auto-switching | Fallback to online API if local server fails mid-run | ### Long-Term (3-6 Months) @@ -712,19 +753,21 @@ For each word and each candidate model: ## 11. Decisions Log -| Date | Decision | Context | Rationale | -| ---------- | ----------------------------- | ---------------------------------------------------- | --------------------------------------------------------------------------- | -| 2026-01-04 | TanStack Router for frontend | Previous project used React Router | Simpler, type-safe routing for the trainer app | -| 2026-01-04 | Vite dev server (no Nginx) | Docker setup for glossa-web | Nginx unnecessary for dev; Vite handles HMR and proxying | -| 2026-01-17 | Backend answer verification | Security vulnerability: correctAnswer exposed in API | Moved verification to server-side, shared schemas | -| 2026-03-26 | Multi-stage Docker builds | glossa-api and glossa-web containers | Smaller images, faster deploys | -| 2026-06-16 | llama.cpp for local LLM | Need local inference on old laptop | GGUF format, OpenAI-compatible API, no dependencies | -| 2026-06-16 | Q4_K_M quantization | Balance size vs quality | Q4_K_M is the community standard for 4-bit inference | -| 2026-06-16 | `-c 2048` context | Default was 4096 | Dictionary entries need ~500 tokens max; frees VRAM | -| 2026-06-16 | `-t 2` physical cores | Default was 4 (HT threads) | Hyperthreading hurts llama.cpp performance | -| 2026-06-17 | Qwen2.5-1.5B as current model | Qwen3.5-4B too slow (47s/word) | 6x speedup (8s/word), quality under evaluation | -| 2026-06-17 | Skip Gemma 4 | E2B Q4_K_M is 3.46GB | Does not fit in 4GB VRAM; lower quants sacrifice quality | -| 2026-06-17 | Skip Ministral-3B | Tokenizer mismatch (Tekken) | Outputs gibberish regardless of template; not fixable without re-conversion | +| Date | Decision | Context | Rationale | +| ---------- | --------------------------------- | ---------------------------------------------------- | --------------------------------------------------------------------------- | +| 2026-01-04 | TanStack Router for frontend | Previous project used React Router | Simpler, type-safe routing for the trainer app | +| 2026-01-04 | Vite dev server (no Nginx) | Docker setup for glossa-web | Nginx unnecessary for dev; Vite handles HMR and proxying | +| 2026-01-17 | Backend answer verification | Security vulnerability: correctAnswer exposed in API | Moved verification to server-side, shared schemas | +| 2026-03-26 | Multi-stage Docker builds | glossa-api and glossa-web containers | Smaller images, faster deploys | +| 2026-06-16 | llama.cpp for local LLM | Need local inference on old laptop | GGUF format, OpenAI-compatible API, no dependencies | +| 2026-06-16 | Q4_K_M quantization | Balance size vs quality | Q4_K_M is the community standard for 4-bit inference | +| 2026-06-16 | `-c 2048` context | Default was 4096 | Dictionary entries need ~500 tokens max; frees VRAM | +| 2026-06-16 | `-t 2` physical cores | Default was 4 (HT threads) | Hyperthreading hurts llama.cpp performance | +| 2026-06-17 | Qwen2.5-1.5B as current model | Qwen3.5-4B too slow (47s/word) | 6x speedup (8s/word), quality under evaluation | +| 2026-06-17 | Skip Gemma 4 | E2B Q4_K_M is 3.46GB | Does not fit in 4GB VRAM; lower quants sacrifice quality | +| 2026-06-17 | Skip Ministral-3B | Tokenizer mismatch (Tekken) | Outputs gibberish regardless of template; not fixable without re-conversion | +| 2026-07-06 | Adapter pattern for LLM providers | Need to evaluate local vs cloud | `utils/llm-adapters/` with factory + types + per-provider implementations | +| 2026-07-06 | Retry + split batching | LLM JSON parse failures on larger batches | `enrichWordWithRetry` retries 3 times, then halves batch until size 1 | --- @@ -739,13 +782,15 @@ For each word and each candidate model: ### Data Pipeline -| Issue | Details | Severity | -| --------------------------------- | ------------------------------------------------------------------------- | ------------------- | -| Ministral-3B tokenizer mismatch | Tekken tokenizer not properly converted to GGUF. Model outputs gibberish. | Blocker - abandoned | -| Qwen2.5-1.5B gender hallucination | Systematic `neuter` default for Romance languages. | Under evaluation | -| No batching | 1 word = 1 call. System prompt re-processed every time. | In progress | -| No retry logic | LLM parse failures are logged and skipped. | Planned | -| Single POS tested | Only nouns validated. Verbs/adjectives need prompt changes. | Known limitation | +| Issue | Details | Severity | +| --------------------------------- | ----------------------------------------------------------------------------------------------- | -------------------------------- | +| Ministral-3B tokenizer mismatch | Tekken tokenizer not properly converted to GGUF. Model outputs gibberish. | Blocker - abandoned | +| Qwen2.5-1.5B gender hallucination | Systematic `neuter` default for Romance languages. | Under evaluation | +| Hardcoded model name | `merge-enriched-data.ts` always writes `"qwen3.5-4b-q4_k_m"` regardless of actual model. | Minor - fix before production | +| Duplicated LANG_MAP/POS_MAP | Identical mapping objects in `create-base-json.ts`, `merge-enriched-data.ts`, `enrich-word.ts`. | Minor - refactor risk | +| OpenAI-compatible timings | `json.timings` is llama.cpp-specific. Will break for OpenRouter/DeepSeek. | Medium - needs graceful fallback | +| Single POS tested | Only nouns validated. Verbs/adjectives need prompt changes. | Known limitation | +| Batch metrics averaging | Split-and-merge averages tokens/sec instead of weighting by token count. | Minor - summary stats only | ### Hardware @@ -763,10 +808,16 @@ For each word and each candidate model: - Node.js + npm - `tsx` installed globally: `npm install -g tsx` -- llama.cpp built from source -- GGUF model downloaded to `~/Downloads/llama.cpp/models/` +- llama.cpp built from source (for local mode) +- GGUF model downloaded to `~/Downloads/llama.cpp/models/` (for local mode) +- API keys set as environment variables (for cloud mode): + ```bash + export DEEPSEEK_API_KEY="sk-..." + export GEMINI_API_KEY="..." + export OPENROUTER_API_KEY="..." + ``` -### Start the LLM Server +### Start the LLM Server (Local Mode) ```bash cd ~/Downloads/llama.cpp @@ -785,6 +836,26 @@ cd ~/Downloads/llama.cpp --prio 2 ``` +### Configure Provider + +Edit `config/llm.ts`: + +```typescript +// Local +export const LLM_CONFIG = { + provider: "local" as const, + url: "http://127.0.0.1:8080/v1/chat/completions", + model: undefined, +}; + +// Gemini +export const LLM_CONFIG = { + provider: "gemini" as const, + url: "", + model: "gemini-2.5-flash-lite", +}; +``` + ### Run the Pipeline ```bash @@ -798,50 +869,56 @@ npx tsx pipeline.ts Starting data pipeline... step 1: scanning the source files... -Scan complete! Found 1 wordlist(s): - - ENGLISH (nouns) +✅ Scan complete! Found 1 wordlist(s): + • ENGLISH (nouns) -... + step 2: creating necessary output folders... +✅ All required output directories have been verified and created successfully. -Pipeline Summary - Duration: 23.0s - Processed: 3 + step 3: verifying local AI engine status... +🟢 Local AI engine is connected and ready for inference! + + step 4: looping through the wordlists... + +Reading list: [ENGLISH] -> [NOUNS] + + Batch 1/1: [house, car, tree, water] + [1/4] (0 failed) Enriched and saved: house.json + [2/4] (0 failed) Enriched and saved: car.json + [3/4] (0 failed) Enriched and saved: tree.json + [4/4] (0 failed) Enriched and saved: water.json + ⏱️ Word took 8.2s + +⏱️ Pipeline Summary + Duration: 32.8s + Processed: 4 Skipped: 0 Failed: 0 - Total: 3 - Throughput: 0.13 words/sec + Total: 4 + Throughput: 0.12 words/sec -LLM Metrics - Calls: 3 - Avg prompt tokens: 308 - Avg completion tokens: 132 - Avg prompt speed: 549.4 tok/s +🤖 LLM Metrics + Calls: 1 + Avg prompt tokens: 312 + Avg completion tokens: 524 + Avg prompt speed: 548.2 tok/s Avg completion speed: 18.8 tok/s Global data pipeline run completed successfully. ``` -### Environment Variables (Online Mode) - -```bash -export DEEPSEEK_API_KEY="sk-..." -export GEMINI_API_KEY="..." -export GROQ_API_KEY="..." -``` - -Then update `config/llm.ts` to point to the online API URL. - --- ## 14. Roadmap -### Phase 1: Batching (Current) +### Phase 1: Batching (Complete) -| Task | Status | Notes | -| ------------------------------------- | ----------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------- | -| Implement configurable batch size | In progress | Single `BATCH_CONFIG.size` value. Prompt formatting: single word -> word array. Response parsing: keyed JSON object. Retry: 50 -> 25 -> 5 -> 1. | -| Verify batching doesn't break quality | Pending | Run 20-word torture suite on Qwen2.5-1.5B with batch sizes 1, 5, 15. Compare output. | -| Measure speedup vs batch size | Pending | Track throughput at 1, 5, 15 on local hardware. | +| Task | Status | Notes | +| ------------------------------------- | ------------ | ------------------------------------------------------------------------------------ | +| Implement configurable batch size | **Complete** | `config/batch.ts` with `size` and `maxRetries` | +| Implement retry + split logic | **Complete** | `enrichWordWithRetry`: 3 retries, then halve batch | +| Verify batching doesn't break quality | Pending | Run 20-word torture suite on Qwen2.5-1.5B with batch sizes 1, 5, 15. Compare output. | +| Measure speedup vs batch size | Pending | Track throughput at 1, 5, 15 on local hardware. | **Goal:** Unlock 5-15x speedup on local, unlock free online API tiers. @@ -849,14 +926,14 @@ Then update `config/llm.ts` to point to the online API URL. ### Phase 2: Model Selection -| Task | Status | Notes | -| ------------------------------------------------ | ------- | ------------------------------------------------------------------------------------------- | -| Download Qwen2.5-3B Q4_K_M | Pending | ~1.9GB, fits in 4GB VRAM. | -| Run torture suite on Qwen2.5-3B (batched) | Pending | Check if gender accuracy improves with 2x params. | -| Test Gemini 2.5 Flash-Lite free tier (batched) | Pending | 1,500 req/day x 50 words = 75k words/day. Zero cost. | -| Test DeepSeek V4 Flash free tier (batched) | Pending | 5M tokens free. ~9k words. | -| Test Groq Llama 3.1 8B (batched, paid if needed) | Pending | Fastest option. ~$7 for 100k words. | -| Decide: local vs online, which model | Pending | Criteria: quality >= 90% gender, 100% JSON, sensible definitions. Then speed, then cost. | +| Task | Status | Notes | +| ------------------------------------------------ | ------- | ---------------------------------------------------------------------------------------- | +| Download Qwen2.5-3B Q4_K_M | Pending | ~1.9GB, fits in 4GB VRAM. | +| Run torture suite on Qwen2.5-3B (batched) | Pending | Check if gender accuracy improves with 2x params. | +| Test Gemini 2.5 Flash-Lite free tier (batched) | Pending | 1,500 req/day x 50 words = 75k words/day. Zero cost. | +| Test DeepSeek V4 Flash free tier (batched) | Pending | 5M tokens free. ~9k words. | +| Test Groq Llama 3.1 8B (batched, paid if needed) | Pending | Fastest option. ~$7 for 100k words. | +| Decide: local vs online, which model | Pending | Criteria: quality >= 90% gender, 100% JSON, sensible definitions. Then speed, then cost. | **Goal:** Pick the model and provider for the 100k word run. @@ -877,14 +954,14 @@ Then update `config/llm.ts` to point to the online API URL. ### Phase 4: Extend -| Task | Status | Notes | -| ---------------------------- | ------- | ------------------------------------------------------------------------------------------------- | -| Multi-POS support | Pending | Verbs, adjectives, adverbs. Each needs prompt variants (conjugations, agreement, etc.). | -| Multi-language source | Pending | German -> French, Italian -> Spanish, etc. Schema already supports any source/target combo. | -| Parallel wordlist processing | Pending | Run `english/nouns` and `english/verbs` simultaneously. | -| Incremental enrichment | Pending | Only process new/changed words in a wordlist. | -| GPU rental integration | Pending | Script to spin up Vast.ai/RunPod, run pipeline, download results. | -| Quality regression tests | Pending | Run torture suite on every model change. | +| Task | Status | Notes | +| ---------------------------- | ------- | ------------------------------------------------------------------------------------------- | +| Multi-POS support | Pending | Verbs, adjectives, adverbs. Each needs prompt variants (conjugations, agreement, etc.). | +| Multi-language source | Pending | German -> French, Italian -> Spanish, etc. Schema already supports any source/target combo. | +| Parallel wordlist processing | Pending | Run `english/nouns` and `english/verbs` simultaneously. | +| Incremental enrichment | Pending | Only process new/changed words in a wordlist. | +| GPU rental integration | Pending | Script to spin up Vast.ai/RunPod, run pipeline, download results. | +| Quality regression tests | Pending | Run torture suite on every model change. | **Goal:** Generalize pipeline for any language direction and POS. @@ -892,10 +969,13 @@ Then update `config/llm.ts` to point to the online API URL. ### Backlog (Unscheduled) -| Task | Context | -| -------------------------------- | ------------------------------------------------------------------------------------------- | -| Batch API discounts | Gemini, Qwen, Azure offer 50% off for 24h SLA. Relevant if running recurring large batches. | -| Model auto-switching | Fallback to online API if local server fails mid-run. | -| Community open-source | Clean up, document, publish for other language learners. | -| Prometheus metrics | `--metrics` flag on llama-server for automated performance tracking. | -| `-c 1024` / `-b 256` experiments | Further VRAM optimization on GTX 950M. Low priority if moving to cloud. | +| Task | Context | +| ------------------------------------------- | ------------------------------------------------------------------------------------------- | +| Batch API discounts | Gemini, Qwen, Azure offer 50% off for 24h SLA. Relevant if running recurring large batches. | +| Model auto-switching | Fallback to online API if local server fails mid-run. | +| Community open-source | Clean up, document, publish for other language learners. | +| Prometheus metrics | `--metrics` flag on llama-server for automated performance tracking. | +| `-c 1024` / `-b 256` experiments | Further VRAM optimization on GTX 950M. Low priority if moving to cloud. | +| Extract shared LANG_MAP/POS_MAP | Single source of truth for language/pos mappings. | +| Fix hardcoded model name | Pass actual model name through enrichment chain. | +| Graceful timing fallback for cloud adapters | Handle missing `timings` field in OpenRouter/DeepSeek responses. |