forgejo-lila/lila
1
0
Fork 0
Code Issues 50 Pull requests Projects 1 Releases Packages Wiki Activity Actions

bugfixing #1

Browse source
This commit is contained in:
lila 2026-07-06 14:27:47 +02:00
parent afd28d934e
commit 2a6c56ed23
12 changed files with 300 additions and 269 deletions
Download patch file Download diff file Expand all files Collapse all files

2
data-pipeline/config/batch.ts
View file

@ -1,3 +1 @@
export const BATCH_CONFIG = { size: 4, maxRetries: 3 } as const;
//1, 2, 5, 10, 20

14
data-pipeline/config/constants.ts Normal file
View file

@ -0,0 +1,14 @@
export const LANG_MAP: Record<string, string> = {
english: "en",
italian: "it",
german: "de",
french: "fr",
spanish: "es",
};
export const POS_MAP: Record<string, string> = {
nouns: "noun",
verbs: "verb",
adverbs: "adverb",
adjectives: "adjective",
};

25
data-pipeline/pipeline.ts
View file

@ -27,7 +27,7 @@ async function main() {
console.log("\n step 2: creating necessary output folders...");
ensureOutputFolders(wordlists);
// step 3: check to verify the AI engine is ready before touching anything
// step 3: check to verify the local AI engine is ready before touching anything
console.log("\n step 3: verifying local AI engine status...");
await checkLlmServer();
@ -99,12 +99,7 @@ async function main() {
);
const targetFilePath = getWordFilePath(word, wordlist.outputDir);
const enrichedData = mergeEnrichedData(
word,
wordlist.language,
wordlist.pos,
senses,
);
const enrichedData = mergeEnrichedData(word, senses);
writeJsonFile(targetFilePath, enrichedData);
@ -118,17 +113,19 @@ async function main() {
}
timer.recordProcessed({
promptTokens: result.metrics.promptTokens / batch.length,
completionTokens: result.metrics.completionTokens / batch.length,
totalTokens: result.metrics.totalTokens / batch.length,
promptTimeMs: result.metrics.promptTimeMs / batch.length,
completionTimeMs: result.metrics.completionTimeMs / batch.length,
promptTokensPerSecond: result.metrics.promptTokensPerSecond,
completionTokensPerSecond: result.metrics.completionTokensPerSecond,
promptTokens: result.metrics.promptTokens,
completionTokens: result.metrics.completionTokens,
totalTokens: result.metrics.totalTokens,
promptTimeMs: result.metrics.promptTimeMs,
completionTimeMs: result.metrics.completionTimeMs,
});
}
console.log(` ${timer.getWordTiming()}`);
// Show ETA every 5 batches or on the last batch
if (batchNum % 5 === 0 || batchNum === totalBatches) {
console.log(` 📊 ${timer.getEta(unprocessedWords.length)}`);
}
} catch (error: unknown) {
const errorMessage =
error instanceof Error ? error.message : String(error);

8
data-pipeline/utils/check-llm-server.ts
View file

@ -1,10 +1,18 @@
import { LLM_CONFIG } from "../config/llm.js";
/**
* Pings the local llama.cpp server to ensure it's up, running, and has a model loaded.
* If the server is offline or still loading, it terminates the pipeline gracefully.
* Skipped entirely when using a cloud provider.
*/
export async function checkLlmServer(
url = "http://127.0.0.1:8080/health",
): Promise<void> {
if (LLM_CONFIG.provider !== "local") {
console.log("🌐 Using cloud provider — skipping local health check.");
return;
}
try {
const response = await fetch(url);

16
data-pipeline/utils/create-base-json.ts
View file

@ -1,20 +1,6 @@
import fs from "fs";
import path from "path";
const LANG_MAP: Record<string, string> = {
english: "en",
italian: "it",
german: "de",
french: "fr",
spanish: "es",
};
const POS_MAP: Record<string, string> = {
nouns: "noun",
verbs: "verb",
adverbs: "adverb",
adjectives: "adjective",
};
import { LANG_MAP, POS_MAP } from "../config/constants.js";
/**
* Creates the base JSON file with word, language, and pos.

17
data-pipeline/utils/enrich-word.ts
View file

@ -1,4 +1,3 @@
// utils/enrich-word.ts
import { ENRICHMENT_SYSTEM_PROMPT } from "../config/prompt.js";
import { createAdapter } from "./llm-adapters/factory.js";
import { BATCH_CONFIG } from "../config/batch.js";
@ -26,8 +25,6 @@ interface LlmResponse {
totalTokens: number;
promptTimeMs: number;
completionTimeMs: number;
promptTokensPerSecond: number;
completionTokensPerSecond: number;
}
export interface EnrichmentResult {
@ -38,13 +35,11 @@ export interface EnrichmentResult {
totalTokens: number;
promptTimeMs: number;
completionTimeMs: number;
promptTokensPerSecond: number;
completionTokensPerSecond: number;
};
}
/**
* Calls the local LLM with the enrichment prompt.
* Calls the LLM with the enrichment prompt.
* Returns the response content and timing metrics.
*/
async function callLlm(words: string[]): Promise<LlmResponse> {
@ -157,8 +152,6 @@ export async function enrichWord(
totalTokens: llmResponse.totalTokens,
promptTimeMs: llmResponse.promptTimeMs,
completionTimeMs: llmResponse.completionTimeMs,
promptTokensPerSecond: llmResponse.promptTokensPerSecond,
completionTokensPerSecond: llmResponse.completionTokensPerSecond,
},
};
}
@ -225,14 +218,6 @@ export async function enrichWordWithRetry(
completionTimeMs:
leftResult.metrics.completionTimeMs +
rightResult.metrics.completionTimeMs,
promptTokensPerSecond:
(leftResult.metrics.promptTokensPerSecond +
rightResult.metrics.promptTokensPerSecond) /
2,
completionTokensPerSecond:
(leftResult.metrics.completionTokensPerSecond +
rightResult.metrics.completionTokensPerSecond) /
2,
};
return { results: merged, metrics: mergedMetrics };

15
data-pipeline/utils/llm-adapters/gemini.ts
View file

@ -28,8 +28,6 @@ export class GeminiAdapter implements LlmAdapter {
totalTokens: number;
promptTimeMs: number;
completionTimeMs: number;
promptTokensPerSecond: number;
completionTokensPerSecond: number;
}> {
const url = `https://generativelanguage.googleapis.com/v1beta/models/${this.model}:generateContent?key=${this.apiKey}`;
@ -72,21 +70,14 @@ export class GeminiAdapter implements LlmAdapter {
const promptTokens = json.usageMetadata.promptTokenCount;
const completionTokens = json.usageMetadata.candidatesTokenCount;
const totalTokens = json.usageMetadata.totalTokenCount;
// Gemini doesn't provide timing breakdown, so we estimate
const promptTimeMs = totalTimeMs * 0.3; // rough estimate
const completionTimeMs = totalTimeMs * 0.7; // rough estimate
return {
content,
promptTokens,
completionTokens,
totalTokens,
promptTimeMs,
completionTimeMs,
promptTokensPerSecond: promptTokens / (promptTimeMs / 1000),
completionTokensPerSecond: completionTokens / (completionTimeMs / 1000),
totalTokens: json.usageMetadata.totalTokenCount,
promptTimeMs: totalTimeMs * 0.3,
completionTimeMs: totalTimeMs * 0.7,
};
}
}

29
data-pipeline/utils/llm-adapters/openai-compatible.ts
View file

@ -7,12 +7,7 @@ interface OpenAiResponse {
completion_tokens: number;
total_tokens: number;
};
timings: {
prompt_ms: number;
predicted_ms: number;
prompt_per_second: number;
predicted_per_second: number;
};
timings?: { prompt_ms: number; predicted_ms: number };
}
export class OpenAiCompatibleAdapter implements LlmAdapter {
@ -36,8 +31,6 @@ export class OpenAiCompatibleAdapter implements LlmAdapter {
totalTokens: number;
promptTimeMs: number;
completionTimeMs: number;
promptTokensPerSecond: number;
completionTokensPerSecond: number;
}> {
const payload: Record<string, unknown> = {
messages: [
@ -61,12 +54,16 @@ export class OpenAiCompatibleAdapter implements LlmAdapter {
headers["Authorization"] = `Bearer ${this.apiKey}`;
}
const startTime = Date.now();
const response = await fetch(this.url, {
method: "POST",
headers,
body: JSON.stringify(payload),
});
const totalTimeMs = Date.now() - startTime;
if (!response.ok) {
throw new Error(`LLM server responded with status: ${response.status}`);
}
@ -78,15 +75,19 @@ export class OpenAiCompatibleAdapter implements LlmAdapter {
throw new Error("LLM response content is empty");
}
const promptTokens = json.usage.prompt_tokens;
const completionTokens = json.usage.completion_tokens;
const promptTimeMs = json.timings?.prompt_ms ?? totalTimeMs * 0.3;
const completionTimeMs = json.timings?.predicted_ms ?? totalTimeMs * 0.7;
return {
content,
promptTokens: json.usage.prompt_tokens,
completionTokens: json.usage.completion_tokens,
promptTokens,
completionTokens,
totalTokens: json.usage.total_tokens,
promptTimeMs: json.timings.prompt_ms,
completionTimeMs: json.timings.predicted_ms,
promptTokensPerSecond: json.timings.prompt_per_second,
completionTokensPerSecond: json.timings.predicted_per_second,
promptTimeMs,
completionTimeMs,
};
}
}

2
data-pipeline/utils/llm-adapters/types.ts
View file

@ -9,7 +9,5 @@ export interface LlmAdapter {
totalTokens: number;
promptTimeMs: number;
completionTimeMs: number;
promptTokensPerSecond: number;
completionTokensPerSecond: number;
}>;
}

37
data-pipeline/utils/merge-enriched-data.ts
View file

@ -1,3 +1,5 @@
import { LLM_CONFIG } from "../config/llm.js";
export type Language = "en" | "de" | "it" | "es" | "fr";
export type Pos = "noun" | "verb" | "adjective" | "adverb";
export type Gender = "masculine" | "feminine" | "neuter" | null;
@ -24,46 +26,19 @@ export interface EnrichedSense {
};
}
const LANG_MAP: Record<string, Language> = {
english: "en",
italian: "it",
german: "de",
french: "fr",
spanish: "es",
};
const POS_MAP: Record<string, Pos> = {
nouns: "noun",
verbs: "verb",
adverbs: "adverb",
adjectives: "adjective",
};
/**
* Merges skeleton data with enriched LLM senses into the final pipeline output.
*/
export function mergeEnrichedData(
word: string,
rawLanguage: string,
rawPos: string,
senses: EnrichedSense[],
): Record<string, unknown> {
const language = LANG_MAP[rawLanguage] || (rawLanguage as Language);
const pos = POS_MAP[rawPos] || (rawPos as Pos);
const fixedSenses = senses.map((sense, index) => ({
...sense,
id: `${word}:${language}:${pos}:${index}`,
language,
pos,
}));
return {
word,
language,
pos,
senses: fixedSenses,
language: senses[0]?.language ?? "en",
pos: senses[0]?.pos ?? "noun",
senses,
enrichedAt: new Date().toISOString(),
model: "qwen3.5-4b-q4_k_m",
model: LLM_CONFIG.model ?? "unknown",
};
}

2
data-pipeline/utils/pipeline-timer.ts
View file

@ -4,8 +4,6 @@ interface LlmMetrics {
totalTokens: number;
promptTimeMs: number;
completionTimeMs: number;
promptTokensPerSecond: number;
completionTokensPerSecond: number;
}
interface PipelineMetrics {

402
documentation/DATA_PIPELINE.md
View file

@ -1,7 +1,7 @@
# Lila Data Pipeline — Technical Documentation
> Multilingual dictionary enrichment pipeline for the Lila vocabulary trainer.
> Last updated: 2026-06-17
> Last updated: 2026-07-06
---
@ -29,15 +29,16 @@
## Quick Reference
| What | Where |
| ----------------- | ----------------------------------- |
| Entry point | `pipeline.ts` |
| LLM config | `config/llm.ts` |
| System prompt | `config/prompt.ts` |
| Output schema | `utils/merge-enriched-data.ts` |
| Batch size config | `config/batch.ts` _(planned)_ |
| Current model | `qwen2.5-1.5b-instruct-q4_k_m.gguf` |
| Target scale | 100,000+ words |
| What | Where |
| ------------- | ----------------------------------- |
| Entry point | `pipeline.ts` |
| LLM config | `config/llm.ts` |
| System prompt | `config/prompt.ts` |
| Batch config | `config/batch.ts` |
| Output schema | `utils/merge-enriched-data.ts` |
| LLM adapters | `utils/llm-adapters/` |
| Current model | `qwen2.5-1.5b-instruct-q4_k_m.gguf` |
| Target scale | 100,000+ words |
---
@ -50,7 +51,7 @@ The Lila Data Pipeline is a TypeScript-based batch processing system that enrich
- A **CEFR-based difficulty level** (`easy` / `medium` / `hard`)
- **Translations** into German, Italian, Spanish, and French, each with grammatical **gender**
The pipeline is designed to scale to **100,000+ words** across multiple languages and parts of speech (nouns, verbs, adjectives, adverbs). It is currently in active development: the core architecture is stable, the LLM integration layer is being evaluated across local and online providers, and a configurable batching system is planned to unlock throughput at scale.
The pipeline is designed to scale to **100,000+ words** across multiple languages and parts of speech (nouns, verbs, adjectives, adverbs). It is currently in active development: the core architecture is stable, the LLM integration layer supports both local and cloud providers via a pluggable adapter pattern, and a configurable batching system with retry/split logic is fully implemented.
### Key Design Principles
@ -59,7 +60,8 @@ The pipeline is designed to scale to **100,000+ words** across multiple language
| **Quality first** | Definitions, examples, translations, and gender must be accurate. Speed and cost are secondary. |
| **Local-first, cloud-fallback** | Local LLMs (llama.cpp) are the default for cost control and data privacy. Online APIs are evaluated as alternatives for speed. |
| **Resumable & idempotent** | Each word writes to its own JSON file. The pipeline skips already-processed words on restart. |
| **Configurable batching** | Batch size (1, 5, 15, 50, etc.) is a single config value. The pipeline adapts without code changes. |
| **Configurable batching** | Batch size is a single config value (`config/batch.ts`). The pipeline adapts without code changes. |
| **Provider-agnostic** | LLM adapters abstract local, OpenRouter, DeepSeek, and Gemini behind a single interface. |
### Open Question: Gender Accuracy
@ -73,14 +75,15 @@ Grammatical gender is currently generated by the LLM as part of the translation
No decision made. Gender handling will be determined by the 20-word quality torture suite.
### Current Status (2026-06-17)
### Current Status (2026-07-06)
- Core pipeline: scanning, enrichment, merging, verification, writing
- Local LLM integration via llama.cpp server (OpenAI-compatible API)
- **Cloud provider adapters**: Gemini, DeepSeek, OpenRouter via `utils/llm-adapters/`
- **Batching with retry/split**: configurable batch size, exponential split-on-failure (4 → 2 → 1)
- Schema validation for generated JSON
- Progress tracking and timing metrics
- **In progress:** Evaluating local models (Qwen2.5-1.5B tested; Qwen2.5-3B download pending)
- **In progress:** Designing configurable batching system
- **Pending:** 20-word quality torture suite (will decide gender approach)
- **Pending:** Online API evaluation (Gemini free tier, DeepSeek, Groq)
@ -94,17 +97,30 @@ source wordlists -> llama.cpp server (LLM) -> merge senses -> verify schema -> w
### Files at a Glance
| File | Purpose |
| ------------------------------- | ---------------------------------------------------------------- |
| `pipeline.ts` | Orchestrator. Scans sources, loops words, coordinates all stages |
| `config/llm.ts` | API URL, default parameters (`temperature`, `max_tokens`, etc.) |
| `config/prompt.ts` | System prompt sent to the LLM |
| `utils/enrich-word.ts` | Calls LLM, parses response, builds `EnrichedSense[]` |
| `utils/merge-enriched-data.ts` | Merges skeleton + enriched senses into final JSON |
| `utils/verify-enriched-file.ts` | Schema validation (required fields, types, gender enum) |
| `utils/check-llm-server.ts` | Health check before pipeline starts |
| `utils/progress-tracker.ts` | `[current/total]` formatting for console output |
| `utils/pipeline-timer.ts` | Per-word and global timing + LLM token metrics |
| File | Purpose |
| ----------------------------------------- | ----------------------------------------------------------------------------------- |
| `pipeline.ts` | Orchestrator. Scans sources, loops words, coordinates all stages |
| `config/llm.ts` | Provider selection, API URL, model name |
| `config/prompt.ts` | System prompt sent to the LLM |
| `config/batch.ts` | Batch size and max retry count |
| `utils/enrich-word.ts` | Calls LLM via adapter, parses response, builds `EnrichedSense[]`, retry/split logic |
| `utils/merge-enriched-data.ts` | Merges skeleton + enriched senses into final JSON; defines TypeScript schema |
| `utils/verify-enriched-file.ts` | Schema validation (required fields, types, gender enum) |
| `utils/check-llm-server.ts` | Health check before pipeline starts |
| `utils/scanning-source-files.ts` | Discovers wordlists from `source-data/` directory |
| `utils/create-base-json.ts` | Writes skeleton `{word, language, pos}` files |
| `utils/write-json-file.ts` | Atomic `.tmp` → rename writes |
| `utils/check-if-json-exists.ts` | Resumability: checks if word already has enriched senses |
| `utils/create-line-reader.ts` | Streaming line reader for large wordlists |
| `utils/create-output-dirs.ts` | Creates `worddata/{language}/{pos}/` folders |
| `utils/delete-file.ts` | Cleanup helper for failed batches |
| `utils/get-word-file-path.ts` | Path construction helper |
| `utils/progress-tracker.ts` | `[current/total]` formatting for console output |
| `utils/pipeline-timer.ts` | Per-word and global timing + LLM token metrics |
| `utils/llm-adapters/factory.ts` | Creates the right adapter based on `LLM_CONFIG.provider` |
| `utils/llm-adapters/types.ts` | `LlmAdapter` interface |
| `utils/llm-adapters/openai-compatible.ts` | Local llama.cpp, OpenRouter, DeepSeek |
| `utils/llm-adapters/gemini.ts` | Google Gemini native API |
### Scale Target
@ -133,7 +149,7 @@ Existing multilingual dictionaries and translation APIs provide raw word-to-word
### The Target User
A language learner using the Lila vocabulary trainer. They see a word, its definition, an example sentence, and translations with gender - all calibrated to their CEFR level (A1-C2).
A language learner using the Lila vocabulary trainer. They see a word, its definition, an example sentence, and translations with gender — all calibrated to their CEFR level (A1-C2).
### Why Not Use Existing Dictionaries?
@ -153,7 +169,7 @@ The pipeline is **direction-agnostic**. A wordlist is defined by:
Current focus: **English -> German/Italian/Spanish/French**
Planned directions include **German -> French**, **Italian -> Spanish**, etc. The LLM prompt and output schema support any combination - the only change is the source wordlist and the target languages specified in the prompt.
Planned directions include **German -> French**, **Italian -> Spanish**, etc. The LLM prompt and output schema support any combination — the only change is the source wordlist and the target languages specified in the prompt.
### Why 100,000+ Words?
@ -182,8 +198,9 @@ Generating 100,000 entries with an LLM introduces risks:
| Hallucinated definitions | Low temperature (0.1), strict system prompt, schema validation |
| Incorrect grammatical gender | Under evaluation: larger models or external lookup |
| Inconsistent difficulty levels | Explicit CEFR mapping in prompt, spot-checking |
| JSON parse failures | Retry logic, schema validation, cleanup on failure |
| JSON parse failures | Retry + split logic, schema validation, cleanup on failure |
| Model drift (online APIs) | Version pinning, local fallback |
| Provider downtime | Adapter pattern allows hot-swapping providers |
### Why TypeScript + Node?
@ -195,7 +212,7 @@ Generating 100,000 entries with an LLM introduces risks:
### Why llama.cpp?
- **GGUF format**: Single-file models, easy to swap, quantize, and version
- **OpenAI-compatible API**: `/v1/chat/completions` means the same code works for local and online models
- **OpenAI-compatible API**: `/v1/chat/completions` means the same adapter code works for local and online models
- **No dependencies**: Self-contained binary, runs on old hardware (tested on GTX 950M)
- **Privacy**: Local inference means no data leaves the machine
@ -206,15 +223,16 @@ Generating 100,000 entries with an LLM introduces risks:
### Pipeline Flow
```
Scan sources -> Check LLM -> Loop words -> Skip if exists -> Create skeleton
-> Call LLM -> Parse JSON -> Merge -> Write atomically -> Verify schema
Scan sources -> Check LLM -> Loop wordlists -> Stream words -> Skip processed
-> Create skeletons (batch) -> Call LLM -> Parse JSON -> Retry/split on failure
-> Merge -> Write atomically -> Verify schema -> Log metrics
```
### Resumability
- **Skip existing**: Checks if `{word}.json` exists with non-empty `senses`
- **Atomic writes**: `.tmp` -> rename, no partial files on crash
- **Cleanup on failure**: Deletes incomplete file, continues to next word
- **Skip existing**: `check-if-json-exists.ts` checks if `{word}.json` exists with non-empty `senses`
- **Atomic writes**: `.tmp` -> rename in `write-json-file.ts`, no partial files on crash
- **Cleanup on failure**: Deletes skeleton files for failed batches, continues to next batch
### Directory Structure
@ -222,32 +240,54 @@ Scan sources -> Check LLM -> Loop words -> Skip if exists -> Create skeleton
data-pipeline/
|-- pipeline.ts # Entry point / orchestrator
|-- config/
| |-- llm.ts # API URL, model params
| |-- llm.ts # Provider, API URL, model name
| |-- prompt.ts # System prompt
|-- utils/ # See source files (provided separately)
| |-- batch.ts # Batch size and retry config
|-- utils/
| |-- enrich-word.ts # LLM call, parse, retry/split
| |-- merge-enriched-data.ts # Schema types + merge logic
| |-- verify-enriched-file.ts # Schema validation
| |-- check-llm-server.ts # Health check
| |-- scanning-source-files.ts # Source discovery
| |-- create-base-json.ts # Skeleton writer
| |-- write-json-file.ts # Atomic JSON writer
| |-- check-if-json-exists.ts # Resumability check
| |-- create-line-reader.ts # Streaming file reader
| |-- create-output-dirs.ts # Directory creation
| |-- delete-file.ts # Cleanup helper
| |-- get-word-file-path.ts # Path helper
| |-- progress-tracker.ts # Console progress formatting
| |-- pipeline-timer.ts # Timing + token metrics
| |-- llm-adapters/
| |-- factory.ts # Adapter selection
| |-- types.ts # LlmAdapter interface
| |-- openai-compatible.ts # Local, OpenRouter, DeepSeek
| |-- gemini.ts # Google Gemini
|-- source-data/
| |-- {language}/
| |-- {pos} # One word per line, no extension
| |-- {pos} # One word per line, no extension
|-- worddata/
|-- {language}/
|-- {pos}/
|-- {word}.json # One self-contained file per word
|-- {word}.json # One self-contained file per word
|-- kaikki-source-files/ # Wiktionary dumps for gender lookup (planned)
```
### Output Schema
Each `.json` file contains: `word`, `language`, `pos`, `senses[]` (each with `sense`, `example`, `difficulty_level`, `translations` per target language), `enrichedAt`, `model`.
Each `.json` file contains: `word`, `language`, `pos`, `senses[]` (each with `id`, `sense`, `example`, `difficulty_level`, `translations` per target language), `enrichedAt`, `model`.
Full TypeScript interfaces: `utils/merge-enriched-data.ts`.
### Error Handling
| Failure | Behavior |
| ----------------------- | ------------------------------ |
| LLM server offline | Hard fail at startup |
| LLM returns bad JSON | Log, delete skeleton, continue |
| Schema validation fails | Log warnings, keep file |
| Individual word fails | Does not stop pipeline |
| Failure | Behavior |
| ------------------------------ | ------------------------------------------------------- |
| LLM server offline | Hard fail at startup (`check-llm-server.ts`) |
| LLM returns bad JSON | Retry up to 3 times, then split batch. Log and continue |
| Schema validation fails | Log warnings, keep file |
| Individual batch fails | Does not stop pipeline; cleans up skeletons |
| Individual word fails (size 1) | Log and continue to next word |
### Metrics
@ -268,19 +308,23 @@ Per-run: words processed/skipped/failed, duration, throughput, LLM token counts
### Configuration
| File | Purpose |
| ------------------ | -------------------------------------------------------------------------- |
| `config/llm.ts` | `LLM_API_URL`, `LLM_DEFAULT_PARAMS` (`temperature`, `top_p`, `max_tokens`) |
| `config/prompt.ts` | System prompt with CEFR mapping, required fields, example output |
| File | Purpose |
| ------------------ | ---------------------------------------------------------------- | ------------ | ---------- | ------------------------- |
| `config/llm.ts` | `provider` (`local` | `openrouter` | `deepseek` | `gemini`), `url`, `model` |
| `config/prompt.ts` | System prompt with CEFR mapping, required fields, example output |
| `config/batch.ts` | `BATCH_CONFIG.size` (words per call), `maxRetries` |
### Key Modules
| File | Responsibility |
| ------------------------------- | ------------------------------------------------------------------------------------------ |
| `utils/enrich-word.ts` | Calls LLM, strips markdown, parses JSON array, builds `EnrichedSense[]` with composite IDs |
| `utils/merge-enriched-data.ts` | Merges skeleton `{word, language, pos}` with LLM senses, adds `enrichedAt` and `model` |
| `utils/verify-enriched-file.ts` | Schema validation: required fields, array lengths, gender enum, translation structure |
| `utils/pipeline-timer.ts` | Tracks per-word and global metrics (duration, tokens, throughput) |
| File | Responsibility |
| ----------------------------------------- | ------------------------------------------------------------------------------------------------------------------------- |
| `utils/enrich-word.ts` | Calls LLM via adapter, strips markdown, parses JSON array, builds `EnrichedSense[]` with composite IDs, retry/split logic |
| `utils/merge-enriched-data.ts` | Merges skeleton `{word, language, pos}` with LLM senses, adds `enrichedAt` and `model` |
| `utils/verify-enriched-file.ts` | Schema validation: required fields, array lengths, gender enum, translation structure |
| `utils/pipeline-timer.ts` | Tracks per-word and global metrics (duration, tokens, throughput) |
| `utils/llm-adapters/factory.ts` | Creates adapter based on `LLM_CONFIG.provider` |
| `utils/llm-adapters/openai-compatible.ts` | OpenAI chat completions API for local llama.cpp, OpenRouter, DeepSeek |
| `utils/llm-adapters/gemini.ts` | Google Gemini `generateContent` API |
### Current Model
@ -375,9 +419,9 @@ The server flags evolved through trial and error on the target hardware (Intel i
### Known Limitations (Current)
- **Gender accuracy**: Qwen2.5-1.5B systematically defaults to `neuter` for Romance languages. Under evaluation whether larger models fix this.
- **No batching**: One word = one LLM call. System prompt re-processed every time.
- **No retry logic**: LLM parse failures are logged and skipped, not retried.
- **Single POS**: Only nouns tested. Verbs/adjectives/adverbs need prompt adjustments.
- **Hardcoded model name**: `merge-enriched-data.ts` hardcodes `"qwen3.5-4b-q4_k_m"` regardless of actual model used.
- **Duplicated mappings**: `LANG_MAP`/`POS_MAP` exist in three separate files.
---
@ -413,7 +457,7 @@ Models are evaluated on three criteria in order of priority: **quality** (defini
### 5.2 Online API Options
Evaluated as fallbacks if local models fail quality or speed targets. All support OpenAI-compatible API.
Evaluated as fallbacks if local models fail quality or speed targets. All support OpenAI-compatible API (except Gemini, which has a native adapter).
| Provider | Model | Input $/1M | Output $/1M | Free Tier | Rate Limit | Est. Cost (100k words) | Est. Time |
| ------------------- | -------------------- | ---------- | ----------- | ------------- | ---------------- | ---------------------- | ------------------- |
@ -461,7 +505,7 @@ on Qwen2.5-3B (local)
**Quality gates:**
- > =90% gender accuracy (de/it/es/fr)
- > = 90% gender accuracy (de/it/es/fr)
- 100% JSON parse rate
- No hallucinated definitions on polysemous words
- Natural, contextually appropriate example sentences
@ -484,7 +528,7 @@ Grammatical gender is embedded in the `translations` object of each sense:
}
```
Early testing with **Qwen2.5-1.5B** showed systematic failure: the model defaults to `neuter` for any translation where it is uncertain. This is particularly broken for Romance languages (Italian, Spanish, French), which do not have a neuter grammatical gender at all - only masculine and feminine.
Early testing with **Qwen2.5-1.5B** showed systematic failure: the model defaults to `neuter` for any translation where it is uncertain. This is particularly broken for Romance languages (Italian, Spanish, French), which do not have a neuter grammatical gender at all — only masculine and feminine.
Whether this is a **model size issue** (1.5B too small to retain gender facts) or a **training data gap** (Qwen2.5 family lacks gender-annotated multilingual data) is unresolved. Pending the Qwen2.5-3B evaluation.
@ -528,27 +572,24 @@ At 1 word per call, 100,000 words = 100,000 LLM requests. Each call re-processes
Single config point controls batch size everywhere:
```typescript
// config/batch.ts (planned)
export const BATCH_CONFIG = {
size: 5, // Change to 15 or 50 to test
get maxTokens() {
return Math.ceil(this.size * 250 * 1.2); // 250 tok/word + 20% buffer
},
} as const;
// config/batch.ts
export const BATCH_CONFIG = { size: 4, maxRetries: 3 } as const;
```
Change `size` to 1, 2, 5, 10, 20, etc. The pipeline adapts without code changes.
### Prompt Structure
**Single word:**
```
Word: house
["house"]
```
**Batch of 5:**
**Batch of 4:**
```
Words: ["house", "car", "tree", "water", "book"]
["house", "car", "tree", "water"]
```
LLM returns a JSON object with word keys:
@ -558,29 +599,28 @@ LLM returns a JSON object with word keys:
"house": [ { "sense": "...", "example": "...", ... } ],
"car": [ { ... } ],
"tree": [ { ... } ],
"water": [ { ... } ],
"book": [ { ... } ]
"water": [ { ... } ]
}
```
### Retry Strategy
### Retry & Split Strategy
If a batch fails (bad JSON, missing key, etc.):
```
Batch of 50 fails
Batch of 4 fails (3 retries exhausted)
|
v
Retry as 2 batches of 25
Split into 2 batches of 2
|
v
If a 25 fails, retry as 5 batches of 5
If a batch of 2 fails (3 retries), split into 2 batches of 1
|
v
If a 5 fails, retry as individual words (fallback)
If a single word fails (3 retries), log and skip
```
This gives resilience without losing the speed benefit of large batches.
This gives resilience without losing the speed benefit of large batches. The `maxRetries` config controls how many attempts are made before splitting.
### Expected Impact by Environment
@ -681,13 +721,14 @@ For each word and each candidate model:
### Near-Term (Next 2-4 Weeks)
| Item | Status | Notes |
| --------------------- | ----------- | ------------------------------------------- |
| Configurable batching | In progress | Single `BATCH_CONFIG.size` value |
| 20-word torture suite | Pending | Decides gender approach and model selection |
| Qwen2.5-3B evaluation | Pending | Download and test |
| Online API testing | Pending | Gemini free tier, DeepSeek, Groq |
| Retry logic | Pending | Exponential backoff on LLM failures |
| Item | Status | Notes |
| ------------------------ | ------------ | --------------------------------------------------------------- |
| Configurable batching | **Complete** | Single `BATCH_CONFIG.size` value, retry/split logic implemented |
| 20-word torture suite | Pending | Decides gender approach and model selection |
| Qwen2.5-3B evaluation | Pending | Download and test |
| Online API testing | Pending | Gemini free tier, DeepSeek, Groq |
| Fix hardcoded model name | Pending | `merge-enriched-data.ts` hardcodes `"qwen3.5-4b"` |
| Extract shared constants | Pending | `LANG_MAP`/`POS_MAP` duplicated in 3 files |
### Medium-Term (1-3 Months)
@ -697,7 +738,7 @@ For each word and each candidate model:
| Multi-language source | German -> French, Italian -> Spanish, etc. |
| Parallel wordlist processing | Run `english/nouns` and `english/verbs` simultaneously |
| Incremental enrichment | Only process new/changed words in a wordlist |
| Model auto-switching | Fallback to online API if local server fails |
| Model auto-switching | Fallback to online API if local server fails mid-run |
### Long-Term (3-6 Months)
@ -712,19 +753,21 @@ For each word and each candidate model:
## 11. Decisions Log
| Date | Decision | Context | Rationale |
| ---------- | ----------------------------- | ---------------------------------------------------- | --------------------------------------------------------------------------- |
| 2026-01-04 | TanStack Router for frontend | Previous project used React Router | Simpler, type-safe routing for the trainer app |
| 2026-01-04 | Vite dev server (no Nginx) | Docker setup for glossa-web | Nginx unnecessary for dev; Vite handles HMR and proxying |
| 2026-01-17 | Backend answer verification | Security vulnerability: correctAnswer exposed in API | Moved verification to server-side, shared schemas |
| 2026-03-26 | Multi-stage Docker builds | glossa-api and glossa-web containers | Smaller images, faster deploys |
| 2026-06-16 | llama.cpp for local LLM | Need local inference on old laptop | GGUF format, OpenAI-compatible API, no dependencies |
| 2026-06-16 | Q4_K_M quantization | Balance size vs quality | Q4_K_M is the community standard for 4-bit inference |
| 2026-06-16 | `-c 2048` context | Default was 4096 | Dictionary entries need ~500 tokens max; frees VRAM |
| 2026-06-16 | `-t 2` physical cores | Default was 4 (HT threads) | Hyperthreading hurts llama.cpp performance |
| 2026-06-17 | Qwen2.5-1.5B as current model | Qwen3.5-4B too slow (47s/word) | 6x speedup (8s/word), quality under evaluation |
| 2026-06-17 | Skip Gemma 4 | E2B Q4_K_M is 3.46GB | Does not fit in 4GB VRAM; lower quants sacrifice quality |
| 2026-06-17 | Skip Ministral-3B | Tokenizer mismatch (Tekken) | Outputs gibberish regardless of template; not fixable without re-conversion |
| Date | Decision | Context | Rationale |
| ---------- | --------------------------------- | ---------------------------------------------------- | --------------------------------------------------------------------------- |
| 2026-01-04 | TanStack Router for frontend | Previous project used React Router | Simpler, type-safe routing for the trainer app |
| 2026-01-04 | Vite dev server (no Nginx) | Docker setup for glossa-web | Nginx unnecessary for dev; Vite handles HMR and proxying |
| 2026-01-17 | Backend answer verification | Security vulnerability: correctAnswer exposed in API | Moved verification to server-side, shared schemas |
| 2026-03-26 | Multi-stage Docker builds | glossa-api and glossa-web containers | Smaller images, faster deploys |
| 2026-06-16 | llama.cpp for local LLM | Need local inference on old laptop | GGUF format, OpenAI-compatible API, no dependencies |
| 2026-06-16 | Q4_K_M quantization | Balance size vs quality | Q4_K_M is the community standard for 4-bit inference |
| 2026-06-16 | `-c 2048` context | Default was 4096 | Dictionary entries need ~500 tokens max; frees VRAM |
| 2026-06-16 | `-t 2` physical cores | Default was 4 (HT threads) | Hyperthreading hurts llama.cpp performance |
| 2026-06-17 | Qwen2.5-1.5B as current model | Qwen3.5-4B too slow (47s/word) | 6x speedup (8s/word), quality under evaluation |
| 2026-06-17 | Skip Gemma 4 | E2B Q4_K_M is 3.46GB | Does not fit in 4GB VRAM; lower quants sacrifice quality |
| 2026-06-17 | Skip Ministral-3B | Tokenizer mismatch (Tekken) | Outputs gibberish regardless of template; not fixable without re-conversion |
| 2026-07-06 | Adapter pattern for LLM providers | Need to evaluate local vs cloud | `utils/llm-adapters/` with factory + types + per-provider implementations |
| 2026-07-06 | Retry + split batching | LLM JSON parse failures on larger batches | `enrichWordWithRetry` retries 3 times, then halves batch until size 1 |
---
@ -739,13 +782,15 @@ For each word and each candidate model:
### Data Pipeline
| Issue | Details | Severity |
| --------------------------------- | ------------------------------------------------------------------------- | ------------------- |
| Ministral-3B tokenizer mismatch | Tekken tokenizer not properly converted to GGUF. Model outputs gibberish. | Blocker - abandoned |
| Qwen2.5-1.5B gender hallucination | Systematic `neuter` default for Romance languages. | Under evaluation |
| No batching | 1 word = 1 call. System prompt re-processed every time. | In progress |
| No retry logic | LLM parse failures are logged and skipped. | Planned |
| Single POS tested | Only nouns validated. Verbs/adjectives need prompt changes. | Known limitation |
| Issue | Details | Severity |
| --------------------------------- | ----------------------------------------------------------------------------------------------- | -------------------------------- |
| Ministral-3B tokenizer mismatch | Tekken tokenizer not properly converted to GGUF. Model outputs gibberish. | Blocker - abandoned |
| Qwen2.5-1.5B gender hallucination | Systematic `neuter` default for Romance languages. | Under evaluation |
| Hardcoded model name | `merge-enriched-data.ts` always writes `"qwen3.5-4b-q4_k_m"` regardless of actual model. | Minor - fix before production |
| Duplicated LANG_MAP/POS_MAP | Identical mapping objects in `create-base-json.ts`, `merge-enriched-data.ts`, `enrich-word.ts`. | Minor - refactor risk |
| OpenAI-compatible timings | `json.timings` is llama.cpp-specific. Will break for OpenRouter/DeepSeek. | Medium - needs graceful fallback |
| Single POS tested | Only nouns validated. Verbs/adjectives need prompt changes. | Known limitation |
| Batch metrics averaging | Split-and-merge averages tokens/sec instead of weighting by token count. | Minor - summary stats only |
### Hardware
@ -763,10 +808,16 @@ For each word and each candidate model:
- Node.js + npm
- `tsx` installed globally: `npm install -g tsx`
- llama.cpp built from source
- GGUF model downloaded to `~/Downloads/llama.cpp/models/`
- llama.cpp built from source (for local mode)
- GGUF model downloaded to `~/Downloads/llama.cpp/models/` (for local mode)
- API keys set as environment variables (for cloud mode):
```bash
export DEEPSEEK_API_KEY="sk-..."
export GEMINI_API_KEY="..."
export OPENROUTER_API_KEY="..."
```
### Start the LLM Server
### Start the LLM Server (Local Mode)
```bash
cd ~/Downloads/llama.cpp
@ -785,6 +836,26 @@ cd ~/Downloads/llama.cpp
--prio 2
```
### Configure Provider
Edit `config/llm.ts`:
```typescript
// Local
export const LLM_CONFIG = {
provider: "local" as const,
url: "http://127.0.0.1:8080/v1/chat/completions",
model: undefined,
};
// Gemini
export const LLM_CONFIG = {
provider: "gemini" as const,
url: "",
model: "gemini-2.5-flash-lite",
};
```
### Run the Pipeline
```bash
@ -798,50 +869,56 @@ npx tsx pipeline.ts
Starting data pipeline...
step 1: scanning the source files...
Scan complete! Found 1 wordlist(s):
- ENGLISH (nouns)
✅ Scan complete! Found 1 wordlist(s):
• ENGLISH (nouns)
...
step 2: creating necessary output folders...
✅ All required output directories have been verified and created successfully.
Pipeline Summary
Duration: 23.0s
Processed: 3
step 3: verifying local AI engine status...
🟢 Local AI engine is connected and ready for inference!
step 4: looping through the wordlists...
Reading list: [ENGLISH] -> [NOUNS]
Batch 1/1: [house, car, tree, water]
[1/4] (0 failed) Enriched and saved: house.json
[2/4] (0 failed) Enriched and saved: car.json
[3/4] (0 failed) Enriched and saved: tree.json
[4/4] (0 failed) Enriched and saved: water.json
⏱️ Word took 8.2s
⏱️ Pipeline Summary
Duration: 32.8s
Processed: 4
Skipped: 0
Failed: 0
Total: 3
Throughput: 0.13 words/sec
Total: 4
Throughput: 0.12 words/sec
LLM Metrics
Calls: 3
Avg prompt tokens: 308
Avg completion tokens: 132
Avg prompt speed: 549.4 tok/s
🤖 LLM Metrics
Calls: 1
Avg prompt tokens: 312
Avg completion tokens: 524
Avg prompt speed: 548.2 tok/s
Avg completion speed: 18.8 tok/s
Global data pipeline run completed successfully.
```
### Environment Variables (Online Mode)
```bash
export DEEPSEEK_API_KEY="sk-..."
export GEMINI_API_KEY="..."
export GROQ_API_KEY="..."
```
Then update `config/llm.ts` to point to the online API URL.
---
## 14. Roadmap
### Phase 1: Batching (Current)
### Phase 1: Batching (Complete)
| Task | Status | Notes |
| ------------------------------------- | ----------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------- |
| Implement configurable batch size | In progress | Single `BATCH_CONFIG.size` value. Prompt formatting: single word -&gt; word array. Response parsing: keyed JSON object. Retry: 50 -&gt; 25 -&gt; 5 -&gt; 1. |
| Verify batching doesn't break quality | Pending | Run 20-word torture suite on Qwen2.5-1.5B with batch sizes 1, 5, 15. Compare output. |
| Measure speedup vs batch size | Pending | Track throughput at 1, 5, 15 on local hardware. |
| Task | Status | Notes |
| ------------------------------------- | ------------ | ------------------------------------------------------------------------------------ |
| Implement configurable batch size | **Complete** | `config/batch.ts` with `size` and `maxRetries` |
| Implement retry + split logic | **Complete** | `enrichWordWithRetry`: 3 retries, then halve batch |
| Verify batching doesn't break quality | Pending | Run 20-word torture suite on Qwen2.5-1.5B with batch sizes 1, 5, 15. Compare output. |
| Measure speedup vs batch size | Pending | Track throughput at 1, 5, 15 on local hardware. |
**Goal:** Unlock 5-15x speedup on local, unlock free online API tiers.
@ -849,14 +926,14 @@ Then update `config/llm.ts` to point to the online API URL.
### Phase 2: Model Selection
| Task | Status | Notes |
| ------------------------------------------------ | ------- | ------------------------------------------------------------------------------------------- |
| Download Qwen2.5-3B Q4_K_M | Pending | ~1.9GB, fits in 4GB VRAM. |
| Run torture suite on Qwen2.5-3B (batched) | Pending | Check if gender accuracy improves with 2x params. |
| Test Gemini 2.5 Flash-Lite free tier (batched) | Pending | 1,500 req/day x 50 words = 75k words/day. Zero cost. |
| Test DeepSeek V4 Flash free tier (batched) | Pending | 5M tokens free. ~9k words. |
| Test Groq Llama 3.1 8B (batched, paid if needed) | Pending | Fastest option. ~$7 for 100k words. |
| Decide: local vs online, which model | Pending | Criteria: quality &gt;= 90% gender, 100% JSON, sensible definitions. Then speed, then cost. |
| Task | Status | Notes |
| ------------------------------------------------ | ------- | ---------------------------------------------------------------------------------------- |
| Download Qwen2.5-3B Q4_K_M | Pending | ~1.9GB, fits in 4GB VRAM. |
| Run torture suite on Qwen2.5-3B (batched) | Pending | Check if gender accuracy improves with 2x params. |
| Test Gemini 2.5 Flash-Lite free tier (batched) | Pending | 1,500 req/day x 50 words = 75k words/day. Zero cost. |
| Test DeepSeek V4 Flash free tier (batched) | Pending | 5M tokens free. ~9k words. |
| Test Groq Llama 3.1 8B (batched, paid if needed) | Pending | Fastest option. ~$7 for 100k words. |
| Decide: local vs online, which model | Pending | Criteria: quality >= 90% gender, 100% JSON, sensible definitions. Then speed, then cost. |
**Goal:** Pick the model and provider for the 100k word run.
@ -877,14 +954,14 @@ Then update `config/llm.ts` to point to the online API URL.
### Phase 4: Extend
| Task | Status | Notes |
| ---------------------------- | ------- | ------------------------------------------------------------------------------------------------- |
| Multi-POS support | Pending | Verbs, adjectives, adverbs. Each needs prompt variants (conjugations, agreement, etc.). |
| Multi-language source | Pending | German -&gt; French, Italian -&gt; Spanish, etc. Schema already supports any source/target combo. |
| Parallel wordlist processing | Pending | Run `english/nouns` and `english/verbs` simultaneously. |
| Incremental enrichment | Pending | Only process new/changed words in a wordlist. |
| GPU rental integration | Pending | Script to spin up Vast.ai/RunPod, run pipeline, download results. |
| Quality regression tests | Pending | Run torture suite on every model change. |
| Task | Status | Notes |
| ---------------------------- | ------- | ------------------------------------------------------------------------------------------- |
| Multi-POS support | Pending | Verbs, adjectives, adverbs. Each needs prompt variants (conjugations, agreement, etc.). |
| Multi-language source | Pending | German -> French, Italian -> Spanish, etc. Schema already supports any source/target combo. |
| Parallel wordlist processing | Pending | Run `english/nouns` and `english/verbs` simultaneously. |
| Incremental enrichment | Pending | Only process new/changed words in a wordlist. |
| GPU rental integration | Pending | Script to spin up Vast.ai/RunPod, run pipeline, download results. |
| Quality regression tests | Pending | Run torture suite on every model change. |
**Goal:** Generalize pipeline for any language direction and POS.
@ -892,10 +969,13 @@ Then update `config/llm.ts` to point to the online API URL.
### Backlog (Unscheduled)
| Task | Context |
| -------------------------------- | ------------------------------------------------------------------------------------------- |
| Batch API discounts | Gemini, Qwen, Azure offer 50% off for 24h SLA. Relevant if running recurring large batches. |
| Model auto-switching | Fallback to online API if local server fails mid-run. |
| Community open-source | Clean up, document, publish for other language learners. |
| Prometheus metrics | `--metrics` flag on llama-server for automated performance tracking. |
| `-c 1024` / `-b 256` experiments | Further VRAM optimization on GTX 950M. Low priority if moving to cloud. |
| Task | Context |
| ------------------------------------------- | ------------------------------------------------------------------------------------------- |
| Batch API discounts | Gemini, Qwen, Azure offer 50% off for 24h SLA. Relevant if running recurring large batches. |
| Model auto-switching | Fallback to online API if local server fails mid-run. |
| Community open-source | Clean up, document, publish for other language learners. |
| Prometheus metrics | `--metrics` flag on llama-server for automated performance tracking. |
| `-c 1024` / `-b 256` experiments | Further VRAM optimization on GTX 950M. Low priority if moving to cloud. |
| Extract shared LANG_MAP/POS_MAP | Single source of truth for language/pos mappings. |
| Fix hardcoded model name | Pass actual model name through enrichment chain. |
| Graceful timing fallback for cloud adapters | Handle missing `timings` field in OpenRouter/DeepSeek responses. |