forgejo-lila/lila
1
0
Fork 0
Code Issues 50 Pull requests Projects 1 Releases Packages Wiki Activity Actions

bugfixing #1

Browse source
This commit is contained in:
lila 2026-07-06 14:27:47 +02:00
parent afd28d934e
commit 2a6c56ed23
12 changed files with 300 additions and 269 deletions
Download patch file Download diff file Expand all files Collapse all files

2
data-pipeline/config/batch.ts
View file

@ -1,3 +1 @@
export const BATCH_CONFIG = { size: 4, maxRetries: 3 } as const; export const BATCH_CONFIG = { size: 4, maxRetries: 3 } as const;
//1, 2, 5, 10, 20

14
data-pipeline/config/constants.ts Normal file
View file

@ -0,0 +1,14 @@
export const LANG_MAP: Record<string, string> = {
english: "en",
italian: "it",
german: "de",
french: "fr",
spanish: "es",
};
export const POS_MAP: Record<string, string> = {
nouns: "noun",
verbs: "verb",
adverbs: "adverb",
adjectives: "adjective",
};

25
data-pipeline/pipeline.ts
View file

@ -27,7 +27,7 @@ async function main() {
console.log("\n step 2: creating necessary output folders..."); console.log("\n step 2: creating necessary output folders...");
ensureOutputFolders(wordlists); ensureOutputFolders(wordlists);
// step 3: check to verify the AI engine is ready before touching anything // step 3: check to verify the local AI engine is ready before touching anything
console.log("\n step 3: verifying local AI engine status..."); console.log("\n step 3: verifying local AI engine status...");
await checkLlmServer(); await checkLlmServer();
@ -99,12 +99,7 @@ async function main() {
); );
const targetFilePath = getWordFilePath(word, wordlist.outputDir); const targetFilePath = getWordFilePath(word, wordlist.outputDir);
const enrichedData = mergeEnrichedData( const enrichedData = mergeEnrichedData(word, senses);
word,
wordlist.language,
wordlist.pos,
senses,
);
writeJsonFile(targetFilePath, enrichedData); writeJsonFile(targetFilePath, enrichedData);
@ -118,17 +113,19 @@ async function main() {
} }
timer.recordProcessed({ timer.recordProcessed({
promptTokens: result.metrics.promptTokens / batch.length, promptTokens: result.metrics.promptTokens,
completionTokens: result.metrics.completionTokens / batch.length, completionTokens: result.metrics.completionTokens,
totalTokens: result.metrics.totalTokens / batch.length, totalTokens: result.metrics.totalTokens,
promptTimeMs: result.metrics.promptTimeMs / batch.length, promptTimeMs: result.metrics.promptTimeMs,
completionTimeMs: result.metrics.completionTimeMs / batch.length, completionTimeMs: result.metrics.completionTimeMs,
promptTokensPerSecond: result.metrics.promptTokensPerSecond,
completionTokensPerSecond: result.metrics.completionTokensPerSecond,
}); });
} }
console.log(` ${timer.getWordTiming()}`); console.log(` ${timer.getWordTiming()}`);
// Show ETA every 5 batches or on the last batch
if (batchNum % 5 === 0 || batchNum === totalBatches) {
console.log(` 📊 ${timer.getEta(unprocessedWords.length)}`);
}
} catch (error: unknown) { } catch (error: unknown) {
const errorMessage = const errorMessage =
error instanceof Error ? error.message : String(error); error instanceof Error ? error.message : String(error);

8
data-pipeline/utils/check-llm-server.ts
View file

@ -1,10 +1,18 @@
import { LLM_CONFIG } from "../config/llm.js";
/** /**
* Pings the local llama.cpp server to ensure it's up, running, and has a model loaded. * Pings the local llama.cpp server to ensure it's up, running, and has a model loaded.
* If the server is offline or still loading, it terminates the pipeline gracefully. * If the server is offline or still loading, it terminates the pipeline gracefully.
* Skipped entirely when using a cloud provider.
*/ */
export async function checkLlmServer( export async function checkLlmServer(
url = "http://127.0.0.1:8080/health", url = "http://127.0.0.1:8080/health",
): Promise<void> { ): Promise<void> {
if (LLM_CONFIG.provider !== "local") {
console.log("🌐 Using cloud provider — skipping local health check.");
return;
}
try { try {
const response = await fetch(url); const response = await fetch(url);

16
data-pipeline/utils/create-base-json.ts
View file

@ -1,20 +1,6 @@
import fs from "fs"; import fs from "fs";
import path from "path"; import path from "path";
import { LANG_MAP, POS_MAP } from "../config/constants.js";
const LANG_MAP: Record<string, string> = {
english: "en",
italian: "it",
german: "de",
french: "fr",
spanish: "es",
};
const POS_MAP: Record<string, string> = {
nouns: "noun",
verbs: "verb",
adverbs: "adverb",
adjectives: "adjective",
};
/** /**
* Creates the base JSON file with word, language, and pos. * Creates the base JSON file with word, language, and pos.

17
data-pipeline/utils/enrich-word.ts
View file

@ -1,4 +1,3 @@
// utils/enrich-word.ts
import { ENRICHMENT_SYSTEM_PROMPT } from "../config/prompt.js"; import { ENRICHMENT_SYSTEM_PROMPT } from "../config/prompt.js";
import { createAdapter } from "./llm-adapters/factory.js"; import { createAdapter } from "./llm-adapters/factory.js";
import { BATCH_CONFIG } from "../config/batch.js"; import { BATCH_CONFIG } from "../config/batch.js";
@ -26,8 +25,6 @@ interface LlmResponse {
totalTokens: number; totalTokens: number;
promptTimeMs: number; promptTimeMs: number;
completionTimeMs: number; completionTimeMs: number;
promptTokensPerSecond: number;
completionTokensPerSecond: number;
} }
export interface EnrichmentResult { export interface EnrichmentResult {
@ -38,13 +35,11 @@ export interface EnrichmentResult {
totalTokens: number; totalTokens: number;
promptTimeMs: number; promptTimeMs: number;
completionTimeMs: number; completionTimeMs: number;
promptTokensPerSecond: number;
completionTokensPerSecond: number;
}; };
} }
/** /**
* Calls the local LLM with the enrichment prompt. * Calls the LLM with the enrichment prompt.
* Returns the response content and timing metrics. * Returns the response content and timing metrics.
*/ */
async function callLlm(words: string[]): Promise<LlmResponse> { async function callLlm(words: string[]): Promise<LlmResponse> {
@ -157,8 +152,6 @@ export async function enrichWord(
totalTokens: llmResponse.totalTokens, totalTokens: llmResponse.totalTokens,
promptTimeMs: llmResponse.promptTimeMs, promptTimeMs: llmResponse.promptTimeMs,
completionTimeMs: llmResponse.completionTimeMs, completionTimeMs: llmResponse.completionTimeMs,
promptTokensPerSecond: llmResponse.promptTokensPerSecond,
completionTokensPerSecond: llmResponse.completionTokensPerSecond,
}, },
}; };
} }
@ -225,14 +218,6 @@ export async function enrichWordWithRetry(
completionTimeMs: completionTimeMs:
leftResult.metrics.completionTimeMs + leftResult.metrics.completionTimeMs +
rightResult.metrics.completionTimeMs, rightResult.metrics.completionTimeMs,
promptTokensPerSecond:
(leftResult.metrics.promptTokensPerSecond +
rightResult.metrics.promptTokensPerSecond) /
2,
completionTokensPerSecond:
(leftResult.metrics.completionTokensPerSecond +
rightResult.metrics.completionTokensPerSecond) /
2,
}; };
return { results: merged, metrics: mergedMetrics }; return { results: merged, metrics: mergedMetrics };

15
data-pipeline/utils/llm-adapters/gemini.ts
View file

@ -28,8 +28,6 @@ export class GeminiAdapter implements LlmAdapter {
totalTokens: number; totalTokens: number;
promptTimeMs: number; promptTimeMs: number;
completionTimeMs: number; completionTimeMs: number;
promptTokensPerSecond: number;
completionTokensPerSecond: number;
}> { }> {
const url = `https://generativelanguage.googleapis.com/v1beta/models/${this.model}:generateContent?key=${this.apiKey}`; const url = `https://generativelanguage.googleapis.com/v1beta/models/${this.model}:generateContent?key=${this.apiKey}`;
@ -72,21 +70,14 @@ export class GeminiAdapter implements LlmAdapter {
const promptTokens = json.usageMetadata.promptTokenCount; const promptTokens = json.usageMetadata.promptTokenCount;
const completionTokens = json.usageMetadata.candidatesTokenCount; const completionTokens = json.usageMetadata.candidatesTokenCount;
const totalTokens = json.usageMetadata.totalTokenCount;
// Gemini doesn't provide timing breakdown, so we estimate
const promptTimeMs = totalTimeMs * 0.3; // rough estimate
const completionTimeMs = totalTimeMs * 0.7; // rough estimate
return { return {
content, content,
promptTokens, promptTokens,
completionTokens, completionTokens,
totalTokens, totalTokens: json.usageMetadata.totalTokenCount,
promptTimeMs, promptTimeMs: totalTimeMs * 0.3,
completionTimeMs, completionTimeMs: totalTimeMs * 0.7,
promptTokensPerSecond: promptTokens / (promptTimeMs / 1000),
completionTokensPerSecond: completionTokens / (completionTimeMs / 1000),
}; };
} }
} }

29
data-pipeline/utils/llm-adapters/openai-compatible.ts
View file

@ -7,12 +7,7 @@ interface OpenAiResponse {
completion_tokens: number; completion_tokens: number;
total_tokens: number; total_tokens: number;
}; };
timings: { timings?: { prompt_ms: number; predicted_ms: number };
prompt_ms: number;
predicted_ms: number;
prompt_per_second: number;
predicted_per_second: number;
};
} }
export class OpenAiCompatibleAdapter implements LlmAdapter { export class OpenAiCompatibleAdapter implements LlmAdapter {
@ -36,8 +31,6 @@ export class OpenAiCompatibleAdapter implements LlmAdapter {
totalTokens: number; totalTokens: number;
promptTimeMs: number; promptTimeMs: number;
completionTimeMs: number; completionTimeMs: number;
promptTokensPerSecond: number;
completionTokensPerSecond: number;
}> { }> {
const payload: Record<string, unknown> = { const payload: Record<string, unknown> = {
messages: [ messages: [
@ -61,12 +54,16 @@ export class OpenAiCompatibleAdapter implements LlmAdapter {
headers["Authorization"] = `Bearer ${this.apiKey}`; headers["Authorization"] = `Bearer ${this.apiKey}`;
} }
const startTime = Date.now();
const response = await fetch(this.url, { const response = await fetch(this.url, {
method: "POST", method: "POST",
headers, headers,
body: JSON.stringify(payload), body: JSON.stringify(payload),
}); });
const totalTimeMs = Date.now() - startTime;
if (!response.ok) { if (!response.ok) {
throw new Error(`LLM server responded with status: ${response.status}`); throw new Error(`LLM server responded with status: ${response.status}`);
} }
@ -78,15 +75,19 @@ export class OpenAiCompatibleAdapter implements LlmAdapter {
throw new Error("LLM response content is empty"); throw new Error("LLM response content is empty");
} }
const promptTokens = json.usage.prompt_tokens;
const completionTokens = json.usage.completion_tokens;
const promptTimeMs = json.timings?.prompt_ms ?? totalTimeMs * 0.3;
const completionTimeMs = json.timings?.predicted_ms ?? totalTimeMs * 0.7;
return { return {
content, content,
promptTokens: json.usage.prompt_tokens, promptTokens,
completionTokens: json.usage.completion_tokens, completionTokens,
totalTokens: json.usage.total_tokens, totalTokens: json.usage.total_tokens,
promptTimeMs: json.timings.prompt_ms, promptTimeMs,
completionTimeMs: json.timings.predicted_ms, completionTimeMs,
promptTokensPerSecond: json.timings.prompt_per_second,
completionTokensPerSecond: json.timings.predicted_per_second,
}; };
} }
} }

2
data-pipeline/utils/llm-adapters/types.ts
View file

@ -9,7 +9,5 @@ export interface LlmAdapter {
totalTokens: number; totalTokens: number;
promptTimeMs: number; promptTimeMs: number;
completionTimeMs: number; completionTimeMs: number;
promptTokensPerSecond: number;
completionTokensPerSecond: number;
}>; }>;
} }

37
data-pipeline/utils/merge-enriched-data.ts
View file

@ -1,3 +1,5 @@
import { LLM_CONFIG } from "../config/llm.js";
export type Language = "en" | "de" | "it" | "es" | "fr"; export type Language = "en" | "de" | "it" | "es" | "fr";
export type Pos = "noun" | "verb" | "adjective" | "adverb"; export type Pos = "noun" | "verb" | "adjective" | "adverb";
export type Gender = "masculine" | "feminine" | "neuter" | null; export type Gender = "masculine" | "feminine" | "neuter" | null;
@ -24,46 +26,19 @@ export interface EnrichedSense {
}; };
} }
const LANG_MAP: Record<string, Language> = {
english: "en",
italian: "it",
german: "de",
french: "fr",
spanish: "es",
};
const POS_MAP: Record<string, Pos> = {
nouns: "noun",
verbs: "verb",
adverbs: "adverb",
adjectives: "adjective",
};
/** /**
* Merges skeleton data with enriched LLM senses into the final pipeline output. * Merges skeleton data with enriched LLM senses into the final pipeline output.
*/ */
export function mergeEnrichedData( export function mergeEnrichedData(
word: string, word: string,
rawLanguage: string,
rawPos: string,
senses: EnrichedSense[], senses: EnrichedSense[],
): Record<string, unknown> { ): Record<string, unknown> {
const language = LANG_MAP[rawLanguage] || (rawLanguage as Language);
const pos = POS_MAP[rawPos] || (rawPos as Pos);
const fixedSenses = senses.map((sense, index) => ({
...sense,
id: `${word}:${language}:${pos}:${index}`,
language,
pos,
}));
return { return {
word, word,
language, language: senses[0]?.language ?? "en",
pos, pos: senses[0]?.pos ?? "noun",
senses: fixedSenses, senses,
enrichedAt: new Date().toISOString(), enrichedAt: new Date().toISOString(),
model: "qwen3.5-4b-q4_k_m", model: LLM_CONFIG.model ?? "unknown",
}; };
} }

2
data-pipeline/utils/pipeline-timer.ts
View file

@ -4,8 +4,6 @@ interface LlmMetrics {
totalTokens: number; totalTokens: number;
promptTimeMs: number; promptTimeMs: number;
completionTimeMs: number; completionTimeMs: number;
promptTokensPerSecond: number;
completionTokensPerSecond: number;
} }
interface PipelineMetrics { interface PipelineMetrics {

270
documentation/DATA_PIPELINE.md
View file

@ -1,7 +1,7 @@
# Lila Data Pipeline — Technical Documentation # Lila Data Pipeline — Technical Documentation
> Multilingual dictionary enrichment pipeline for the Lila vocabulary trainer. > Multilingual dictionary enrichment pipeline for the Lila vocabulary trainer.
> Last updated: 2026-06-17 > Last updated: 2026-07-06
--- ---
@ -30,12 +30,13 @@
## Quick Reference ## Quick Reference
| What | Where | | What | Where |
| ----------------- | ----------------------------------- | | ------------- | ----------------------------------- |
| Entry point | `pipeline.ts` | | Entry point | `pipeline.ts` |
| LLM config | `config/llm.ts` | | LLM config | `config/llm.ts` |
| System prompt | `config/prompt.ts` | | System prompt | `config/prompt.ts` |
| Batch config | `config/batch.ts` |
| Output schema | `utils/merge-enriched-data.ts` | | Output schema | `utils/merge-enriched-data.ts` |
| Batch size config | `config/batch.ts` _(planned)_ | | LLM adapters | `utils/llm-adapters/` |
| Current model | `qwen2.5-1.5b-instruct-q4_k_m.gguf` | | Current model | `qwen2.5-1.5b-instruct-q4_k_m.gguf` |
| Target scale | 100,000+ words | | Target scale | 100,000+ words |
@ -50,7 +51,7 @@ The Lila Data Pipeline is a TypeScript-based batch processing system that enrich
- A **CEFR-based difficulty level** (`easy` / `medium` / `hard`) - A **CEFR-based difficulty level** (`easy` / `medium` / `hard`)
- **Translations** into German, Italian, Spanish, and French, each with grammatical **gender** - **Translations** into German, Italian, Spanish, and French, each with grammatical **gender**
The pipeline is designed to scale to **100,000+ words** across multiple languages and parts of speech (nouns, verbs, adjectives, adverbs). It is currently in active development: the core architecture is stable, the LLM integration layer is being evaluated across local and online providers, and a configurable batching system is planned to unlock throughput at scale. The pipeline is designed to scale to **100,000+ words** across multiple languages and parts of speech (nouns, verbs, adjectives, adverbs). It is currently in active development: the core architecture is stable, the LLM integration layer supports both local and cloud providers via a pluggable adapter pattern, and a configurable batching system with retry/split logic is fully implemented.
### Key Design Principles ### Key Design Principles
@ -59,7 +60,8 @@ The pipeline is designed to scale to **100,000+ words** across multiple language
| **Quality first** | Definitions, examples, translations, and gender must be accurate. Speed and cost are secondary. | | **Quality first** | Definitions, examples, translations, and gender must be accurate. Speed and cost are secondary. |
| **Local-first, cloud-fallback** | Local LLMs (llama.cpp) are the default for cost control and data privacy. Online APIs are evaluated as alternatives for speed. | | **Local-first, cloud-fallback** | Local LLMs (llama.cpp) are the default for cost control and data privacy. Online APIs are evaluated as alternatives for speed. |
| **Resumable & idempotent** | Each word writes to its own JSON file. The pipeline skips already-processed words on restart. | | **Resumable & idempotent** | Each word writes to its own JSON file. The pipeline skips already-processed words on restart. |
| **Configurable batching** | Batch size (1, 5, 15, 50, etc.) is a single config value. The pipeline adapts without code changes. | | **Configurable batching** | Batch size is a single config value (`config/batch.ts`). The pipeline adapts without code changes. |
| **Provider-agnostic** | LLM adapters abstract local, OpenRouter, DeepSeek, and Gemini behind a single interface. |
### Open Question: Gender Accuracy ### Open Question: Gender Accuracy
@ -73,14 +75,15 @@ Grammatical gender is currently generated by the LLM as part of the translation
No decision made. Gender handling will be determined by the 20-word quality torture suite. No decision made. Gender handling will be determined by the 20-word quality torture suite.
### Current Status (2026-06-17) ### Current Status (2026-07-06)
- Core pipeline: scanning, enrichment, merging, verification, writing - Core pipeline: scanning, enrichment, merging, verification, writing
- Local LLM integration via llama.cpp server (OpenAI-compatible API) - Local LLM integration via llama.cpp server (OpenAI-compatible API)
- **Cloud provider adapters**: Gemini, DeepSeek, OpenRouter via `utils/llm-adapters/`
- **Batching with retry/split**: configurable batch size, exponential split-on-failure (4 → 2 → 1)
- Schema validation for generated JSON - Schema validation for generated JSON
- Progress tracking and timing metrics - Progress tracking and timing metrics
- **In progress:** Evaluating local models (Qwen2.5-1.5B tested; Qwen2.5-3B download pending) - **In progress:** Evaluating local models (Qwen2.5-1.5B tested; Qwen2.5-3B download pending)
- **In progress:** Designing configurable batching system
- **Pending:** 20-word quality torture suite (will decide gender approach) - **Pending:** 20-word quality torture suite (will decide gender approach)
- **Pending:** Online API evaluation (Gemini free tier, DeepSeek, Groq) - **Pending:** Online API evaluation (Gemini free tier, DeepSeek, Groq)
@ -95,16 +98,29 @@ source wordlists -> llama.cpp server (LLM) -> merge senses -> verify schema -> w
### Files at a Glance ### Files at a Glance
| File | Purpose | | File | Purpose |
| ------------------------------- | ---------------------------------------------------------------- | | ----------------------------------------- | ----------------------------------------------------------------------------------- |
| `pipeline.ts` | Orchestrator. Scans sources, loops words, coordinates all stages | | `pipeline.ts` | Orchestrator. Scans sources, loops words, coordinates all stages |
| `config/llm.ts` | API URL, default parameters (`temperature`, `max_tokens`, etc.) | | `config/llm.ts` | Provider selection, API URL, model name |
| `config/prompt.ts` | System prompt sent to the LLM | | `config/prompt.ts` | System prompt sent to the LLM |
| `utils/enrich-word.ts` | Calls LLM, parses response, builds `EnrichedSense[]` | | `config/batch.ts` | Batch size and max retry count |
| `utils/merge-enriched-data.ts` | Merges skeleton + enriched senses into final JSON | | `utils/enrich-word.ts` | Calls LLM via adapter, parses response, builds `EnrichedSense[]`, retry/split logic |
| `utils/merge-enriched-data.ts` | Merges skeleton + enriched senses into final JSON; defines TypeScript schema |
| `utils/verify-enriched-file.ts` | Schema validation (required fields, types, gender enum) | | `utils/verify-enriched-file.ts` | Schema validation (required fields, types, gender enum) |
| `utils/check-llm-server.ts` | Health check before pipeline starts | | `utils/check-llm-server.ts` | Health check before pipeline starts |
| `utils/scanning-source-files.ts` | Discovers wordlists from `source-data/` directory |
| `utils/create-base-json.ts` | Writes skeleton `{word, language, pos}` files |
| `utils/write-json-file.ts` | Atomic `.tmp` → rename writes |
| `utils/check-if-json-exists.ts` | Resumability: checks if word already has enriched senses |
| `utils/create-line-reader.ts` | Streaming line reader for large wordlists |
| `utils/create-output-dirs.ts` | Creates `worddata/{language}/{pos}/` folders |
| `utils/delete-file.ts` | Cleanup helper for failed batches |
| `utils/get-word-file-path.ts` | Path construction helper |
| `utils/progress-tracker.ts` | `[current/total]` formatting for console output | | `utils/progress-tracker.ts` | `[current/total]` formatting for console output |
| `utils/pipeline-timer.ts` | Per-word and global timing + LLM token metrics | | `utils/pipeline-timer.ts` | Per-word and global timing + LLM token metrics |
| `utils/llm-adapters/factory.ts` | Creates the right adapter based on `LLM_CONFIG.provider` |
| `utils/llm-adapters/types.ts` | `LlmAdapter` interface |
| `utils/llm-adapters/openai-compatible.ts` | Local llama.cpp, OpenRouter, DeepSeek |
| `utils/llm-adapters/gemini.ts` | Google Gemini native API |
### Scale Target ### Scale Target
@ -133,7 +149,7 @@ Existing multilingual dictionaries and translation APIs provide raw word-to-word
### The Target User ### The Target User
A language learner using the Lila vocabulary trainer. They see a word, its definition, an example sentence, and translations with gender - all calibrated to their CEFR level (A1-C2). A language learner using the Lila vocabulary trainer. They see a word, its definition, an example sentence, and translations with gender — all calibrated to their CEFR level (A1-C2).
### Why Not Use Existing Dictionaries? ### Why Not Use Existing Dictionaries?
@ -153,7 +169,7 @@ The pipeline is **direction-agnostic**. A wordlist is defined by:
Current focus: **English -> German/Italian/Spanish/French** Current focus: **English -> German/Italian/Spanish/French**
Planned directions include **German -> French**, **Italian -> Spanish**, etc. The LLM prompt and output schema support any combination - the only change is the source wordlist and the target languages specified in the prompt. Planned directions include **German -> French**, **Italian -> Spanish**, etc. The LLM prompt and output schema support any combination — the only change is the source wordlist and the target languages specified in the prompt.
### Why 100,000+ Words? ### Why 100,000+ Words?
@ -182,8 +198,9 @@ Generating 100,000 entries with an LLM introduces risks:
| Hallucinated definitions | Low temperature (0.1), strict system prompt, schema validation | | Hallucinated definitions | Low temperature (0.1), strict system prompt, schema validation |
| Incorrect grammatical gender | Under evaluation: larger models or external lookup | | Incorrect grammatical gender | Under evaluation: larger models or external lookup |
| Inconsistent difficulty levels | Explicit CEFR mapping in prompt, spot-checking | | Inconsistent difficulty levels | Explicit CEFR mapping in prompt, spot-checking |
| JSON parse failures | Retry logic, schema validation, cleanup on failure | | JSON parse failures | Retry + split logic, schema validation, cleanup on failure |
| Model drift (online APIs) | Version pinning, local fallback | | Model drift (online APIs) | Version pinning, local fallback |
| Provider downtime | Adapter pattern allows hot-swapping providers |
### Why TypeScript + Node? ### Why TypeScript + Node?
@ -195,7 +212,7 @@ Generating 100,000 entries with an LLM introduces risks:
### Why llama.cpp? ### Why llama.cpp?
- **GGUF format**: Single-file models, easy to swap, quantize, and version - **GGUF format**: Single-file models, easy to swap, quantize, and version
- **OpenAI-compatible API**: `/v1/chat/completions` means the same code works for local and online models - **OpenAI-compatible API**: `/v1/chat/completions` means the same adapter code works for local and online models
- **No dependencies**: Self-contained binary, runs on old hardware (tested on GTX 950M) - **No dependencies**: Self-contained binary, runs on old hardware (tested on GTX 950M)
- **Privacy**: Local inference means no data leaves the machine - **Privacy**: Local inference means no data leaves the machine
@ -206,15 +223,16 @@ Generating 100,000 entries with an LLM introduces risks:
### Pipeline Flow ### Pipeline Flow
``` ```
Scan sources -> Check LLM -> Loop words -> Skip if exists -> Create skeleton Scan sources -> Check LLM -> Loop wordlists -> Stream words -> Skip processed
-> Call LLM -> Parse JSON -> Merge -> Write atomically -> Verify schema -> Create skeletons (batch) -> Call LLM -> Parse JSON -> Retry/split on failure
-> Merge -> Write atomically -> Verify schema -> Log metrics
``` ```
### Resumability ### Resumability
- **Skip existing**: Checks if `{word}.json` exists with non-empty `senses` - **Skip existing**: `check-if-json-exists.ts` checks if `{word}.json` exists with non-empty `senses`
- **Atomic writes**: `.tmp` -> rename, no partial files on crash - **Atomic writes**: `.tmp` -> rename in `write-json-file.ts`, no partial files on crash
- **Cleanup on failure**: Deletes incomplete file, continues to next word - **Cleanup on failure**: Deletes skeleton files for failed batches, continues to next batch
### Directory Structure ### Directory Structure
@ -222,9 +240,29 @@ Scan sources -> Check LLM -> Loop words -> Skip if exists -> Create skeleton
data-pipeline/ data-pipeline/
|-- pipeline.ts # Entry point / orchestrator |-- pipeline.ts # Entry point / orchestrator
|-- config/ |-- config/
| |-- llm.ts # API URL, model params | |-- llm.ts # Provider, API URL, model name
| |-- prompt.ts # System prompt | |-- prompt.ts # System prompt
|-- utils/ # See source files (provided separately) | |-- batch.ts # Batch size and retry config
|-- utils/
| |-- enrich-word.ts # LLM call, parse, retry/split
| |-- merge-enriched-data.ts # Schema types + merge logic
| |-- verify-enriched-file.ts # Schema validation
| |-- check-llm-server.ts # Health check
| |-- scanning-source-files.ts # Source discovery
| |-- create-base-json.ts # Skeleton writer
| |-- write-json-file.ts # Atomic JSON writer
| |-- check-if-json-exists.ts # Resumability check
| |-- create-line-reader.ts # Streaming file reader
| |-- create-output-dirs.ts # Directory creation
| |-- delete-file.ts # Cleanup helper
| |-- get-word-file-path.ts # Path helper
| |-- progress-tracker.ts # Console progress formatting
| |-- pipeline-timer.ts # Timing + token metrics
| |-- llm-adapters/
| |-- factory.ts # Adapter selection
| |-- types.ts # LlmAdapter interface
| |-- openai-compatible.ts # Local, OpenRouter, DeepSeek
| |-- gemini.ts # Google Gemini
|-- source-data/ |-- source-data/
| |-- {language}/ | |-- {language}/
| |-- {pos} # One word per line, no extension | |-- {pos} # One word per line, no extension
@ -232,22 +270,24 @@ data-pipeline/
|-- {language}/ |-- {language}/
|-- {pos}/ |-- {pos}/
|-- {word}.json # One self-contained file per word |-- {word}.json # One self-contained file per word
|-- kaikki-source-files/ # Wiktionary dumps for gender lookup (planned)
``` ```
### Output Schema ### Output Schema
Each `.json` file contains: `word`, `language`, `pos`, `senses[]` (each with `sense`, `example`, `difficulty_level`, `translations` per target language), `enrichedAt`, `model`. Each `.json` file contains: `word`, `language`, `pos`, `senses[]` (each with `id`, `sense`, `example`, `difficulty_level`, `translations` per target language), `enrichedAt`, `model`.
Full TypeScript interfaces: `utils/merge-enriched-data.ts`. Full TypeScript interfaces: `utils/merge-enriched-data.ts`.
### Error Handling ### Error Handling
| Failure | Behavior | | Failure | Behavior |
| ----------------------- | ------------------------------ | | ------------------------------ | ------------------------------------------------------- |
| LLM server offline | Hard fail at startup | | LLM server offline | Hard fail at startup (`check-llm-server.ts`) |
| LLM returns bad JSON | Log, delete skeleton, continue | | LLM returns bad JSON | Retry up to 3 times, then split batch. Log and continue |
| Schema validation fails | Log warnings, keep file | | Schema validation fails | Log warnings, keep file |
| Individual word fails | Does not stop pipeline | | Individual batch fails | Does not stop pipeline; cleans up skeletons |
| Individual word fails (size 1) | Log and continue to next word |
### Metrics ### Metrics
@ -269,18 +309,22 @@ Per-run: words processed/skipped/failed, duration, throughput, LLM token counts
### Configuration ### Configuration
| File | Purpose | | File | Purpose |
| ------------------ | -------------------------------------------------------------------------- | | ------------------ | ---------------------------------------------------------------- | ------------ | ---------- | ------------------------- |
| `config/llm.ts` | `LLM_API_URL`, `LLM_DEFAULT_PARAMS` (`temperature`, `top_p`, `max_tokens`) | | `config/llm.ts` | `provider` (`local` | `openrouter` | `deepseek` | `gemini`), `url`, `model` |
| `config/prompt.ts` | System prompt with CEFR mapping, required fields, example output | | `config/prompt.ts` | System prompt with CEFR mapping, required fields, example output |
| `config/batch.ts` | `BATCH_CONFIG.size` (words per call), `maxRetries` |
### Key Modules ### Key Modules
| File | Responsibility | | File | Responsibility |
| ------------------------------- | ------------------------------------------------------------------------------------------ | | ----------------------------------------- | ------------------------------------------------------------------------------------------------------------------------- |
| `utils/enrich-word.ts` | Calls LLM, strips markdown, parses JSON array, builds `EnrichedSense[]` with composite IDs | | `utils/enrich-word.ts` | Calls LLM via adapter, strips markdown, parses JSON array, builds `EnrichedSense[]` with composite IDs, retry/split logic |
| `utils/merge-enriched-data.ts` | Merges skeleton `{word, language, pos}` with LLM senses, adds `enrichedAt` and `model` | | `utils/merge-enriched-data.ts` | Merges skeleton `{word, language, pos}` with LLM senses, adds `enrichedAt` and `model` |
| `utils/verify-enriched-file.ts` | Schema validation: required fields, array lengths, gender enum, translation structure | | `utils/verify-enriched-file.ts` | Schema validation: required fields, array lengths, gender enum, translation structure |
| `utils/pipeline-timer.ts` | Tracks per-word and global metrics (duration, tokens, throughput) | | `utils/pipeline-timer.ts` | Tracks per-word and global metrics (duration, tokens, throughput) |
| `utils/llm-adapters/factory.ts` | Creates adapter based on `LLM_CONFIG.provider` |
| `utils/llm-adapters/openai-compatible.ts` | OpenAI chat completions API for local llama.cpp, OpenRouter, DeepSeek |
| `utils/llm-adapters/gemini.ts` | Google Gemini `generateContent` API |
### Current Model ### Current Model
@ -375,9 +419,9 @@ The server flags evolved through trial and error on the target hardware (Intel i
### Known Limitations (Current) ### Known Limitations (Current)
- **Gender accuracy**: Qwen2.5-1.5B systematically defaults to `neuter` for Romance languages. Under evaluation whether larger models fix this. - **Gender accuracy**: Qwen2.5-1.5B systematically defaults to `neuter` for Romance languages. Under evaluation whether larger models fix this.
- **No batching**: One word = one LLM call. System prompt re-processed every time.
- **No retry logic**: LLM parse failures are logged and skipped, not retried.
- **Single POS**: Only nouns tested. Verbs/adjectives/adverbs need prompt adjustments. - **Single POS**: Only nouns tested. Verbs/adjectives/adverbs need prompt adjustments.
- **Hardcoded model name**: `merge-enriched-data.ts` hardcodes `"qwen3.5-4b-q4_k_m"` regardless of actual model used.
- **Duplicated mappings**: `LANG_MAP`/`POS_MAP` exist in three separate files.
--- ---
@ -413,7 +457,7 @@ Models are evaluated on three criteria in order of priority: **quality** (defini
### 5.2 Online API Options ### 5.2 Online API Options
Evaluated as fallbacks if local models fail quality or speed targets. All support OpenAI-compatible API. Evaluated as fallbacks if local models fail quality or speed targets. All support OpenAI-compatible API (except Gemini, which has a native adapter).
| Provider | Model | Input $/1M | Output $/1M | Free Tier | Rate Limit | Est. Cost (100k words) | Est. Time | | Provider | Model | Input $/1M | Output $/1M | Free Tier | Rate Limit | Est. Cost (100k words) | Est. Time |
| ------------------- | -------------------- | ---------- | ----------- | ------------- | ---------------- | ---------------------- | ------------------- | | ------------------- | -------------------- | ---------- | ----------- | ------------- | ---------------- | ---------------------- | ------------------- |
@ -484,7 +528,7 @@ Grammatical gender is embedded in the `translations` object of each sense:
} }
``` ```
Early testing with **Qwen2.5-1.5B** showed systematic failure: the model defaults to `neuter` for any translation where it is uncertain. This is particularly broken for Romance languages (Italian, Spanish, French), which do not have a neuter grammatical gender at all - only masculine and feminine. Early testing with **Qwen2.5-1.5B** showed systematic failure: the model defaults to `neuter` for any translation where it is uncertain. This is particularly broken for Romance languages (Italian, Spanish, French), which do not have a neuter grammatical gender at all — only masculine and feminine.
Whether this is a **model size issue** (1.5B too small to retain gender facts) or a **training data gap** (Qwen2.5 family lacks gender-annotated multilingual data) is unresolved. Pending the Qwen2.5-3B evaluation. Whether this is a **model size issue** (1.5B too small to retain gender facts) or a **training data gap** (Qwen2.5 family lacks gender-annotated multilingual data) is unresolved. Pending the Qwen2.5-3B evaluation.
@ -528,27 +572,24 @@ At 1 word per call, 100,000 words = 100,000 LLM requests. Each call re-processes
Single config point controls batch size everywhere: Single config point controls batch size everywhere:
```typescript ```typescript
// config/batch.ts (planned) // config/batch.ts
export const BATCH_CONFIG = { export const BATCH_CONFIG = { size: 4, maxRetries: 3 } as const;
size: 5, // Change to 15 or 50 to test
get maxTokens() {
return Math.ceil(this.size * 250 * 1.2); // 250 tok/word + 20% buffer
},
} as const;
``` ```
Change `size` to 1, 2, 5, 10, 20, etc. The pipeline adapts without code changes.
### Prompt Structure ### Prompt Structure
**Single word:** **Single word:**
``` ```
Word: house ["house"]
``` ```
**Batch of 5:** **Batch of 4:**
``` ```
Words: ["house", "car", "tree", "water", "book"] ["house", "car", "tree", "water"]
``` ```
LLM returns a JSON object with word keys: LLM returns a JSON object with word keys:
@ -558,29 +599,28 @@ LLM returns a JSON object with word keys:
"house": [ { "sense": "...", "example": "...", ... } ], "house": [ { "sense": "...", "example": "...", ... } ],
"car": [ { ... } ], "car": [ { ... } ],
"tree": [ { ... } ], "tree": [ { ... } ],
"water": [ { ... } ], "water": [ { ... } ]
"book": [ { ... } ]
} }
``` ```
### Retry Strategy ### Retry & Split Strategy
If a batch fails (bad JSON, missing key, etc.): If a batch fails (bad JSON, missing key, etc.):
``` ```
Batch of 50 fails Batch of 4 fails (3 retries exhausted)
| |
v v
Retry as 2 batches of 25 Split into 2 batches of 2
| |
v v
If a 25 fails, retry as 5 batches of 5 If a batch of 2 fails (3 retries), split into 2 batches of 1
| |
v v
If a 5 fails, retry as individual words (fallback) If a single word fails (3 retries), log and skip
``` ```
This gives resilience without losing the speed benefit of large batches. This gives resilience without losing the speed benefit of large batches. The `maxRetries` config controls how many attempts are made before splitting.
### Expected Impact by Environment ### Expected Impact by Environment
@ -682,12 +722,13 @@ For each word and each candidate model:
### Near-Term (Next 2-4 Weeks) ### Near-Term (Next 2-4 Weeks)
| Item | Status | Notes | | Item | Status | Notes |
| --------------------- | ----------- | ------------------------------------------- | | ------------------------ | ------------ | --------------------------------------------------------------- |
| Configurable batching | In progress | Single `BATCH_CONFIG.size` value | | Configurable batching | **Complete** | Single `BATCH_CONFIG.size` value, retry/split logic implemented |
| 20-word torture suite | Pending | Decides gender approach and model selection | | 20-word torture suite | Pending | Decides gender approach and model selection |
| Qwen2.5-3B evaluation | Pending | Download and test | | Qwen2.5-3B evaluation | Pending | Download and test |
| Online API testing | Pending | Gemini free tier, DeepSeek, Groq | | Online API testing | Pending | Gemini free tier, DeepSeek, Groq |
| Retry logic | Pending | Exponential backoff on LLM failures | | Fix hardcoded model name | Pending | `merge-enriched-data.ts` hardcodes `"qwen3.5-4b"` |
| Extract shared constants | Pending | `LANG_MAP`/`POS_MAP` duplicated in 3 files |
### Medium-Term (1-3 Months) ### Medium-Term (1-3 Months)
@ -697,7 +738,7 @@ For each word and each candidate model:
| Multi-language source | German -> French, Italian -> Spanish, etc. | | Multi-language source | German -> French, Italian -> Spanish, etc. |
| Parallel wordlist processing | Run `english/nouns` and `english/verbs` simultaneously | | Parallel wordlist processing | Run `english/nouns` and `english/verbs` simultaneously |
| Incremental enrichment | Only process new/changed words in a wordlist | | Incremental enrichment | Only process new/changed words in a wordlist |
| Model auto-switching | Fallback to online API if local server fails | | Model auto-switching | Fallback to online API if local server fails mid-run |
### Long-Term (3-6 Months) ### Long-Term (3-6 Months)
@ -713,7 +754,7 @@ For each word and each candidate model:
## 11. Decisions Log ## 11. Decisions Log
| Date | Decision | Context | Rationale | | Date | Decision | Context | Rationale |
| ---------- | ----------------------------- | ---------------------------------------------------- | --------------------------------------------------------------------------- | | ---------- | --------------------------------- | ---------------------------------------------------- | --------------------------------------------------------------------------- |
| 2026-01-04 | TanStack Router for frontend | Previous project used React Router | Simpler, type-safe routing for the trainer app | | 2026-01-04 | TanStack Router for frontend | Previous project used React Router | Simpler, type-safe routing for the trainer app |
| 2026-01-04 | Vite dev server (no Nginx) | Docker setup for glossa-web | Nginx unnecessary for dev; Vite handles HMR and proxying | | 2026-01-04 | Vite dev server (no Nginx) | Docker setup for glossa-web | Nginx unnecessary for dev; Vite handles HMR and proxying |
| 2026-01-17 | Backend answer verification | Security vulnerability: correctAnswer exposed in API | Moved verification to server-side, shared schemas | | 2026-01-17 | Backend answer verification | Security vulnerability: correctAnswer exposed in API | Moved verification to server-side, shared schemas |
@ -725,6 +766,8 @@ For each word and each candidate model:
| 2026-06-17 | Qwen2.5-1.5B as current model | Qwen3.5-4B too slow (47s/word) | 6x speedup (8s/word), quality under evaluation | | 2026-06-17 | Qwen2.5-1.5B as current model | Qwen3.5-4B too slow (47s/word) | 6x speedup (8s/word), quality under evaluation |
| 2026-06-17 | Skip Gemma 4 | E2B Q4_K_M is 3.46GB | Does not fit in 4GB VRAM; lower quants sacrifice quality | | 2026-06-17 | Skip Gemma 4 | E2B Q4_K_M is 3.46GB | Does not fit in 4GB VRAM; lower quants sacrifice quality |
| 2026-06-17 | Skip Ministral-3B | Tokenizer mismatch (Tekken) | Outputs gibberish regardless of template; not fixable without re-conversion | | 2026-06-17 | Skip Ministral-3B | Tokenizer mismatch (Tekken) | Outputs gibberish regardless of template; not fixable without re-conversion |
| 2026-07-06 | Adapter pattern for LLM providers | Need to evaluate local vs cloud | `utils/llm-adapters/` with factory + types + per-provider implementations |
| 2026-07-06 | Retry + split batching | LLM JSON parse failures on larger batches | `enrichWordWithRetry` retries 3 times, then halves batch until size 1 |
--- ---
@ -740,12 +783,14 @@ For each word and each candidate model:
### Data Pipeline ### Data Pipeline
| Issue | Details | Severity | | Issue | Details | Severity |
| --------------------------------- | ------------------------------------------------------------------------- | ------------------- | | --------------------------------- | ----------------------------------------------------------------------------------------------- | -------------------------------- |
| Ministral-3B tokenizer mismatch | Tekken tokenizer not properly converted to GGUF. Model outputs gibberish. | Blocker - abandoned | | Ministral-3B tokenizer mismatch | Tekken tokenizer not properly converted to GGUF. Model outputs gibberish. | Blocker - abandoned |
| Qwen2.5-1.5B gender hallucination | Systematic `neuter` default for Romance languages. | Under evaluation | | Qwen2.5-1.5B gender hallucination | Systematic `neuter` default for Romance languages. | Under evaluation |
| No batching | 1 word = 1 call. System prompt re-processed every time. | In progress | | Hardcoded model name | `merge-enriched-data.ts` always writes `"qwen3.5-4b-q4_k_m"` regardless of actual model. | Minor - fix before production |
| No retry logic | LLM parse failures are logged and skipped. | Planned | | Duplicated LANG_MAP/POS_MAP | Identical mapping objects in `create-base-json.ts`, `merge-enriched-data.ts`, `enrich-word.ts`. | Minor - refactor risk |
| OpenAI-compatible timings | `json.timings` is llama.cpp-specific. Will break for OpenRouter/DeepSeek. | Medium - needs graceful fallback |
| Single POS tested | Only nouns validated. Verbs/adjectives need prompt changes. | Known limitation | | Single POS tested | Only nouns validated. Verbs/adjectives need prompt changes. | Known limitation |
| Batch metrics averaging | Split-and-merge averages tokens/sec instead of weighting by token count. | Minor - summary stats only |
### Hardware ### Hardware
@ -763,10 +808,16 @@ For each word and each candidate model:
- Node.js + npm - Node.js + npm
- `tsx` installed globally: `npm install -g tsx` - `tsx` installed globally: `npm install -g tsx`
- llama.cpp built from source - llama.cpp built from source (for local mode)
- GGUF model downloaded to `~/Downloads/llama.cpp/models/` - GGUF model downloaded to `~/Downloads/llama.cpp/models/` (for local mode)
- API keys set as environment variables (for cloud mode):
```bash
export DEEPSEEK_API_KEY="sk-..."
export GEMINI_API_KEY="..."
export OPENROUTER_API_KEY="..."
```
### Start the LLM Server ### Start the LLM Server (Local Mode)
```bash ```bash
cd ~/Downloads/llama.cpp cd ~/Downloads/llama.cpp
@ -785,6 +836,26 @@ cd ~/Downloads/llama.cpp
--prio 2 --prio 2
``` ```
### Configure Provider
Edit `config/llm.ts`:
```typescript
// Local
export const LLM_CONFIG = {
provider: "local" as const,
url: "http://127.0.0.1:8080/v1/chat/completions",
model: undefined,
};
// Gemini
export const LLM_CONFIG = {
provider: "gemini" as const,
url: "",
model: "gemini-2.5-flash-lite",
};
```
### Run the Pipeline ### Run the Pipeline
```bash ```bash
@ -798,48 +869,54 @@ npx tsx pipeline.ts
Starting data pipeline... Starting data pipeline...
step 1: scanning the source files... step 1: scanning the source files...
Scan complete! Found 1 wordlist(s): ✅ Scan complete! Found 1 wordlist(s):
- ENGLISH (nouns) • ENGLISH (nouns)
... step 2: creating necessary output folders...
✅ All required output directories have been verified and created successfully.
Pipeline Summary step 3: verifying local AI engine status...
Duration: 23.0s 🟢 Local AI engine is connected and ready for inference!
Processed: 3
step 4: looping through the wordlists...
Reading list: [ENGLISH] -> [NOUNS]
Batch 1/1: [house, car, tree, water]
[1/4] (0 failed) Enriched and saved: house.json
[2/4] (0 failed) Enriched and saved: car.json
[3/4] (0 failed) Enriched and saved: tree.json
[4/4] (0 failed) Enriched and saved: water.json
⏱️ Word took 8.2s
⏱️ Pipeline Summary
Duration: 32.8s
Processed: 4
Skipped: 0 Skipped: 0
Failed: 0 Failed: 0
Total: 3 Total: 4
Throughput: 0.13 words/sec Throughput: 0.12 words/sec
LLM Metrics 🤖 LLM Metrics
Calls: 3 Calls: 1
Avg prompt tokens: 308 Avg prompt tokens: 312
Avg completion tokens: 132 Avg completion tokens: 524
Avg prompt speed: 549.4 tok/s Avg prompt speed: 548.2 tok/s
Avg completion speed: 18.8 tok/s Avg completion speed: 18.8 tok/s
Global data pipeline run completed successfully. Global data pipeline run completed successfully.
``` ```
### Environment Variables (Online Mode)
```bash
export DEEPSEEK_API_KEY="sk-..."
export GEMINI_API_KEY="..."
export GROQ_API_KEY="..."
```
Then update `config/llm.ts` to point to the online API URL.
--- ---
## 14. Roadmap ## 14. Roadmap
### Phase 1: Batching (Current) ### Phase 1: Batching (Complete)
| Task | Status | Notes | | Task | Status | Notes |
| ------------------------------------- | ----------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------- | | ------------------------------------- | ------------ | ------------------------------------------------------------------------------------ |
| Implement configurable batch size | In progress | Single `BATCH_CONFIG.size` value. Prompt formatting: single word -&gt; word array. Response parsing: keyed JSON object. Retry: 50 -&gt; 25 -&gt; 5 -&gt; 1. | | Implement configurable batch size | **Complete** | `config/batch.ts` with `size` and `maxRetries` |
| Implement retry + split logic | **Complete** | `enrichWordWithRetry`: 3 retries, then halve batch |
| Verify batching doesn't break quality | Pending | Run 20-word torture suite on Qwen2.5-1.5B with batch sizes 1, 5, 15. Compare output. | | Verify batching doesn't break quality | Pending | Run 20-word torture suite on Qwen2.5-1.5B with batch sizes 1, 5, 15. Compare output. |
| Measure speedup vs batch size | Pending | Track throughput at 1, 5, 15 on local hardware. | | Measure speedup vs batch size | Pending | Track throughput at 1, 5, 15 on local hardware. |
@ -850,13 +927,13 @@ Then update `config/llm.ts` to point to the online API URL.
### Phase 2: Model Selection ### Phase 2: Model Selection
| Task | Status | Notes | | Task | Status | Notes |
| ------------------------------------------------ | ------- | ------------------------------------------------------------------------------------------- | | ------------------------------------------------ | ------- | ---------------------------------------------------------------------------------------- |
| Download Qwen2.5-3B Q4_K_M | Pending | ~1.9GB, fits in 4GB VRAM. | | Download Qwen2.5-3B Q4_K_M | Pending | ~1.9GB, fits in 4GB VRAM. |
| Run torture suite on Qwen2.5-3B (batched) | Pending | Check if gender accuracy improves with 2x params. | | Run torture suite on Qwen2.5-3B (batched) | Pending | Check if gender accuracy improves with 2x params. |
| Test Gemini 2.5 Flash-Lite free tier (batched) | Pending | 1,500 req/day x 50 words = 75k words/day. Zero cost. | | Test Gemini 2.5 Flash-Lite free tier (batched) | Pending | 1,500 req/day x 50 words = 75k words/day. Zero cost. |
| Test DeepSeek V4 Flash free tier (batched) | Pending | 5M tokens free. ~9k words. | | Test DeepSeek V4 Flash free tier (batched) | Pending | 5M tokens free. ~9k words. |
| Test Groq Llama 3.1 8B (batched, paid if needed) | Pending | Fastest option. ~$7 for 100k words. | | Test Groq Llama 3.1 8B (batched, paid if needed) | Pending | Fastest option. ~$7 for 100k words. |
| Decide: local vs online, which model | Pending | Criteria: quality &gt;= 90% gender, 100% JSON, sensible definitions. Then speed, then cost. | | Decide: local vs online, which model | Pending | Criteria: quality >= 90% gender, 100% JSON, sensible definitions. Then speed, then cost. |
**Goal:** Pick the model and provider for the 100k word run. **Goal:** Pick the model and provider for the 100k word run.
@ -878,9 +955,9 @@ Then update `config/llm.ts` to point to the online API URL.
### Phase 4: Extend ### Phase 4: Extend
| Task | Status | Notes | | Task | Status | Notes |
| ---------------------------- | ------- | ------------------------------------------------------------------------------------------------- | | ---------------------------- | ------- | ------------------------------------------------------------------------------------------- |
| Multi-POS support | Pending | Verbs, adjectives, adverbs. Each needs prompt variants (conjugations, agreement, etc.). | | Multi-POS support | Pending | Verbs, adjectives, adverbs. Each needs prompt variants (conjugations, agreement, etc.). |
| Multi-language source | Pending | German -&gt; French, Italian -&gt; Spanish, etc. Schema already supports any source/target combo. | | Multi-language source | Pending | German -> French, Italian -> Spanish, etc. Schema already supports any source/target combo. |
| Parallel wordlist processing | Pending | Run `english/nouns` and `english/verbs` simultaneously. | | Parallel wordlist processing | Pending | Run `english/nouns` and `english/verbs` simultaneously. |
| Incremental enrichment | Pending | Only process new/changed words in a wordlist. | | Incremental enrichment | Pending | Only process new/changed words in a wordlist. |
| GPU rental integration | Pending | Script to spin up Vast.ai/RunPod, run pipeline, download results. | | GPU rental integration | Pending | Script to spin up Vast.ai/RunPod, run pipeline, download results. |
@ -893,9 +970,12 @@ Then update `config/llm.ts` to point to the online API URL.
### Backlog (Unscheduled) ### Backlog (Unscheduled)
| Task | Context | | Task | Context |
| -------------------------------- | ------------------------------------------------------------------------------------------- | | ------------------------------------------- | ------------------------------------------------------------------------------------------- |
| Batch API discounts | Gemini, Qwen, Azure offer 50% off for 24h SLA. Relevant if running recurring large batches. | | Batch API discounts | Gemini, Qwen, Azure offer 50% off for 24h SLA. Relevant if running recurring large batches. |
| Model auto-switching | Fallback to online API if local server fails mid-run. | | Model auto-switching | Fallback to online API if local server fails mid-run. |
| Community open-source | Clean up, document, publish for other language learners. | | Community open-source | Clean up, document, publish for other language learners. |
| Prometheus metrics | `--metrics` flag on llama-server for automated performance tracking. | | Prometheus metrics | `--metrics` flag on llama-server for automated performance tracking. |
| `-c 1024` / `-b 256` experiments | Further VRAM optimization on GTX 950M. Low priority if moving to cloud. | | `-c 1024` / `-b 256` experiments | Further VRAM optimization on GTX 950M. Low priority if moving to cloud. |
| Extract shared LANG_MAP/POS_MAP | Single source of truth for language/pos mappings. |
| Fix hardcoded model name | Pass actual model name through enrichment chain. |
| Graceful timing fallback for cloud adapters | Handle missing `timings` field in OpenRouter/DeepSeek responses. |