wip

README.md

@@ -99,9 +99,3 @@ lila/
 ├── documentation/   — Project docs (this directory)
 └── Caddyfile, docker-compose.yml, etc.
 ```
-
----
-
-## License
-
-TBD

data-pipeline/config/batch.ts

@@ -0,0 +1,3 @@
+export const BATCH_CONFIG = { size: 4, maxRetries: 3 } as const;
+
+//1, 2, 5, 10, 20

data-pipeline/config/llm.ts

@@ -0,0 +1,5 @@
+export const LLM_CONFIG = {
+  provider: "local" as "local" | "openrouter" | "deepseek" | "gemini",
+  url: "http://127.0.0.1:8080/v1/chat/completions",
+  model: undefined as string | undefined,
+} as const;

data-pipeline/config/prompt.ts

@@ -0,0 +1,34 @@
+export const ENRICHMENT_SYSTEM_PROMPT = `You are a multilingual dictionary engine. Output ONLY a JSON object. No markdown, no explanations.
+
+For each English noun provided, generate 1-2 distinct senses.
+
+CEFR difficulty mapping:
+- A1/A2 → easy
+- B1/B2 → medium
+- C1/C2 → hard
+
+Each sense must have:
+- sense: student-friendly definition, max 15 words
+- example: natural sentence using the word
+- difficulty_level: easy, medium, or hard
+- translations: object with keys de, it, es, fr; each value is an array of {word, gender} where gender MUST be masculine, feminine, or neuter. Use null ONLY if the language has no grammatical gender for that word.
+
+Output format: JSON object where keys are the input words, values are arrays of sense objects.
+
+Example for ["house"]:
+{
+  "house": [
+    {
+      "sense": "A building for human habitation.",
+      "example": "They bought a house in the city.",
+      "difficulty_level": "easy",
+      "translations": {
+        "de": [{"word": "Haus", "gender": "neuter"}],
+        "it": [{"word": "casa", "gender": "feminine"}],
+        "es": [{"word": "casa", "gender": "feminine"}],
+        "fr": [{"word": "maison", "gender": "feminine"}]
+      }
+    }
+  ]
+}
+`;

data-pipeline/pipeline.ts

@@ -0,0 +1,160 @@
+import { isWordProcessed } from "./utils/check-if-json-exists.js";
+import { createBaseJson } from "./utils/create-base-json.js";
+import { ensureOutputFolders } from "./utils/create-output-dirs.js";
+import { scanSourceData } from "./utils/scanning-source-files.js";
+import { createLineReader } from "./utils/create-line-reader.js";
+import { checkLlmServer } from "./utils/check-llm-server.js";
+import { getWordFilePath } from "./utils/get-word-file-path.js";
+import { mergeEnrichedData } from "./utils/merge-enriched-data.js";
+import { enrichWordWithRetry } from "./utils/enrich-word.js";
+import { writeJsonFile } from "./utils/write-json-file.js";
+import { deleteFileIfExists } from "./utils/delete-file.js";
+import { PipelineTimer } from "./utils/pipeline-timer.js";
+import { ProgressTracker } from "./utils/progress-tracker.js";
+import { verifyEnrichedFile } from "./utils/verify-enriched-file.js";
+import { BATCH_CONFIG } from "./config/batch.js";
+
+async function main() {
+  console.log("Starting data pipeline...\n");
+
+  const timer = new PipelineTimer();
+
+  // step 1: scanning for source files
+  console.log("\n step 1: scanning the source files...");
+  const wordlists = scanSourceData(import.meta.dirname);
+
+  // step 2: ensuring output folders exist
+  console.log("\n step 2: creating necessary output folders...");
+  ensureOutputFolders(wordlists);
+
+  // step 3: check to verify the AI engine is ready before touching anything
+  console.log("\n step 3: verifying local AI engine status...");
+  await checkLlmServer();
+
+  // Step 4: Loop through the wordlists array
+  console.log("\n step 4: looping through the wordlists...");
+
+  for (const wordlist of wordlists) {
+    console.log(
+      `\nReading list: [${wordlist.language.toUpperCase()}] -> [${wordlist.pos.toUpperCase()}]`,
+    );
+
+    const rl = createLineReader(wordlist.sourcePath);
+
+    // Collect words and count them
+    const words: string[] = [];
+    for await (const line of rl) {
+      const word = line.trim().toLowerCase();
+      if (word) words.push(word);
+    }
+
+    // Filter out already-processed words
+    const unprocessedWords = words.filter(
+      (word) => !isWordProcessed(word, wordlist.outputDir),
+    );
+
+    const skippedCount = words.length - unprocessedWords.length;
+    if (skippedCount > 0) {
+      console.log(`  Skipped ${skippedCount} already-processed words`);
+    }
+
+    const progress = new ProgressTracker(unprocessedWords.length);
+
+    // Step 5: Process in batches
+    for (let i = 0; i < unprocessedWords.length; i += BATCH_CONFIG.size) {
+      const batch = unprocessedWords.slice(i, i + BATCH_CONFIG.size);
+      const batchNum = Math.floor(i / BATCH_CONFIG.size) + 1;
+      const totalBatches = Math.ceil(
+        unprocessedWords.length / BATCH_CONFIG.size,
+      );
+      const batchLabel = `Batch ${batchNum}/${totalBatches}`;
+
+      console.log(`\n  ${batchLabel}: [${batch.join(", ")}]`);
+
+      // Create skeletons for all words in batch
+      for (const word of batch) {
+        createBaseJson(
+          word,
+          wordlist.outputDir,
+          wordlist.language,
+          wordlist.pos,
+        );
+      }
+
+      timer.startWord();
+
+      try {
+        // Step 6: enrich batch with senses (with retry/split)
+        const result = await enrichWordWithRetry(
+          batch,
+          wordlist.language,
+          wordlist.pos,
+        );
+
+        // Step 7: write each word's result
+        for (const [word, senses] of result.results) {
+          progress.next();
+          console.log(
+            `  ${progress.format(`Enriched and saved: ${word}.json`)}`,
+          );
+
+          const targetFilePath = getWordFilePath(word, wordlist.outputDir);
+          const enrichedData = mergeEnrichedData(
+            word,
+            wordlist.language,
+            wordlist.pos,
+            senses,
+          );
+
+          writeJsonFile(targetFilePath, enrichedData);
+
+          // Verify the generated file
+          const verification = verifyEnrichedFile(targetFilePath);
+          if (!verification.valid) {
+            console.error(`  Warning: Schema violations in ${word}.json:`);
+            for (const error of verification.errors) {
+              console.error(`    - ${error}`);
+            }
+          }
+
+          timer.recordProcessed({
+            promptTokens: result.metrics.promptTokens / batch.length,
+            completionTokens: result.metrics.completionTokens / batch.length,
+            totalTokens: result.metrics.totalTokens / batch.length,
+            promptTimeMs: result.metrics.promptTimeMs / batch.length,
+            completionTimeMs: result.metrics.completionTimeMs / batch.length,
+            promptTokensPerSecond: result.metrics.promptTokensPerSecond,
+            completionTokensPerSecond: result.metrics.completionTokensPerSecond,
+          });
+        }
+
+        console.log(`  ${timer.getWordTiming()}`);
+      } catch (error: unknown) {
+        const errorMessage =
+          error instanceof Error ? error.message : String(error);
+        console.error(
+          `  Failed to enrich batch [${batch.join(", ")}]: ${errorMessage}`,
+        );
+
+        // Cleanup: delete skeleton files for failed batch
+        for (const word of batch) {
+          const targetFilePath = getWordFilePath(word, wordlist.outputDir);
+          deleteFileIfExists(targetFilePath);
+          console.log(`  Removed incomplete file: ${word}.json`);
+          progress.recordFailed();
+        }
+
+        timer.recordFailed();
+      }
+    }
+  }
+
+  timer.stop();
+  console.log("\n" + timer.getSummary());
+  console.log("\nGlobal data pipeline run completed successfully.");
+}
+
+// Fire the orchestrator block
+main().catch((err) => {
+  console.error("Critical unexpected pipeline failure:", err);
+});

data-pipeline/source-data/english/nouns

@@ -0,0 +1,20 @@
+house
+time
+water
+year
+people
+day
+way
+man
+woman
+child
+work
+life
+world
+hand
+eye
+book
+friend
+school
+city
+family

data-pipeline/utils/check-if-json-exists.ts

@@ -0,0 +1,23 @@
+import fs from "fs";
+import path from "path";
+
+/**
+ * Checks if a JSON file for the given word exists AND contains enriched data.
+ * Returns false for skeleton files (missing senses array).
+ */
+export function isWordProcessed(word: string, outputDir: string): boolean {
+  const targetFilePath = path.join(outputDir, `${word}.json`);
+
+  if (!fs.existsSync(targetFilePath)) {
+    return false;
+  }
+
+  try {
+    const content = fs.readFileSync(targetFilePath, "utf-8");
+    const data = JSON.parse(content) as Record<string, unknown>;
+    return Array.isArray(data["senses"]) && data["senses"].length > 0;
+  } catch (_error: unknown) {
+    // Corrupted file => treat as not processed
+    return false;
+  }
+}

data-pipeline/utils/check-llm-server.ts

@@ -0,0 +1,43 @@
+/**
+ * Pings the local llama.cpp server to ensure it's up, running, and has a model loaded.
+ * If the server is offline or still loading, it terminates the pipeline gracefully.
+ */
+export async function checkLlmServer(
+  url = "http://127.0.0.1:8080/health",
+): Promise<void> {
+  try {
+    const response = await fetch(url);
+
+    // llama.cpp returns a 503 status if the server is up but the model weights are still loading
+    if (response.status === 503) {
+      console.error(
+        "\n ⏳ Local AI engine is starting up, but the model is still loading into memory.",
+      );
+      console.error(
+        "👉 Please wait a minute for the weights to load, then run the pipeline again.\n",
+      );
+      process.exit(1);
+    }
+
+    // Parse the JSON health response (expected: { status: "ok" })
+    const data = (await response.json()) as { status?: string };
+
+    if (response.ok && data.status === "ok") {
+      console.log("🟢 Local AI engine is connected and ready for inference!");
+      return;
+    }
+
+    // Catch-all for unexpected active server responses
+    console.error(
+      `\n ❌ Unknown response from local AI engine health check (Status: ${response.status}).`,
+    );
+    process.exit(1);
+  } catch (_error: unknown) {
+    console.error("\n ❌ Could not connect to the local AI engine.");
+    console.error(`🔗 Attempted endpoint: ${url}`);
+    console.error(
+      "👉 Make sure your './llama-server' command is actively running in another terminal tab!\n",
+    );
+    process.exit(1);
+  }
+}

data-pipeline/utils/create-base-json.ts

@@ -0,0 +1,41 @@
+import fs from "fs";
+import path from "path";
+
+const LANG_MAP: Record<string, string> = {
+  english: "en",
+  italian: "it",
+  german: "de",
+  french: "fr",
+  spanish: "es",
+};
+
+const POS_MAP: Record<string, string> = {
+  nouns: "noun",
+  verbs: "verb",
+  adverbs: "adverb",
+  adjectives: "adjective",
+};
+
+/**
+ * Creates the base JSON file with word, language, and pos.
+ * No logging — the orchestrator handles all console output.
+ */
+export function createBaseJson(
+  word: string,
+  outputDir: string,
+  rawLanguage: string,
+  rawPos: string,
+): void {
+  const targetFilePath = path.join(outputDir, `${word}.json`);
+
+  const dbLanguage = LANG_MAP[rawLanguage] || rawLanguage;
+  const dbPos = POS_MAP[rawPos] || rawPos;
+
+  const initialData = { word, language: dbLanguage, pos: dbPos };
+
+  fs.writeFileSync(
+    targetFilePath,
+    JSON.stringify(initialData, null, 2),
+    "utf-8",
+  );
+}

data-pipeline/utils/create-line-reader.ts

@@ -0,0 +1,11 @@
+import fs from "fs";
+import readline from "readline";
+
+/**
+ * Creates a line-by-line reader stream for a given file path.
+ */
+export function createLineReader(sourcePath: string): readline.Interface {
+  const fileStream = fs.createReadStream(sourcePath, "utf-8");
+
+  return readline.createInterface({ input: fileStream, crlfDelay: Infinity });
+}

data-pipeline/utils/create-output-dirs.ts

@@ -0,0 +1,20 @@
+import fs from "fs";
+import type { Wordlist } from "./scanning-source-files.js";
+
+/**
+ * Takes a list of scanned datasets and creates their output folders if missing.
+ */
+export function ensureOutputFolders(wordlists: Wordlist[]): void {
+  for (const wordlist of wordlists) {
+    if (!fs.existsSync(wordlist.outputDir)) {
+      fs.mkdirSync(wordlist.outputDir, { recursive: true });
+      console.log(
+        `📁 Created target folder: worddata/${wordlist.language}/${wordlist.pos}`,
+      );
+    }
+  }
+
+  console.log(
+    "✅ All required output directories have been verified and created successfully.",
+  );
+}

data-pipeline/utils/delete-file.ts

@@ -0,0 +1,10 @@
+import fs from "fs";
+
+/**
+ * Deletes a file if it exists. Silently ignores missing files.
+ */
+export function deleteFileIfExists(filePath: string): void {
+  if (fs.existsSync(filePath)) {
+    fs.unlinkSync(filePath);
+  }
+}

data-pipeline/utils/enrich-word.ts

@@ -0,0 +1,240 @@
+// utils/enrich-word.ts
+import { ENRICHMENT_SYSTEM_PROMPT } from "../config/prompt.js";
+import { createAdapter } from "./llm-adapters/factory.js";
+import { BATCH_CONFIG } from "../config/batch.js";
+import type { Language, Pos, EnrichedSense } from "./merge-enriched-data.js";
+
+const LANG_MAP: Record<string, string> = {
+  english: "en",
+  italian: "it",
+  german: "de",
+  french: "fr",
+  spanish: "es",
+};
+
+const POS_MAP: Record<string, string> = {
+  nouns: "noun",
+  verbs: "verb",
+  adverbs: "adverb",
+  adjectives: "adjective",
+};
+
+interface LlmResponse {
+  content: string;
+  promptTokens: number;
+  completionTokens: number;
+  totalTokens: number;
+  promptTimeMs: number;
+  completionTimeMs: number;
+  promptTokensPerSecond: number;
+  completionTokensPerSecond: number;
+}
+
+export interface EnrichmentResult {
+  results: Map<string, EnrichedSense[]>;
+  metrics: {
+    promptTokens: number;
+    completionTokens: number;
+    totalTokens: number;
+    promptTimeMs: number;
+    completionTimeMs: number;
+    promptTokensPerSecond: number;
+    completionTokensPerSecond: number;
+  };
+}
+
+/**
+ * Calls the local LLM with the enrichment prompt.
+ * Returns the response content and timing metrics.
+ */
+async function callLlm(words: string[]): Promise<LlmResponse> {
+  const adapter = createAdapter();
+  return adapter.call(words, ENRICHMENT_SYSTEM_PROMPT);
+}
+
+/**
+ * Strips markdown code blocks and extracts the JSON object from raw LLM output.
+ * Throws if no valid JSON object braces are found.
+ */
+function sanitizeLlmOutput(raw: string): string {
+  const cleaned = raw.replace(/```json\s*/g, "").replace(/```\s*$/g, "");
+  const start = cleaned.indexOf("{");
+  const end = cleaned.lastIndexOf("}");
+  if (start === -1 || end === -1) {
+    throw new Error("No JSON object found in LLM output");
+  }
+  return cleaned.slice(start, end + 1);
+}
+
+/**
+ * Parses the LLM response string into a JavaScript object.
+ * Throws if the response is not valid JSON or not an object with expected keys.
+ */
+export function parseLlmResponse(
+  rawJson: string,
+  expectedWords: string[],
+): Record<string, unknown> {
+  let parsed: unknown;
+
+  try {
+    const sanitized = sanitizeLlmOutput(rawJson);
+    parsed = JSON.parse(sanitized);
+  } catch (error: unknown) {
+    throw new Error(`Failed to parse LLM output as JSON: ${rawJson}`, {
+      cause: error,
+    });
+  }
+
+  if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) {
+    throw new Error("LLM output is not a JSON object");
+  }
+
+  const obj = parsed as Record<string, unknown>;
+
+  for (const word of expectedWords) {
+    if (!(word in obj)) {
+      throw new Error(`Missing key in LLM output: "${word}"`);
+    }
+    if (!Array.isArray(obj[word]) || (obj[word] as unknown[]).length === 0) {
+      throw new Error(`LLM output for "${word}" is not a non-empty array`);
+    }
+  }
+
+  return obj;
+}
+
+/**
+ * Takes parsed LLM output and builds final enriched objects with composite IDs.
+ */
+export function buildEnrichedData(
+  parsed: Record<string, unknown>,
+  rawLanguage: string,
+  rawPos: string,
+): Map<string, EnrichedSense[]> {
+  const language = (LANG_MAP[rawLanguage] || rawLanguage) as Language;
+  const pos = (POS_MAP[rawPos] || rawPos) as Pos;
+  const results = new Map<string, EnrichedSense[]>();
+
+  for (const [word, sensesArray] of Object.entries(parsed)) {
+    const senses = (sensesArray as unknown[]).map((item, index) => {
+      const sense = item as Omit<
+        EnrichedSense,
+        "id" | "word" | "language" | "pos"
+      >;
+
+      return {
+        id: `${word}:${language}:${pos}:${index}`,
+        word,
+        language,
+        pos,
+        ...sense,
+      } as EnrichedSense;
+    });
+
+    results.set(word, senses);
+  }
+
+  return results;
+}
+
+/**
+ * Enriches a batch of words by calling the LLM, parsing the response, and building final data.
+ */
+export async function enrichWord(
+  words: string[],
+  rawLanguage: string,
+  rawPos: string,
+): Promise<EnrichmentResult> {
+  const llmResponse = await callLlm(words);
+  const parsed = parseLlmResponse(llmResponse.content, words);
+  const results = buildEnrichedData(parsed, rawLanguage, rawPos);
+
+  return {
+    results,
+    metrics: {
+      promptTokens: llmResponse.promptTokens,
+      completionTokens: llmResponse.completionTokens,
+      totalTokens: llmResponse.totalTokens,
+      promptTimeMs: llmResponse.promptTimeMs,
+      completionTimeMs: llmResponse.completionTimeMs,
+      promptTokensPerSecond: llmResponse.promptTokensPerSecond,
+      completionTokensPerSecond: llmResponse.completionTokensPerSecond,
+    },
+  };
+}
+
+/**
+ * Enriches a batch of words with retry and split-on-failure logic.
+ * Retries up to BATCH_CONFIG.maxRetries times, then splits batch in half and retries each half.
+ * Continues splitting until batch size is 1, then throws if still failing.
+ */
+export async function enrichWordWithRetry(
+  words: string[],
+  rawLanguage: string,
+  rawPos: string,
+  attempt: number = 1,
+): Promise<EnrichmentResult> {
+  try {
+    return await enrichWord(words, rawLanguage, rawPos);
+  } catch (error: unknown) {
+    const errorMessage = error instanceof Error ? error.message : String(error);
+
+    if (words.length === 1) {
+      throw new Error(
+        `Failed to enrich word "${words[0]}" after ${attempt} attempts: ${errorMessage}`,
+        { cause: error },
+      );
+    }
+
+    if (attempt < BATCH_CONFIG.maxRetries) {
+      console.log(
+        `  Retry ${attempt}/${BATCH_CONFIG.maxRetries} for batch [${words.join(", ")}]: ${errorMessage}`,
+      );
+      return enrichWordWithRetry(words, rawLanguage, rawPos, attempt + 1);
+    }
+
+    // Max retries reached — split and retry
+    console.log(
+      `  Splitting batch [${words.join(", ")}] after ${BATCH_CONFIG.maxRetries} failed attempts`,
+    );
+
+    const half = Math.ceil(words.length / 2);
+    const left = words.slice(0, half);
+    const right = words.slice(half);
+
+    const leftResult = await enrichWordWithRetry(left, rawLanguage, rawPos, 1);
+    const rightResult = await enrichWordWithRetry(
+      right,
+      rawLanguage,
+      rawPos,
+      1,
+    );
+
+    // Merge results
+    const merged = new Map([...leftResult.results, ...rightResult.results]);
+    const mergedMetrics = {
+      promptTokens:
+        leftResult.metrics.promptTokens + rightResult.metrics.promptTokens,
+      completionTokens:
+        leftResult.metrics.completionTokens +
+        rightResult.metrics.completionTokens,
+      totalTokens:
+        leftResult.metrics.totalTokens + rightResult.metrics.totalTokens,
+      promptTimeMs:
+        leftResult.metrics.promptTimeMs + rightResult.metrics.promptTimeMs,
+      completionTimeMs:
+        leftResult.metrics.completionTimeMs +
+        rightResult.metrics.completionTimeMs,
+      promptTokensPerSecond:
+        (leftResult.metrics.promptTokensPerSecond +
+          rightResult.metrics.promptTokensPerSecond) /
+        2,
+      completionTokensPerSecond:
+        (leftResult.metrics.completionTokensPerSecond +
+          rightResult.metrics.completionTokensPerSecond) /
+        2,
+    };
+
+    return { results: merged, metrics: mergedMetrics };
+  }
+}

data-pipeline/utils/get-word-file-path.ts

@@ -0,0 +1,5 @@
+import path from "path";
+
+export function getWordFilePath(word: string, outputDir: string): string {
+  return path.join(outputDir, `${word}.json`);
+}

data-pipeline/utils/llm-adapters/factory.ts

@@ -0,0 +1,30 @@
+import { LLM_CONFIG } from "../../config/llm.js";
+import { OpenAiCompatibleAdapter } from "./openai-compatible.js";
+import { GeminiAdapter } from "./gemini.js";
+import type { LlmAdapter } from "./types.js";
+
+export function createAdapter(): LlmAdapter {
+  switch (LLM_CONFIG.provider) {
+    case "local":
+      return new OpenAiCompatibleAdapter(LLM_CONFIG.url);
+    case "openrouter":
+      return new OpenAiCompatibleAdapter(
+        LLM_CONFIG.url,
+        process.env["OPENROUTER_API_KEY"],
+        LLM_CONFIG.model,
+      );
+    case "deepseek":
+      return new OpenAiCompatibleAdapter(
+        LLM_CONFIG.url,
+        process.env["DEEPSEEK_API_KEY"],
+        LLM_CONFIG.model,
+      );
+    case "gemini": {
+      const apiKey = process.env["GEMINI_API_KEY"];
+      if (!apiKey) throw new Error("GEMINI_API_KEY env var not set");
+      if (!LLM_CONFIG.model)
+        throw new Error("LLM_CONFIG.model required for gemini");
+      return new GeminiAdapter(apiKey, LLM_CONFIG.model);
+    }
+  }
+}

data-pipeline/utils/llm-adapters/gemini.ts

@@ -0,0 +1,92 @@
+import type { LlmAdapter } from "./types.js";
+
+interface GeminiResponse {
+  candidates: Array<{ content: { parts: Array<{ text: string }> } }>;
+  usageMetadata: {
+    promptTokenCount: number;
+    candidatesTokenCount: number;
+    totalTokenCount: number;
+  };
+}
+
+export class GeminiAdapter implements LlmAdapter {
+  private apiKey: string;
+  private model: string;
+
+  constructor(apiKey: string, model: string) {
+    this.apiKey = apiKey;
+    this.model = model;
+  }
+
+  async call(
+    words: string[],
+    systemPrompt: string,
+  ): Promise<{
+    content: string;
+    promptTokens: number;
+    completionTokens: number;
+    totalTokens: number;
+    promptTimeMs: number;
+    completionTimeMs: number;
+    promptTokensPerSecond: number;
+    completionTokensPerSecond: number;
+  }> {
+    const url = `https://generativelanguage.googleapis.com/v1beta/models/${this.model}:generateContent?key=${this.apiKey}`;
+
+    const payload = {
+      contents: [
+        {
+          role: "user",
+          parts: [
+            { text: systemPrompt + "\n\nWords: " + JSON.stringify(words) },
+          ],
+        },
+      ],
+      generationConfig: {
+        temperature: 0.1,
+        topP: 0.9,
+        maxOutputTokens: Math.ceil(words.length * 250 * 1.2),
+      },
+    };
+
+    const startTime = Date.now();
+
+    const response = await fetch(url, {
+      method: "POST",
+      headers: { "Content-Type": "application/json" },
+      body: JSON.stringify(payload),
+    });
+
+    const totalTimeMs = Date.now() - startTime;
+
+    if (!response.ok) {
+      throw new Error(`Gemini API responded with status: ${response.status}`);
+    }
+
+    const json = (await response.json()) as GeminiResponse;
+
+    const content = json.candidates[0]?.content?.parts[0]?.text;
+    if (!content) {
+      throw new Error("Gemini response content is empty");
+    }
+
+    const promptTokens = json.usageMetadata.promptTokenCount;
+    const completionTokens = json.usageMetadata.candidatesTokenCount;
+    const totalTokens = json.usageMetadata.totalTokenCount;
+
+    // Gemini doesn't provide timing breakdown, so we estimate
+    const promptTimeMs = totalTimeMs * 0.3; // rough estimate
+    const completionTimeMs = totalTimeMs * 0.7; // rough estimate
+
+    return {
+      content,
+      promptTokens,
+      completionTokens,
+      totalTokens,
+      promptTimeMs,
+      completionTimeMs,
+      promptTokensPerSecond: promptTokens / (promptTimeMs / 1000),
+      completionTokensPerSecond: completionTokens / (completionTimeMs / 1000),
+    };
+  }
+}

data-pipeline/utils/llm-adapters/openai-compatible.ts

@@ -0,0 +1,92 @@
+import type { LlmAdapter } from "./types.js";
+
+interface OpenAiResponse {
+  choices: Array<{ message: { content: string } }>;
+  usage: {
+    prompt_tokens: number;
+    completion_tokens: number;
+    total_tokens: number;
+  };
+  timings: {
+    prompt_ms: number;
+    predicted_ms: number;
+    prompt_per_second: number;
+    predicted_per_second: number;
+  };
+}
+
+export class OpenAiCompatibleAdapter implements LlmAdapter {
+  private url: string;
+  private apiKey: string | undefined;
+  private model: string | undefined;
+
+  constructor(url: string, apiKey?: string, model?: string) {
+    this.url = url;
+    this.apiKey = apiKey;
+    this.model = model;
+  }
+
+  async call(
+    words: string[],
+    systemPrompt: string,
+  ): Promise<{
+    content: string;
+    promptTokens: number;
+    completionTokens: number;
+    totalTokens: number;
+    promptTimeMs: number;
+    completionTimeMs: number;
+    promptTokensPerSecond: number;
+    completionTokensPerSecond: number;
+  }> {
+    const payload: Record<string, unknown> = {
+      messages: [
+        { role: "system", content: systemPrompt },
+        { role: "user", content: JSON.stringify(words) },
+      ],
+      temperature: 0.1,
+      top_p: 0.9,
+      max_tokens: Math.ceil(words.length * 250 * 1.2),
+    };
+
+    if (this.model) {
+      payload["model"] = this.model;
+    }
+
+    const headers: Record<string, string> = {
+      "Content-Type": "application/json",
+    };
+
+    if (this.apiKey) {
+      headers["Authorization"] = `Bearer ${this.apiKey}`;
+    }
+
+    const response = await fetch(this.url, {
+      method: "POST",
+      headers,
+      body: JSON.stringify(payload),
+    });
+
+    if (!response.ok) {
+      throw new Error(`LLM server responded with status: ${response.status}`);
+    }
+
+    const json = (await response.json()) as OpenAiResponse;
+
+    const content = json.choices[0]?.message?.content;
+    if (!content) {
+      throw new Error("LLM response content is empty");
+    }
+
+    return {
+      content,
+      promptTokens: json.usage.prompt_tokens,
+      completionTokens: json.usage.completion_tokens,
+      totalTokens: json.usage.total_tokens,
+      promptTimeMs: json.timings.prompt_ms,
+      completionTimeMs: json.timings.predicted_ms,
+      promptTokensPerSecond: json.timings.prompt_per_second,
+      completionTokensPerSecond: json.timings.predicted_per_second,
+    };
+  }
+}

data-pipeline/utils/llm-adapters/types.ts

@@ -0,0 +1,15 @@
+export interface LlmAdapter {
+  call(
+    words: string[],
+    systemPrompt: string,
+  ): Promise<{
+    content: string;
+    promptTokens: number;
+    completionTokens: number;
+    totalTokens: number;
+    promptTimeMs: number;
+    completionTimeMs: number;
+    promptTokensPerSecond: number;
+    completionTokensPerSecond: number;
+  }>;
+}

data-pipeline/utils/merge-enriched-data.ts

@@ -0,0 +1,69 @@
+export type Language = "en" | "de" | "it" | "es" | "fr";
+export type Pos = "noun" | "verb" | "adjective" | "adverb";
+export type Gender = "masculine" | "feminine" | "neuter" | null;
+export type Difficulty = "easy" | "medium" | "hard";
+
+export interface Translation {
+  word: string;
+  gender: Gender;
+}
+
+export interface EnrichedSense {
+  id: string;
+  word: string;
+  language: Language;
+  pos: Pos;
+  sense: string;
+  example: string;
+  difficulty_level: Difficulty;
+  translations: {
+    de: Translation[];
+    it: Translation[];
+    es: Translation[];
+    fr: Translation[];
+  };
+}
+
+const LANG_MAP: Record<string, Language> = {
+  english: "en",
+  italian: "it",
+  german: "de",
+  french: "fr",
+  spanish: "es",
+};
+
+const POS_MAP: Record<string, Pos> = {
+  nouns: "noun",
+  verbs: "verb",
+  adverbs: "adverb",
+  adjectives: "adjective",
+};
+
+/**
+ * Merges skeleton data with enriched LLM senses into the final pipeline output.
+ */
+export function mergeEnrichedData(
+  word: string,
+  rawLanguage: string,
+  rawPos: string,
+  senses: EnrichedSense[],
+): Record<string, unknown> {
+  const language = LANG_MAP[rawLanguage] || (rawLanguage as Language);
+  const pos = POS_MAP[rawPos] || (rawPos as Pos);
+
+  const fixedSenses = senses.map((sense, index) => ({
+    ...sense,
+    id: `${word}:${language}:${pos}:${index}`,
+    language,
+    pos,
+  }));
+
+  return {
+    word,
+    language,
+    pos,
+    senses: fixedSenses,
+    enrichedAt: new Date().toISOString(),
+    model: "qwen3.5-4b-q4_k_m",
+  };
+}

data-pipeline/utils/pipeline-timer.ts

@@ -0,0 +1,162 @@
+interface LlmMetrics {
+  promptTokens: number;
+  completionTokens: number;
+  totalTokens: number;
+  promptTimeMs: number;
+  completionTimeMs: number;
+  promptTokensPerSecond: number;
+  completionTokensPerSecond: number;
+}
+
+interface PipelineMetrics {
+  startTime: Date;
+  endTime?: Date;
+  wordsProcessed: number;
+  wordsSkipped: number;
+  wordsFailed: number;
+  llmCalls: number;
+  totalPromptTokens: number;
+  totalCompletionTokens: number;
+  totalTokens: number;
+  totalPromptTimeMs: number;
+  totalCompletionTimeMs: number;
+  currentWordStartTime?: Date;
+}
+
+/**
+ * Simple timer and metrics tracker for the pipeline.
+ * Tracks both pipeline throughput and LLM performance.
+ */
+export class PipelineTimer {
+  private metrics: PipelineMetrics;
+
+  constructor() {
+    this.metrics = {
+      startTime: new Date(),
+      wordsProcessed: 0,
+      wordsSkipped: 0,
+      wordsFailed: 0,
+      llmCalls: 0,
+      totalPromptTokens: 0,
+      totalCompletionTokens: 0,
+      totalTokens: 0,
+      totalPromptTimeMs: 0,
+      totalCompletionTimeMs: 0,
+    };
+  }
+
+  startWord(): void {
+    this.metrics.currentWordStartTime = new Date();
+  }
+
+  getWordDurationMs(): number {
+    if (!this.metrics.currentWordStartTime) return 0;
+    return new Date().getTime() - this.metrics.currentWordStartTime.getTime();
+  }
+
+  recordProcessed(llmMetrics?: LlmMetrics): void {
+    this.metrics.wordsProcessed++;
+    if (llmMetrics) {
+      this.metrics.llmCalls++;
+      this.metrics.totalPromptTokens += llmMetrics.promptTokens;
+      this.metrics.totalCompletionTokens += llmMetrics.completionTokens;
+      this.metrics.totalTokens += llmMetrics.totalTokens;
+      this.metrics.totalPromptTimeMs += llmMetrics.promptTimeMs;
+      this.metrics.totalCompletionTimeMs += llmMetrics.completionTimeMs;
+    }
+  }
+
+  recordSkipped(): void {
+    this.metrics.wordsSkipped++;
+  }
+
+  recordFailed(): void {
+    this.metrics.wordsFailed++;
+  }
+
+  stop(): void {
+    this.metrics.endTime = new Date();
+  }
+
+  getWordTiming(): string {
+    const durationMs = this.getWordDurationMs();
+    const durationSec = (durationMs / 1000).toFixed(1);
+    return `⏱️  Word took ${durationSec}s`;
+  }
+
+  getEta(totalWords: number): string {
+    const processed = this.metrics.wordsProcessed;
+    const remaining = totalWords - processed - this.metrics.wordsSkipped;
+
+    if (processed === 0 || remaining <= 0) return "ETA: calculating...";
+
+    const elapsedMs = new Date().getTime() - this.metrics.startTime.getTime();
+    const avgMsPerWord = elapsedMs / processed;
+    const etaMs = avgMsPerWord * remaining;
+
+    const etaMin = Math.round(etaMs / 60000);
+    const etaHour = (etaMs / 3600000).toFixed(1);
+
+    if (etaMin < 60) {
+      return `ETA: ${etaMin} min`;
+    }
+    return `ETA: ${etaHour} hours`;
+  }
+
+  getSummary(): string {
+    const end = this.metrics.endTime || new Date();
+    const durationMs = end.getTime() - this.metrics.startTime.getTime();
+    const durationSec = (durationMs / 1000).toFixed(1);
+
+    const total =
+      this.metrics.wordsProcessed +
+      this.metrics.wordsSkipped +
+      this.metrics.wordsFailed;
+    const throughput =
+      this.metrics.wordsProcessed > 0
+        ? (this.metrics.wordsProcessed / (durationMs / 1000)).toFixed(2)
+        : "0";
+
+    const avgPromptTokens =
+      this.metrics.llmCalls > 0
+        ? (this.metrics.totalPromptTokens / this.metrics.llmCalls).toFixed(0)
+        : "0";
+    const avgCompletionTokens =
+      this.metrics.llmCalls > 0
+        ? (this.metrics.totalCompletionTokens / this.metrics.llmCalls).toFixed(
+            0,
+          )
+        : "0";
+    const avgPromptSpeed =
+      this.metrics.totalPromptTimeMs > 0
+        ? (
+            this.metrics.totalPromptTokens /
+            (this.metrics.totalPromptTimeMs / 1000)
+          ).toFixed(1)
+        : "0";
+    const avgCompletionSpeed =
+      this.metrics.totalCompletionTimeMs > 0
+        ? (
+            this.metrics.totalCompletionTokens /
+            (this.metrics.totalCompletionTimeMs / 1000)
+          ).toFixed(1)
+        : "0";
+
+    return [
+      `⏱️  Pipeline Summary`,
+      `   Duration: ${durationSec}s`,
+      `   Processed: ${this.metrics.wordsProcessed}`,
+      `   Skipped: ${this.metrics.wordsSkipped}`,
+      `   Failed: ${this.metrics.wordsFailed}`,
+      `   Total: ${total}`,
+      `   Throughput: ${throughput} words/sec`,
+      ``,
+      `🤖 LLM Metrics`,
+      `   Calls: ${this.metrics.llmCalls}`,
+      `   Avg prompt tokens: ${avgPromptTokens}`,
+      `   Avg completion tokens: ${avgCompletionTokens}`,
+      `   Avg prompt speed: ${avgPromptSpeed} tok/s`,
+      `   Avg completion speed: ${avgCompletionSpeed} tok/s`,
+    ].join("\n");
+  }
+}

data-pipeline/utils/progress-tracker.ts

@@ -0,0 +1,27 @@
+/**
+ * Simple progress tracker for pipeline execution.
+ */
+export class ProgressTracker {
+  private current: number;
+  private failed: number;
+  private total: number;
+
+  constructor(total: number) {
+    this.current = 0;
+    this.failed = 0;
+    this.total = total;
+  }
+
+  next(): number {
+    this.current++;
+    return this.current;
+  }
+
+  recordFailed(): void {
+    this.failed++;
+  }
+
+  format(label: string): string {
+    return `[${this.current}/${this.total}] (${this.failed} failed) ${label}`;
+  }
+}

data-pipeline/utils/scanning-source-files.ts

@@ -0,0 +1,61 @@
+import fs from "fs";
+import path from "path";
+
+// Define a simple shape for what a discovered dataset looks like
+export interface Wordlist {
+  language: string;
+  pos: string;
+  sourcePath: string;
+  outputDir: string;
+}
+
+/**
+ * Scans the source-data directory to find all available word lists.
+ */
+export function scanSourceData(baseDir: string): Wordlist[] {
+  const sourceBaseDir = path.join(baseDir, "source-data");
+  const discoveredWordlists: Wordlist[] = [];
+
+  // Safety check: if there's no source-data folder, return an empty array
+  if (!fs.existsSync(sourceBaseDir)) {
+    return discoveredWordlists;
+  }
+
+  // 1. Read the language directories (e.g., ['english'])
+  const languages = fs.readdirSync(sourceBaseDir);
+
+  for (const lang of languages) {
+    const langFolderPath = path.join(sourceBaseDir, lang);
+
+    // Make sure it's a directory, not a stray file
+    if (!fs.statSync(langFolderPath).isDirectory()) continue;
+
+    // 2. Read the files inside the language folder (e.g., ['nouns'])
+    const posFiles = fs.readdirSync(langFolderPath);
+
+    for (const pos of posFiles) {
+      const fullSourcePath = path.join(langFolderPath, pos);
+
+      // Make sure it's a file (like your extensionless "nouns" file)
+      if (!fs.statSync(fullSourcePath).isFile()) continue;
+
+      // 3. Package everything into a flat item and add it to our array
+      discoveredWordlists.push({
+        language: lang,
+        pos: pos,
+        sourcePath: fullSourcePath,
+        outputDir: path.join(baseDir, "worddata", lang, pos),
+      });
+    }
+  }
+
+  // show summary
+  console.log(
+    `✅ Scan complete! Found ${discoveredWordlists.length} wordlist(s):`,
+  );
+  for (const list of discoveredWordlists) {
+    console.log(`   • ${list.language.toUpperCase()} (${list.pos})`);
+  }
+
+  return discoveredWordlists;
+}

data-pipeline/utils/verify-enriched-file.ts

@@ -0,0 +1,96 @@
+import fs from "fs";
+
+interface VerificationResult {
+  valid: boolean;
+  errors: string[];
+}
+
+/**
+ * Verifies that an enriched JSON file matches the expected schema.
+ * Returns detailed error messages for any violations.
+ */
+export function verifyEnrichedFile(filePath: string): VerificationResult {
+  const errors: string[] = [];
+
+  if (!fs.existsSync(filePath)) {
+    return { valid: false, errors: ["File does not exist"] };
+  }
+
+  let data: unknown;
+  try {
+    data = JSON.parse(fs.readFileSync(filePath, "utf-8"));
+  } catch (_error: unknown) {
+    return { valid: false, errors: ["Invalid JSON syntax"] };
+  }
+
+  if (typeof data !== "object" || data === null || Array.isArray(data)) {
+    return { valid: false, errors: ["Root must be an object"] };
+  }
+
+  const obj = data as Record<string, unknown>;
+
+  // Required top-level fields
+  const requiredFields = ["word", "language", "pos", "senses"];
+  for (const field of requiredFields) {
+    if (!(field in obj)) {
+      errors.push(`Missing required field: "${field}"`);
+    }
+  }
+
+  // Validate senses array
+  if (!Array.isArray(obj["senses"])) {
+    errors.push('"senses" must be an array');
+  } else if (obj["senses"].length === 0) {
+    errors.push('"senses" array cannot be empty');
+  } else {
+    for (let i = 0; i < obj["senses"].length; i++) {
+      const sense = obj["senses"][i] as Record<string, unknown>;
+      const sensePrefix = `senses[${i}]`;
+
+      if (!sense["sense"] || typeof sense["sense"] !== "string") {
+        errors.push(`${sensePrefix}: missing or invalid "sense"`);
+      }
+      if (!sense["example"] || typeof sense["example"] !== "string") {
+        errors.push(`${sensePrefix}: missing or invalid "example"`);
+      }
+      if (
+        !["easy", "medium", "hard"].includes(
+          sense["difficulty_level"] as string,
+        )
+      ) {
+        errors.push(`${sensePrefix}: invalid "difficulty_level"`);
+      }
+      if (!sense["translations"] || typeof sense["translations"] !== "object") {
+        errors.push(`${sensePrefix}: missing "translations"`);
+      } else {
+        const trans = sense["translations"] as Record<string, unknown>;
+        for (const lang of ["de", "it", "es", "fr"]) {
+          if (!Array.isArray(trans[lang])) {
+            errors.push(
+              `${sensePrefix}: missing or invalid "${lang}" translations`,
+            );
+          } else {
+            for (let j = 0; j < (trans[lang] as unknown[]).length; j++) {
+              const t = (trans[lang] as unknown[])[j] as Record<
+                string,
+                unknown
+              >;
+              if (!t["word"] || typeof t["word"] !== "string") {
+                errors.push(`${sensePrefix}.${lang}[${j}]: missing "word"`);
+              }
+              if (
+                !["masculine", "feminine", "neuter", null].includes(
+                  t["gender"] as string | null,
+                )
+              ) {
+                errors.push(`${sensePrefix}.${lang}[${j}]: invalid "gender"`);
+              }
+            }
+          }
+        }
+      }
+    }
+  }
+
+  return { valid: errors.length === 0, errors };
+}

data-pipeline/utils/write-json-file.ts

@@ -0,0 +1,11 @@
+import fs from "fs";
+
+/**
+ * Writes data as formatted JSON to a file path.
+ * Safely catches and re-throws file system errors.
+ */
+export function writeJsonFile(filePath: string, data: unknown): void {
+  const tempPath = `${filePath}.tmp`;
+  fs.writeFileSync(tempPath, JSON.stringify(data, null, 2), "utf-8");
+  fs.renameSync(tempPath, filePath);
+}

documentation/pipeline/ENGLISH_NOUNS.md

@@ -76,7 +76,7 @@ example output:
   "headword": "house",
   "language": "en",
   "pos": "noun",
-  "glosses": ["A building for human habitation."],
+  "sense": ["A building for human habitation."],
   "examples": ["They bought a house in the city."],
   "translations": {
     "de": [{ "word": "Haus", "gender": "neuter" }],
@@ -90,7 +90,7 @@ example output:
   "headword": "house",
   "language": "en",
   "pos": "noun",
-  "glosses": ["A noble family or lineage."],
+  "sense": ["A noble family or lineage."],
   "examples": ["The House of Tudor ruled England."],
   "translations": {
     "de": [
@@ -104,7 +104,7 @@ example output:
   "headword": "bank",
   "language": "en",
   "pos": "noun",
-  "glosses": ["An institution where one can place and borrow money."],
+  "sense": ["An institution where one can place and borrow money."],
   "examples": ["She deposited her paycheck at the bank."],
   "translations": {
     "de": [{ "word": "Bank", "gender": "feminine" }],
@@ -118,7 +118,7 @@ example output:
   "headword": "bank",
   "language": "en",
   "pos": "noun",
-  "glosses": ["The land alongside a river or lake."],
+  "sense": ["The land alongside a river or lake."],
   "examples": ["They picnicked on the bank of the river."],
   "translations": {
     "de": [{ "word": "Ufer", "gender": "neuter" }],
@@ -132,7 +132,7 @@ example output:
   "headword": "bank",
   "language": "en",
   "pos": "noun",
-  "glosses": ["A collection or store of something held in reserve."],
+  "sense": ["A collection or store of something held in reserve."],
   "examples": ["The hospital keeps a blood bank."],
   "translations": {
     "de": [{ "word": "Bank", "gender": "feminine" }]
@@ -336,7 +336,7 @@ read to decide, not to archive.
 Notes:
 
 - the miss list needs to get verified/re-worked later on
-- the kaikki data contains ipas and links to audio files, add them later if needed, they are not needed now"
+- the kaikki data contains ipas and links to audio files, add them later if needed, they are not needed now
 - use online llms to set the difficulty(cefr) of the not shipped words
 - add plurals from kaikki/wiktionary
 - postgres sync to prod db