wip

data-pipeline/utils/check-if-json-exists.ts

@@ -0,0 +1,23 @@
+import fs from "fs";
+import path from "path";
+
+/**
+ * Checks if a JSON file for the given word exists AND contains enriched data.
+ * Returns false for skeleton files (missing senses array).
+ */
+export function isWordProcessed(word: string, outputDir: string): boolean {
+  const targetFilePath = path.join(outputDir, `${word}.json`);
+
+  if (!fs.existsSync(targetFilePath)) {
+    return false;
+  }
+
+  try {
+    const content = fs.readFileSync(targetFilePath, "utf-8");
+    const data = JSON.parse(content) as Record<string, unknown>;
+    return Array.isArray(data["senses"]) && data["senses"].length > 0;
+  } catch (_error: unknown) {
+    // Corrupted file => treat as not processed
+    return false;
+  }
+}

data-pipeline/utils/check-llm-server.ts

@@ -0,0 +1,43 @@
+/**
+ * Pings the local llama.cpp server to ensure it's up, running, and has a model loaded.
+ * If the server is offline or still loading, it terminates the pipeline gracefully.
+ */
+export async function checkLlmServer(
+  url = "http://127.0.0.1:8080/health",
+): Promise<void> {
+  try {
+    const response = await fetch(url);
+
+    // llama.cpp returns a 503 status if the server is up but the model weights are still loading
+    if (response.status === 503) {
+      console.error(
+        "\n ⏳ Local AI engine is starting up, but the model is still loading into memory.",
+      );
+      console.error(
+        "👉 Please wait a minute for the weights to load, then run the pipeline again.\n",
+      );
+      process.exit(1);
+    }
+
+    // Parse the JSON health response (expected: { status: "ok" })
+    const data = (await response.json()) as { status?: string };
+
+    if (response.ok && data.status === "ok") {
+      console.log("🟢 Local AI engine is connected and ready for inference!");
+      return;
+    }
+
+    // Catch-all for unexpected active server responses
+    console.error(
+      `\n ❌ Unknown response from local AI engine health check (Status: ${response.status}).`,
+    );
+    process.exit(1);
+  } catch (_error: unknown) {
+    console.error("\n ❌ Could not connect to the local AI engine.");
+    console.error(`🔗 Attempted endpoint: ${url}`);
+    console.error(
+      "👉 Make sure your './llama-server' command is actively running in another terminal tab!\n",
+    );
+    process.exit(1);
+  }
+}

data-pipeline/utils/create-base-json.ts

@@ -0,0 +1,41 @@
+import fs from "fs";
+import path from "path";
+
+const LANG_MAP: Record<string, string> = {
+  english: "en",
+  italian: "it",
+  german: "de",
+  french: "fr",
+  spanish: "es",
+};
+
+const POS_MAP: Record<string, string> = {
+  nouns: "noun",
+  verbs: "verb",
+  adverbs: "adverb",
+  adjectives: "adjective",
+};
+
+/**
+ * Creates the base JSON file with word, language, and pos.
+ * No logging — the orchestrator handles all console output.
+ */
+export function createBaseJson(
+  word: string,
+  outputDir: string,
+  rawLanguage: string,
+  rawPos: string,
+): void {
+  const targetFilePath = path.join(outputDir, `${word}.json`);
+
+  const dbLanguage = LANG_MAP[rawLanguage] || rawLanguage;
+  const dbPos = POS_MAP[rawPos] || rawPos;
+
+  const initialData = { word, language: dbLanguage, pos: dbPos };
+
+  fs.writeFileSync(
+    targetFilePath,
+    JSON.stringify(initialData, null, 2),
+    "utf-8",
+  );
+}

data-pipeline/utils/create-line-reader.ts

@@ -0,0 +1,11 @@
+import fs from "fs";
+import readline from "readline";
+
+/**
+ * Creates a line-by-line reader stream for a given file path.
+ */
+export function createLineReader(sourcePath: string): readline.Interface {
+  const fileStream = fs.createReadStream(sourcePath, "utf-8");
+
+  return readline.createInterface({ input: fileStream, crlfDelay: Infinity });
+}

data-pipeline/utils/create-output-dirs.ts

@@ -0,0 +1,20 @@
+import fs from "fs";
+import type { Wordlist } from "./scanning-source-files.js";
+
+/**
+ * Takes a list of scanned datasets and creates their output folders if missing.
+ */
+export function ensureOutputFolders(wordlists: Wordlist[]): void {
+  for (const wordlist of wordlists) {
+    if (!fs.existsSync(wordlist.outputDir)) {
+      fs.mkdirSync(wordlist.outputDir, { recursive: true });
+      console.log(
+        `📁 Created target folder: worddata/${wordlist.language}/${wordlist.pos}`,
+      );
+    }
+  }
+
+  console.log(
+    "✅ All required output directories have been verified and created successfully.",
+  );
+}

data-pipeline/utils/delete-file.ts

@@ -0,0 +1,10 @@
+import fs from "fs";
+
+/**
+ * Deletes a file if it exists. Silently ignores missing files.
+ */
+export function deleteFileIfExists(filePath: string): void {
+  if (fs.existsSync(filePath)) {
+    fs.unlinkSync(filePath);
+  }
+}

data-pipeline/utils/enrich-word.ts

@@ -0,0 +1,240 @@
+// utils/enrich-word.ts
+import { ENRICHMENT_SYSTEM_PROMPT } from "../config/prompt.js";
+import { createAdapter } from "./llm-adapters/factory.js";
+import { BATCH_CONFIG } from "../config/batch.js";
+import type { Language, Pos, EnrichedSense } from "./merge-enriched-data.js";
+
+const LANG_MAP: Record<string, string> = {
+  english: "en",
+  italian: "it",
+  german: "de",
+  french: "fr",
+  spanish: "es",
+};
+
+const POS_MAP: Record<string, string> = {
+  nouns: "noun",
+  verbs: "verb",
+  adverbs: "adverb",
+  adjectives: "adjective",
+};
+
+interface LlmResponse {
+  content: string;
+  promptTokens: number;
+  completionTokens: number;
+  totalTokens: number;
+  promptTimeMs: number;
+  completionTimeMs: number;
+  promptTokensPerSecond: number;
+  completionTokensPerSecond: number;
+}
+
+export interface EnrichmentResult {
+  results: Map<string, EnrichedSense[]>;
+  metrics: {
+    promptTokens: number;
+    completionTokens: number;
+    totalTokens: number;
+    promptTimeMs: number;
+    completionTimeMs: number;
+    promptTokensPerSecond: number;
+    completionTokensPerSecond: number;
+  };
+}
+
+/**
+ * Calls the local LLM with the enrichment prompt.
+ * Returns the response content and timing metrics.
+ */
+async function callLlm(words: string[]): Promise<LlmResponse> {
+  const adapter = createAdapter();
+  return adapter.call(words, ENRICHMENT_SYSTEM_PROMPT);
+}
+
+/**
+ * Strips markdown code blocks and extracts the JSON object from raw LLM output.
+ * Throws if no valid JSON object braces are found.
+ */
+function sanitizeLlmOutput(raw: string): string {
+  const cleaned = raw.replace(/```json\s*/g, "").replace(/```\s*$/g, "");
+  const start = cleaned.indexOf("{");
+  const end = cleaned.lastIndexOf("}");
+  if (start === -1 || end === -1) {
+    throw new Error("No JSON object found in LLM output");
+  }
+  return cleaned.slice(start, end + 1);
+}
+
+/**
+ * Parses the LLM response string into a JavaScript object.
+ * Throws if the response is not valid JSON or not an object with expected keys.
+ */
+export function parseLlmResponse(
+  rawJson: string,
+  expectedWords: string[],
+): Record<string, unknown> {
+  let parsed: unknown;
+
+  try {
+    const sanitized = sanitizeLlmOutput(rawJson);
+    parsed = JSON.parse(sanitized);
+  } catch (error: unknown) {
+    throw new Error(`Failed to parse LLM output as JSON: ${rawJson}`, {
+      cause: error,
+    });
+  }
+
+  if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) {
+    throw new Error("LLM output is not a JSON object");
+  }
+
+  const obj = parsed as Record<string, unknown>;
+
+  for (const word of expectedWords) {
+    if (!(word in obj)) {
+      throw new Error(`Missing key in LLM output: "${word}"`);
+    }
+    if (!Array.isArray(obj[word]) || (obj[word] as unknown[]).length === 0) {
+      throw new Error(`LLM output for "${word}" is not a non-empty array`);
+    }
+  }
+
+  return obj;
+}
+
+/**
+ * Takes parsed LLM output and builds final enriched objects with composite IDs.
+ */
+export function buildEnrichedData(
+  parsed: Record<string, unknown>,
+  rawLanguage: string,
+  rawPos: string,
+): Map<string, EnrichedSense[]> {
+  const language = (LANG_MAP[rawLanguage] || rawLanguage) as Language;
+  const pos = (POS_MAP[rawPos] || rawPos) as Pos;
+  const results = new Map<string, EnrichedSense[]>();
+
+  for (const [word, sensesArray] of Object.entries(parsed)) {
+    const senses = (sensesArray as unknown[]).map((item, index) => {
+      const sense = item as Omit<
+        EnrichedSense,
+        "id" | "word" | "language" | "pos"
+      >;
+
+      return {
+        id: `${word}:${language}:${pos}:${index}`,
+        word,
+        language,
+        pos,
+        ...sense,
+      } as EnrichedSense;
+    });
+
+    results.set(word, senses);
+  }
+
+  return results;
+}
+
+/**
+ * Enriches a batch of words by calling the LLM, parsing the response, and building final data.
+ */
+export async function enrichWord(
+  words: string[],
+  rawLanguage: string,
+  rawPos: string,
+): Promise<EnrichmentResult> {
+  const llmResponse = await callLlm(words);
+  const parsed = parseLlmResponse(llmResponse.content, words);
+  const results = buildEnrichedData(parsed, rawLanguage, rawPos);
+
+  return {
+    results,
+    metrics: {
+      promptTokens: llmResponse.promptTokens,
+      completionTokens: llmResponse.completionTokens,
+      totalTokens: llmResponse.totalTokens,
+      promptTimeMs: llmResponse.promptTimeMs,
+      completionTimeMs: llmResponse.completionTimeMs,
+      promptTokensPerSecond: llmResponse.promptTokensPerSecond,
+      completionTokensPerSecond: llmResponse.completionTokensPerSecond,
+    },
+  };
+}
+
+/**
+ * Enriches a batch of words with retry and split-on-failure logic.
+ * Retries up to BATCH_CONFIG.maxRetries times, then splits batch in half and retries each half.
+ * Continues splitting until batch size is 1, then throws if still failing.
+ */
+export async function enrichWordWithRetry(
+  words: string[],
+  rawLanguage: string,
+  rawPos: string,
+  attempt: number = 1,
+): Promise<EnrichmentResult> {
+  try {
+    return await enrichWord(words, rawLanguage, rawPos);
+  } catch (error: unknown) {
+    const errorMessage = error instanceof Error ? error.message : String(error);
+
+    if (words.length === 1) {
+      throw new Error(
+        `Failed to enrich word "${words[0]}" after ${attempt} attempts: ${errorMessage}`,
+        { cause: error },
+      );
+    }
+
+    if (attempt < BATCH_CONFIG.maxRetries) {
+      console.log(
+        `  Retry ${attempt}/${BATCH_CONFIG.maxRetries} for batch [${words.join(", ")}]: ${errorMessage}`,
+      );
+      return enrichWordWithRetry(words, rawLanguage, rawPos, attempt + 1);
+    }
+
+    // Max retries reached — split and retry
+    console.log(
+      `  Splitting batch [${words.join(", ")}] after ${BATCH_CONFIG.maxRetries} failed attempts`,
+    );
+
+    const half = Math.ceil(words.length / 2);
+    const left = words.slice(0, half);
+    const right = words.slice(half);
+
+    const leftResult = await enrichWordWithRetry(left, rawLanguage, rawPos, 1);
+    const rightResult = await enrichWordWithRetry(
+      right,
+      rawLanguage,
+      rawPos,
+      1,
+    );
+
+    // Merge results
+    const merged = new Map([...leftResult.results, ...rightResult.results]);
+    const mergedMetrics = {
+      promptTokens:
+        leftResult.metrics.promptTokens + rightResult.metrics.promptTokens,
+      completionTokens:
+        leftResult.metrics.completionTokens +
+        rightResult.metrics.completionTokens,
+      totalTokens:
+        leftResult.metrics.totalTokens + rightResult.metrics.totalTokens,
+      promptTimeMs:
+        leftResult.metrics.promptTimeMs + rightResult.metrics.promptTimeMs,
+      completionTimeMs:
+        leftResult.metrics.completionTimeMs +
+        rightResult.metrics.completionTimeMs,
+      promptTokensPerSecond:
+        (leftResult.metrics.promptTokensPerSecond +
+          rightResult.metrics.promptTokensPerSecond) /
+        2,
+      completionTokensPerSecond:
+        (leftResult.metrics.completionTokensPerSecond +
+          rightResult.metrics.completionTokensPerSecond) /
+        2,
+    };
+
+    return { results: merged, metrics: mergedMetrics };
+  }
+}

data-pipeline/utils/get-word-file-path.ts

@@ -0,0 +1,5 @@
+import path from "path";
+
+export function getWordFilePath(word: string, outputDir: string): string {
+  return path.join(outputDir, `${word}.json`);
+}

data-pipeline/utils/llm-adapters/factory.ts

@@ -0,0 +1,30 @@
+import { LLM_CONFIG } from "../../config/llm.js";
+import { OpenAiCompatibleAdapter } from "./openai-compatible.js";
+import { GeminiAdapter } from "./gemini.js";
+import type { LlmAdapter } from "./types.js";
+
+export function createAdapter(): LlmAdapter {
+  switch (LLM_CONFIG.provider) {
+    case "local":
+      return new OpenAiCompatibleAdapter(LLM_CONFIG.url);
+    case "openrouter":
+      return new OpenAiCompatibleAdapter(
+        LLM_CONFIG.url,
+        process.env["OPENROUTER_API_KEY"],
+        LLM_CONFIG.model,
+      );
+    case "deepseek":
+      return new OpenAiCompatibleAdapter(
+        LLM_CONFIG.url,
+        process.env["DEEPSEEK_API_KEY"],
+        LLM_CONFIG.model,
+      );
+    case "gemini": {
+      const apiKey = process.env["GEMINI_API_KEY"];
+      if (!apiKey) throw new Error("GEMINI_API_KEY env var not set");
+      if (!LLM_CONFIG.model)
+        throw new Error("LLM_CONFIG.model required for gemini");
+      return new GeminiAdapter(apiKey, LLM_CONFIG.model);
+    }
+  }
+}

data-pipeline/utils/llm-adapters/gemini.ts

@@ -0,0 +1,92 @@
+import type { LlmAdapter } from "./types.js";
+
+interface GeminiResponse {
+  candidates: Array<{ content: { parts: Array<{ text: string }> } }>;
+  usageMetadata: {
+    promptTokenCount: number;
+    candidatesTokenCount: number;
+    totalTokenCount: number;
+  };
+}
+
+export class GeminiAdapter implements LlmAdapter {
+  private apiKey: string;
+  private model: string;
+
+  constructor(apiKey: string, model: string) {
+    this.apiKey = apiKey;
+    this.model = model;
+  }
+
+  async call(
+    words: string[],
+    systemPrompt: string,
+  ): Promise<{
+    content: string;
+    promptTokens: number;
+    completionTokens: number;
+    totalTokens: number;
+    promptTimeMs: number;
+    completionTimeMs: number;
+    promptTokensPerSecond: number;
+    completionTokensPerSecond: number;
+  }> {
+    const url = `https://generativelanguage.googleapis.com/v1beta/models/${this.model}:generateContent?key=${this.apiKey}`;
+
+    const payload = {
+      contents: [
+        {
+          role: "user",
+          parts: [
+            { text: systemPrompt + "\n\nWords: " + JSON.stringify(words) },
+          ],
+        },
+      ],
+      generationConfig: {
+        temperature: 0.1,
+        topP: 0.9,
+        maxOutputTokens: Math.ceil(words.length * 250 * 1.2),
+      },
+    };
+
+    const startTime = Date.now();
+
+    const response = await fetch(url, {
+      method: "POST",
+      headers: { "Content-Type": "application/json" },
+      body: JSON.stringify(payload),
+    });
+
+    const totalTimeMs = Date.now() - startTime;
+
+    if (!response.ok) {
+      throw new Error(`Gemini API responded with status: ${response.status}`);
+    }
+
+    const json = (await response.json()) as GeminiResponse;
+
+    const content = json.candidates[0]?.content?.parts[0]?.text;
+    if (!content) {
+      throw new Error("Gemini response content is empty");
+    }
+
+    const promptTokens = json.usageMetadata.promptTokenCount;
+    const completionTokens = json.usageMetadata.candidatesTokenCount;
+    const totalTokens = json.usageMetadata.totalTokenCount;
+
+    // Gemini doesn't provide timing breakdown, so we estimate
+    const promptTimeMs = totalTimeMs * 0.3; // rough estimate
+    const completionTimeMs = totalTimeMs * 0.7; // rough estimate
+
+    return {
+      content,
+      promptTokens,
+      completionTokens,
+      totalTokens,
+      promptTimeMs,
+      completionTimeMs,
+      promptTokensPerSecond: promptTokens / (promptTimeMs / 1000),
+      completionTokensPerSecond: completionTokens / (completionTimeMs / 1000),
+    };
+  }
+}

data-pipeline/utils/llm-adapters/openai-compatible.ts

@@ -0,0 +1,92 @@
+import type { LlmAdapter } from "./types.js";
+
+interface OpenAiResponse {
+  choices: Array<{ message: { content: string } }>;
+  usage: {
+    prompt_tokens: number;
+    completion_tokens: number;
+    total_tokens: number;
+  };
+  timings: {
+    prompt_ms: number;
+    predicted_ms: number;
+    prompt_per_second: number;
+    predicted_per_second: number;
+  };
+}
+
+export class OpenAiCompatibleAdapter implements LlmAdapter {
+  private url: string;
+  private apiKey: string | undefined;
+  private model: string | undefined;
+
+  constructor(url: string, apiKey?: string, model?: string) {
+    this.url = url;
+    this.apiKey = apiKey;
+    this.model = model;
+  }
+
+  async call(
+    words: string[],
+    systemPrompt: string,
+  ): Promise<{
+    content: string;
+    promptTokens: number;
+    completionTokens: number;
+    totalTokens: number;
+    promptTimeMs: number;
+    completionTimeMs: number;
+    promptTokensPerSecond: number;
+    completionTokensPerSecond: number;
+  }> {
+    const payload: Record<string, unknown> = {
+      messages: [
+        { role: "system", content: systemPrompt },
+        { role: "user", content: JSON.stringify(words) },
+      ],
+      temperature: 0.1,
+      top_p: 0.9,
+      max_tokens: Math.ceil(words.length * 250 * 1.2),
+    };
+
+    if (this.model) {
+      payload["model"] = this.model;
+    }
+
+    const headers: Record<string, string> = {
+      "Content-Type": "application/json",
+    };
+
+    if (this.apiKey) {
+      headers["Authorization"] = `Bearer ${this.apiKey}`;
+    }
+
+    const response = await fetch(this.url, {
+      method: "POST",
+      headers,
+      body: JSON.stringify(payload),
+    });
+
+    if (!response.ok) {
+      throw new Error(`LLM server responded with status: ${response.status}`);
+    }
+
+    const json = (await response.json()) as OpenAiResponse;
+
+    const content = json.choices[0]?.message?.content;
+    if (!content) {
+      throw new Error("LLM response content is empty");
+    }
+
+    return {
+      content,
+      promptTokens: json.usage.prompt_tokens,
+      completionTokens: json.usage.completion_tokens,
+      totalTokens: json.usage.total_tokens,
+      promptTimeMs: json.timings.prompt_ms,
+      completionTimeMs: json.timings.predicted_ms,
+      promptTokensPerSecond: json.timings.prompt_per_second,
+      completionTokensPerSecond: json.timings.predicted_per_second,
+    };
+  }
+}

data-pipeline/utils/llm-adapters/types.ts

@@ -0,0 +1,15 @@
+export interface LlmAdapter {
+  call(
+    words: string[],
+    systemPrompt: string,
+  ): Promise<{
+    content: string;
+    promptTokens: number;
+    completionTokens: number;
+    totalTokens: number;
+    promptTimeMs: number;
+    completionTimeMs: number;
+    promptTokensPerSecond: number;
+    completionTokensPerSecond: number;
+  }>;
+}

data-pipeline/utils/merge-enriched-data.ts

@@ -0,0 +1,69 @@
+export type Language = "en" | "de" | "it" | "es" | "fr";
+export type Pos = "noun" | "verb" | "adjective" | "adverb";
+export type Gender = "masculine" | "feminine" | "neuter" | null;
+export type Difficulty = "easy" | "medium" | "hard";
+
+export interface Translation {
+  word: string;
+  gender: Gender;
+}
+
+export interface EnrichedSense {
+  id: string;
+  word: string;
+  language: Language;
+  pos: Pos;
+  sense: string;
+  example: string;
+  difficulty_level: Difficulty;
+  translations: {
+    de: Translation[];
+    it: Translation[];
+    es: Translation[];
+    fr: Translation[];
+  };
+}
+
+const LANG_MAP: Record<string, Language> = {
+  english: "en",
+  italian: "it",
+  german: "de",
+  french: "fr",
+  spanish: "es",
+};
+
+const POS_MAP: Record<string, Pos> = {
+  nouns: "noun",
+  verbs: "verb",
+  adverbs: "adverb",
+  adjectives: "adjective",
+};
+
+/**
+ * Merges skeleton data with enriched LLM senses into the final pipeline output.
+ */
+export function mergeEnrichedData(
+  word: string,
+  rawLanguage: string,
+  rawPos: string,
+  senses: EnrichedSense[],
+): Record<string, unknown> {
+  const language = LANG_MAP[rawLanguage] || (rawLanguage as Language);
+  const pos = POS_MAP[rawPos] || (rawPos as Pos);
+
+  const fixedSenses = senses.map((sense, index) => ({
+    ...sense,
+    id: `${word}:${language}:${pos}:${index}`,
+    language,
+    pos,
+  }));
+
+  return {
+    word,
+    language,
+    pos,
+    senses: fixedSenses,
+    enrichedAt: new Date().toISOString(),
+    model: "qwen3.5-4b-q4_k_m",
+  };
+}

data-pipeline/utils/pipeline-timer.ts

@@ -0,0 +1,162 @@
+interface LlmMetrics {
+  promptTokens: number;
+  completionTokens: number;
+  totalTokens: number;
+  promptTimeMs: number;
+  completionTimeMs: number;
+  promptTokensPerSecond: number;
+  completionTokensPerSecond: number;
+}
+
+interface PipelineMetrics {
+  startTime: Date;
+  endTime?: Date;
+  wordsProcessed: number;
+  wordsSkipped: number;
+  wordsFailed: number;
+  llmCalls: number;
+  totalPromptTokens: number;
+  totalCompletionTokens: number;
+  totalTokens: number;
+  totalPromptTimeMs: number;
+  totalCompletionTimeMs: number;
+  currentWordStartTime?: Date;
+}
+
+/**
+ * Simple timer and metrics tracker for the pipeline.
+ * Tracks both pipeline throughput and LLM performance.
+ */
+export class PipelineTimer {
+  private metrics: PipelineMetrics;
+
+  constructor() {
+    this.metrics = {
+      startTime: new Date(),
+      wordsProcessed: 0,
+      wordsSkipped: 0,
+      wordsFailed: 0,
+      llmCalls: 0,
+      totalPromptTokens: 0,
+      totalCompletionTokens: 0,
+      totalTokens: 0,
+      totalPromptTimeMs: 0,
+      totalCompletionTimeMs: 0,
+    };
+  }
+
+  startWord(): void {
+    this.metrics.currentWordStartTime = new Date();
+  }
+
+  getWordDurationMs(): number {
+    if (!this.metrics.currentWordStartTime) return 0;
+    return new Date().getTime() - this.metrics.currentWordStartTime.getTime();
+  }
+
+  recordProcessed(llmMetrics?: LlmMetrics): void {
+    this.metrics.wordsProcessed++;
+    if (llmMetrics) {
+      this.metrics.llmCalls++;
+      this.metrics.totalPromptTokens += llmMetrics.promptTokens;
+      this.metrics.totalCompletionTokens += llmMetrics.completionTokens;
+      this.metrics.totalTokens += llmMetrics.totalTokens;
+      this.metrics.totalPromptTimeMs += llmMetrics.promptTimeMs;
+      this.metrics.totalCompletionTimeMs += llmMetrics.completionTimeMs;
+    }
+  }
+
+  recordSkipped(): void {
+    this.metrics.wordsSkipped++;
+  }
+
+  recordFailed(): void {
+    this.metrics.wordsFailed++;
+  }
+
+  stop(): void {
+    this.metrics.endTime = new Date();
+  }
+
+  getWordTiming(): string {
+    const durationMs = this.getWordDurationMs();
+    const durationSec = (durationMs / 1000).toFixed(1);
+    return `⏱️  Word took ${durationSec}s`;
+  }
+
+  getEta(totalWords: number): string {
+    const processed = this.metrics.wordsProcessed;
+    const remaining = totalWords - processed - this.metrics.wordsSkipped;
+
+    if (processed === 0 || remaining <= 0) return "ETA: calculating...";
+
+    const elapsedMs = new Date().getTime() - this.metrics.startTime.getTime();
+    const avgMsPerWord = elapsedMs / processed;
+    const etaMs = avgMsPerWord * remaining;
+
+    const etaMin = Math.round(etaMs / 60000);
+    const etaHour = (etaMs / 3600000).toFixed(1);
+
+    if (etaMin < 60) {
+      return `ETA: ${etaMin} min`;
+    }
+    return `ETA: ${etaHour} hours`;
+  }
+
+  getSummary(): string {
+    const end = this.metrics.endTime || new Date();
+    const durationMs = end.getTime() - this.metrics.startTime.getTime();
+    const durationSec = (durationMs / 1000).toFixed(1);
+
+    const total =
+      this.metrics.wordsProcessed +
+      this.metrics.wordsSkipped +
+      this.metrics.wordsFailed;
+    const throughput =
+      this.metrics.wordsProcessed > 0
+        ? (this.metrics.wordsProcessed / (durationMs / 1000)).toFixed(2)
+        : "0";
+
+    const avgPromptTokens =
+      this.metrics.llmCalls > 0
+        ? (this.metrics.totalPromptTokens / this.metrics.llmCalls).toFixed(0)
+        : "0";
+    const avgCompletionTokens =
+      this.metrics.llmCalls > 0
+        ? (this.metrics.totalCompletionTokens / this.metrics.llmCalls).toFixed(
+            0,
+          )
+        : "0";
+    const avgPromptSpeed =
+      this.metrics.totalPromptTimeMs > 0
+        ? (
+            this.metrics.totalPromptTokens /
+            (this.metrics.totalPromptTimeMs / 1000)
+          ).toFixed(1)
+        : "0";
+    const avgCompletionSpeed =
+      this.metrics.totalCompletionTimeMs > 0
+        ? (
+            this.metrics.totalCompletionTokens /
+            (this.metrics.totalCompletionTimeMs / 1000)
+          ).toFixed(1)
+        : "0";
+
+    return [
+      `⏱️  Pipeline Summary`,
+      `   Duration: ${durationSec}s`,
+      `   Processed: ${this.metrics.wordsProcessed}`,
+      `   Skipped: ${this.metrics.wordsSkipped}`,
+      `   Failed: ${this.metrics.wordsFailed}`,
+      `   Total: ${total}`,
+      `   Throughput: ${throughput} words/sec`,
+      ``,
+      `🤖 LLM Metrics`,
+      `   Calls: ${this.metrics.llmCalls}`,
+      `   Avg prompt tokens: ${avgPromptTokens}`,
+      `   Avg completion tokens: ${avgCompletionTokens}`,
+      `   Avg prompt speed: ${avgPromptSpeed} tok/s`,
+      `   Avg completion speed: ${avgCompletionSpeed} tok/s`,
+    ].join("\n");
+  }
+}

data-pipeline/utils/progress-tracker.ts

@@ -0,0 +1,27 @@
+/**
+ * Simple progress tracker for pipeline execution.
+ */
+export class ProgressTracker {
+  private current: number;
+  private failed: number;
+  private total: number;
+
+  constructor(total: number) {
+    this.current = 0;
+    this.failed = 0;
+    this.total = total;
+  }
+
+  next(): number {
+    this.current++;
+    return this.current;
+  }
+
+  recordFailed(): void {
+    this.failed++;
+  }
+
+  format(label: string): string {
+    return `[${this.current}/${this.total}] (${this.failed} failed) ${label}`;
+  }
+}

data-pipeline/utils/scanning-source-files.ts

@@ -0,0 +1,61 @@
+import fs from "fs";
+import path from "path";
+
+// Define a simple shape for what a discovered dataset looks like
+export interface Wordlist {
+  language: string;
+  pos: string;
+  sourcePath: string;
+  outputDir: string;
+}
+
+/**
+ * Scans the source-data directory to find all available word lists.
+ */
+export function scanSourceData(baseDir: string): Wordlist[] {
+  const sourceBaseDir = path.join(baseDir, "source-data");
+  const discoveredWordlists: Wordlist[] = [];
+
+  // Safety check: if there's no source-data folder, return an empty array
+  if (!fs.existsSync(sourceBaseDir)) {
+    return discoveredWordlists;
+  }
+
+  // 1. Read the language directories (e.g., ['english'])
+  const languages = fs.readdirSync(sourceBaseDir);
+
+  for (const lang of languages) {
+    const langFolderPath = path.join(sourceBaseDir, lang);
+
+    // Make sure it's a directory, not a stray file
+    if (!fs.statSync(langFolderPath).isDirectory()) continue;
+
+    // 2. Read the files inside the language folder (e.g., ['nouns'])
+    const posFiles = fs.readdirSync(langFolderPath);
+
+    for (const pos of posFiles) {
+      const fullSourcePath = path.join(langFolderPath, pos);
+
+      // Make sure it's a file (like your extensionless "nouns" file)
+      if (!fs.statSync(fullSourcePath).isFile()) continue;
+
+      // 3. Package everything into a flat item and add it to our array
+      discoveredWordlists.push({
+        language: lang,
+        pos: pos,
+        sourcePath: fullSourcePath,
+        outputDir: path.join(baseDir, "worddata", lang, pos),
+      });
+    }
+  }
+
+  // show summary
+  console.log(
+    `✅ Scan complete! Found ${discoveredWordlists.length} wordlist(s):`,
+  );
+  for (const list of discoveredWordlists) {
+    console.log(`   • ${list.language.toUpperCase()} (${list.pos})`);
+  }
+
+  return discoveredWordlists;
+}

data-pipeline/utils/verify-enriched-file.ts

@@ -0,0 +1,96 @@
+import fs from "fs";
+
+interface VerificationResult {
+  valid: boolean;
+  errors: string[];
+}
+
+/**
+ * Verifies that an enriched JSON file matches the expected schema.
+ * Returns detailed error messages for any violations.
+ */
+export function verifyEnrichedFile(filePath: string): VerificationResult {
+  const errors: string[] = [];
+
+  if (!fs.existsSync(filePath)) {
+    return { valid: false, errors: ["File does not exist"] };
+  }
+
+  let data: unknown;
+  try {
+    data = JSON.parse(fs.readFileSync(filePath, "utf-8"));
+  } catch (_error: unknown) {
+    return { valid: false, errors: ["Invalid JSON syntax"] };
+  }
+
+  if (typeof data !== "object" || data === null || Array.isArray(data)) {
+    return { valid: false, errors: ["Root must be an object"] };
+  }
+
+  const obj = data as Record<string, unknown>;
+
+  // Required top-level fields
+  const requiredFields = ["word", "language", "pos", "senses"];
+  for (const field of requiredFields) {
+    if (!(field in obj)) {
+      errors.push(`Missing required field: "${field}"`);
+    }
+  }
+
+  // Validate senses array
+  if (!Array.isArray(obj["senses"])) {
+    errors.push('"senses" must be an array');
+  } else if (obj["senses"].length === 0) {
+    errors.push('"senses" array cannot be empty');
+  } else {
+    for (let i = 0; i < obj["senses"].length; i++) {
+      const sense = obj["senses"][i] as Record<string, unknown>;
+      const sensePrefix = `senses[${i}]`;
+
+      if (!sense["sense"] || typeof sense["sense"] !== "string") {
+        errors.push(`${sensePrefix}: missing or invalid "sense"`);
+      }
+      if (!sense["example"] || typeof sense["example"] !== "string") {
+        errors.push(`${sensePrefix}: missing or invalid "example"`);
+      }
+      if (
+        !["easy", "medium", "hard"].includes(
+          sense["difficulty_level"] as string,
+        )
+      ) {
+        errors.push(`${sensePrefix}: invalid "difficulty_level"`);
+      }
+      if (!sense["translations"] || typeof sense["translations"] !== "object") {
+        errors.push(`${sensePrefix}: missing "translations"`);
+      } else {
+        const trans = sense["translations"] as Record<string, unknown>;
+        for (const lang of ["de", "it", "es", "fr"]) {
+          if (!Array.isArray(trans[lang])) {
+            errors.push(
+              `${sensePrefix}: missing or invalid "${lang}" translations`,
+            );
+          } else {
+            for (let j = 0; j < (trans[lang] as unknown[]).length; j++) {
+              const t = (trans[lang] as unknown[])[j] as Record<
+                string,
+                unknown
+              >;
+              if (!t["word"] || typeof t["word"] !== "string") {
+                errors.push(`${sensePrefix}.${lang}[${j}]: missing "word"`);
+              }
+              if (
+                !["masculine", "feminine", "neuter", null].includes(
+                  t["gender"] as string | null,
+                )
+              ) {
+                errors.push(`${sensePrefix}.${lang}[${j}]: invalid "gender"`);
+              }
+            }
+          }
+        }
+      }
+    }
+  }
+
+  return { valid: errors.length === 0, errors };
+}

data-pipeline/utils/write-json-file.ts

@@ -0,0 +1,11 @@
+import fs from "fs";
+
+/**
+ * Writes data as formatted JSON to a file path.
+ * Safely catches and re-throws file system errors.
+ */
+export function writeJsonFile(filePath: string, data: unknown): void {
+  const tempPath = `${filePath}.tmp`;
+  fs.writeFileSync(tempPath, JSON.stringify(data, null, 2), "utf-8");
+  fs.renameSync(tempPath, filePath);
+}