bugfixing #1

data-pipeline/config/batch.ts

@@ -1,3 +1 @@
 export const BATCH_CONFIG = { size: 4, maxRetries: 3 } as const;
-
-//1, 2, 5, 10, 20

data-pipeline/config/constants.ts

@@ -0,0 +1,14 @@
+export const LANG_MAP: Record<string, string> = {
+  english: "en",
+  italian: "it",
+  german: "de",
+  french: "fr",
+  spanish: "es",
+};
+
+export const POS_MAP: Record<string, string> = {
+  nouns: "noun",
+  verbs: "verb",
+  adverbs: "adverb",
+  adjectives: "adjective",
+};

data-pipeline/pipeline.ts

@@ -27,7 +27,7 @@ async function main() {
   console.log("\n step 2: creating necessary output folders...");
   ensureOutputFolders(wordlists);
 
-  // step 3: check to verify the AI engine is ready before touching anything
+  // step 3: check to verify the local AI engine is ready before touching anything
   console.log("\n step 3: verifying local AI engine status...");
   await checkLlmServer();
 
@@ -99,12 +99,7 @@ async function main() {
           );
 
           const targetFilePath = getWordFilePath(word, wordlist.outputDir);
-          const enrichedData = mergeEnrichedData(
-            word,
-            wordlist.language,
-            wordlist.pos,
-            senses,
-          );
+          const enrichedData = mergeEnrichedData(word, senses);
 
           writeJsonFile(targetFilePath, enrichedData);
 
@@ -118,17 +113,19 @@ async function main() {
           }
 
           timer.recordProcessed({
-            promptTokens: result.metrics.promptTokens / batch.length,
-            completionTokens: result.metrics.completionTokens / batch.length,
-            totalTokens: result.metrics.totalTokens / batch.length,
-            promptTimeMs: result.metrics.promptTimeMs / batch.length,
-            completionTimeMs: result.metrics.completionTimeMs / batch.length,
-            promptTokensPerSecond: result.metrics.promptTokensPerSecond,
-            completionTokensPerSecond: result.metrics.completionTokensPerSecond,
+            promptTokens: result.metrics.promptTokens,
+            completionTokens: result.metrics.completionTokens,
+            totalTokens: result.metrics.totalTokens,
+            promptTimeMs: result.metrics.promptTimeMs,
+            completionTimeMs: result.metrics.completionTimeMs,
           });
         }
 
         console.log(`  ${timer.getWordTiming()}`);
+        // Show ETA every 5 batches or on the last batch
+        if (batchNum % 5 === 0 || batchNum === totalBatches) {
+          console.log(`  📊 ${timer.getEta(unprocessedWords.length)}`);
+        }
       } catch (error: unknown) {
         const errorMessage =
           error instanceof Error ? error.message : String(error);

data-pipeline/utils/check-llm-server.ts

@@ -1,10 +1,18 @@
+import { LLM_CONFIG } from "../config/llm.js";
+
 /**
  * Pings the local llama.cpp server to ensure it's up, running, and has a model loaded.
  * If the server is offline or still loading, it terminates the pipeline gracefully.
+ * Skipped entirely when using a cloud provider.
  */
 export async function checkLlmServer(
   url = "http://127.0.0.1:8080/health",
 ): Promise<void> {
+  if (LLM_CONFIG.provider !== "local") {
+    console.log("🌐 Using cloud provider — skipping local health check.");
+    return;
+  }
+
   try {
     const response = await fetch(url);
 

data-pipeline/utils/create-base-json.ts

@@ -1,20 +1,6 @@
 import fs from "fs";
 import path from "path";
-
-const LANG_MAP: Record<string, string> = {
-  english: "en",
-  italian: "it",
-  german: "de",
-  french: "fr",
-  spanish: "es",
-};
-
-const POS_MAP: Record<string, string> = {
-  nouns: "noun",
-  verbs: "verb",
-  adverbs: "adverb",
-  adjectives: "adjective",
-};
+import { LANG_MAP, POS_MAP } from "../config/constants.js";
 
 /**
  * Creates the base JSON file with word, language, and pos.

data-pipeline/utils/enrich-word.ts

@@ -1,4 +1,3 @@
-// utils/enrich-word.ts
 import { ENRICHMENT_SYSTEM_PROMPT } from "../config/prompt.js";
 import { createAdapter } from "./llm-adapters/factory.js";
 import { BATCH_CONFIG } from "../config/batch.js";
@@ -26,8 +25,6 @@ interface LlmResponse {
   totalTokens: number;
   promptTimeMs: number;
   completionTimeMs: number;
-  promptTokensPerSecond: number;
-  completionTokensPerSecond: number;
 }
 
 export interface EnrichmentResult {
@@ -38,13 +35,11 @@ export interface EnrichmentResult {
     totalTokens: number;
     promptTimeMs: number;
     completionTimeMs: number;
-    promptTokensPerSecond: number;
-    completionTokensPerSecond: number;
   };
 }
 
 /**
- * Calls the local LLM with the enrichment prompt.
+ * Calls the LLM with the enrichment prompt.
  * Returns the response content and timing metrics.
  */
 async function callLlm(words: string[]): Promise<LlmResponse> {
@@ -157,8 +152,6 @@ export async function enrichWord(
       totalTokens: llmResponse.totalTokens,
       promptTimeMs: llmResponse.promptTimeMs,
       completionTimeMs: llmResponse.completionTimeMs,
-      promptTokensPerSecond: llmResponse.promptTokensPerSecond,
-      completionTokensPerSecond: llmResponse.completionTokensPerSecond,
     },
   };
 }
@@ -225,14 +218,6 @@ export async function enrichWordWithRetry(
       completionTimeMs:
         leftResult.metrics.completionTimeMs +
         rightResult.metrics.completionTimeMs,
-      promptTokensPerSecond:
-        (leftResult.metrics.promptTokensPerSecond +
-          rightResult.metrics.promptTokensPerSecond) /
-        2,
-      completionTokensPerSecond:
-        (leftResult.metrics.completionTokensPerSecond +
-          rightResult.metrics.completionTokensPerSecond) /
-        2,
     };
 
     return { results: merged, metrics: mergedMetrics };

data-pipeline/utils/llm-adapters/gemini.ts

@@ -28,8 +28,6 @@ export class GeminiAdapter implements LlmAdapter {
     totalTokens: number;
     promptTimeMs: number;
     completionTimeMs: number;
-    promptTokensPerSecond: number;
-    completionTokensPerSecond: number;
   }> {
     const url = `https://generativelanguage.googleapis.com/v1beta/models/${this.model}:generateContent?key=${this.apiKey}`;
 
@@ -72,21 +70,14 @@ export class GeminiAdapter implements LlmAdapter {
 
     const promptTokens = json.usageMetadata.promptTokenCount;
     const completionTokens = json.usageMetadata.candidatesTokenCount;
-    const totalTokens = json.usageMetadata.totalTokenCount;
-
-    // Gemini doesn't provide timing breakdown, so we estimate
-    const promptTimeMs = totalTimeMs * 0.3; // rough estimate
-    const completionTimeMs = totalTimeMs * 0.7; // rough estimate
 
     return {
       content,
       promptTokens,
       completionTokens,
-      totalTokens,
-      promptTimeMs,
-      completionTimeMs,
-      promptTokensPerSecond: promptTokens / (promptTimeMs / 1000),
-      completionTokensPerSecond: completionTokens / (completionTimeMs / 1000),
+      totalTokens: json.usageMetadata.totalTokenCount,
+      promptTimeMs: totalTimeMs * 0.3,
+      completionTimeMs: totalTimeMs * 0.7,
     };
   }
 }

data-pipeline/utils/llm-adapters/openai-compatible.ts

@@ -7,12 +7,7 @@ interface OpenAiResponse {
     completion_tokens: number;
     total_tokens: number;
   };
-  timings: {
-    prompt_ms: number;
-    predicted_ms: number;
-    prompt_per_second: number;
-    predicted_per_second: number;
-  };
+  timings?: { prompt_ms: number; predicted_ms: number };
 }
 
 export class OpenAiCompatibleAdapter implements LlmAdapter {
@@ -36,8 +31,6 @@ export class OpenAiCompatibleAdapter implements LlmAdapter {
     totalTokens: number;
     promptTimeMs: number;
     completionTimeMs: number;
-    promptTokensPerSecond: number;
-    completionTokensPerSecond: number;
   }> {
     const payload: Record<string, unknown> = {
       messages: [
@@ -61,12 +54,16 @@ export class OpenAiCompatibleAdapter implements LlmAdapter {
       headers["Authorization"] = `Bearer ${this.apiKey}`;
     }
 
+    const startTime = Date.now();
+
     const response = await fetch(this.url, {
       method: "POST",
       headers,
       body: JSON.stringify(payload),
     });
 
+    const totalTimeMs = Date.now() - startTime;
+
     if (!response.ok) {
       throw new Error(`LLM server responded with status: ${response.status}`);
     }
@@ -78,15 +75,19 @@ export class OpenAiCompatibleAdapter implements LlmAdapter {
       throw new Error("LLM response content is empty");
     }
 
+    const promptTokens = json.usage.prompt_tokens;
+    const completionTokens = json.usage.completion_tokens;
+
+    const promptTimeMs = json.timings?.prompt_ms ?? totalTimeMs * 0.3;
+    const completionTimeMs = json.timings?.predicted_ms ?? totalTimeMs * 0.7;
+
     return {
       content,
-      promptTokens: json.usage.prompt_tokens,
-      completionTokens: json.usage.completion_tokens,
+      promptTokens,
+      completionTokens,
       totalTokens: json.usage.total_tokens,
-      promptTimeMs: json.timings.prompt_ms,
-      completionTimeMs: json.timings.predicted_ms,
-      promptTokensPerSecond: json.timings.prompt_per_second,
-      completionTokensPerSecond: json.timings.predicted_per_second,
+      promptTimeMs,
+      completionTimeMs,
     };
   }
 }

data-pipeline/utils/llm-adapters/types.ts

@@ -9,7 +9,5 @@ export interface LlmAdapter {
     totalTokens: number;
     promptTimeMs: number;
     completionTimeMs: number;
-    promptTokensPerSecond: number;
-    completionTokensPerSecond: number;
   }>;
 }

data-pipeline/utils/merge-enriched-data.ts

@@ -1,3 +1,5 @@
+import { LLM_CONFIG } from "../config/llm.js";
+
 export type Language = "en" | "de" | "it" | "es" | "fr";
 export type Pos = "noun" | "verb" | "adjective" | "adverb";
 export type Gender = "masculine" | "feminine" | "neuter" | null;
@@ -24,46 +26,19 @@ export interface EnrichedSense {
   };
 }
 
-const LANG_MAP: Record<string, Language> = {
-  english: "en",
-  italian: "it",
-  german: "de",
-  french: "fr",
-  spanish: "es",
-};
-
-const POS_MAP: Record<string, Pos> = {
-  nouns: "noun",
-  verbs: "verb",
-  adverbs: "adverb",
-  adjectives: "adjective",
-};
-
 /**
  * Merges skeleton data with enriched LLM senses into the final pipeline output.
  */
 export function mergeEnrichedData(
   word: string,
-  rawLanguage: string,
-  rawPos: string,
   senses: EnrichedSense[],
 ): Record<string, unknown> {
-  const language = LANG_MAP[rawLanguage] || (rawLanguage as Language);
-  const pos = POS_MAP[rawPos] || (rawPos as Pos);
-
-  const fixedSenses = senses.map((sense, index) => ({
-    ...sense,
-    id: `${word}:${language}:${pos}:${index}`,
-    language,
-    pos,
-  }));
-
   return {
     word,
-    language,
-    pos,
-    senses: fixedSenses,
+    language: senses[0]?.language ?? "en",
+    pos: senses[0]?.pos ?? "noun",
+    senses,
     enrichedAt: new Date().toISOString(),
-    model: "qwen3.5-4b-q4_k_m",
+    model: LLM_CONFIG.model ?? "unknown",
   };
 }

data-pipeline/utils/pipeline-timer.ts

@@ -4,8 +4,6 @@ interface LlmMetrics {
   totalTokens: number;
   promptTimeMs: number;
   completionTimeMs: number;
-  promptTokensPerSecond: number;
-  completionTokensPerSecond: number;
 }
 
 interface PipelineMetrics {