bugfixing #1

data-pipeline/utils/check-llm-server.ts

@@ -1,10 +1,18 @@
+import { LLM_CONFIG } from "../config/llm.js";
+
 /**
  * Pings the local llama.cpp server to ensure it's up, running, and has a model loaded.
  * If the server is offline or still loading, it terminates the pipeline gracefully.
+ * Skipped entirely when using a cloud provider.
  */
 export async function checkLlmServer(
   url = "http://127.0.0.1:8080/health",
 ): Promise<void> {
+  if (LLM_CONFIG.provider !== "local") {
+    console.log("🌐 Using cloud provider — skipping local health check.");
+    return;
+  }
+
   try {
     const response = await fetch(url);
 

data-pipeline/utils/create-base-json.ts

@@ -1,20 +1,6 @@
 import fs from "fs";
 import path from "path";
-
-const LANG_MAP: Record<string, string> = {
-  english: "en",
-  italian: "it",
-  german: "de",
-  french: "fr",
-  spanish: "es",
-};
-
-const POS_MAP: Record<string, string> = {
-  nouns: "noun",
-  verbs: "verb",
-  adverbs: "adverb",
-  adjectives: "adjective",
-};
+import { LANG_MAP, POS_MAP } from "../config/constants.js";
 
 /**
  * Creates the base JSON file with word, language, and pos.

data-pipeline/utils/enrich-word.ts

@@ -1,4 +1,3 @@
-// utils/enrich-word.ts
 import { ENRICHMENT_SYSTEM_PROMPT } from "../config/prompt.js";
 import { createAdapter } from "./llm-adapters/factory.js";
 import { BATCH_CONFIG } from "../config/batch.js";
@@ -26,8 +25,6 @@ interface LlmResponse {
   totalTokens: number;
   promptTimeMs: number;
   completionTimeMs: number;
-  promptTokensPerSecond: number;
-  completionTokensPerSecond: number;
 }
 
 export interface EnrichmentResult {
@@ -38,13 +35,11 @@ export interface EnrichmentResult {
     totalTokens: number;
     promptTimeMs: number;
     completionTimeMs: number;
-    promptTokensPerSecond: number;
-    completionTokensPerSecond: number;
   };
 }
 
 /**
- * Calls the local LLM with the enrichment prompt.
+ * Calls the LLM with the enrichment prompt.
  * Returns the response content and timing metrics.
  */
 async function callLlm(words: string[]): Promise<LlmResponse> {
@@ -157,8 +152,6 @@ export async function enrichWord(
       totalTokens: llmResponse.totalTokens,
       promptTimeMs: llmResponse.promptTimeMs,
       completionTimeMs: llmResponse.completionTimeMs,
-      promptTokensPerSecond: llmResponse.promptTokensPerSecond,
-      completionTokensPerSecond: llmResponse.completionTokensPerSecond,
     },
   };
 }
@@ -225,14 +218,6 @@ export async function enrichWordWithRetry(
       completionTimeMs:
         leftResult.metrics.completionTimeMs +
         rightResult.metrics.completionTimeMs,
-      promptTokensPerSecond:
-        (leftResult.metrics.promptTokensPerSecond +
-          rightResult.metrics.promptTokensPerSecond) /
-        2,
-      completionTokensPerSecond:
-        (leftResult.metrics.completionTokensPerSecond +
-          rightResult.metrics.completionTokensPerSecond) /
-        2,
     };
 
     return { results: merged, metrics: mergedMetrics };

data-pipeline/utils/llm-adapters/gemini.ts

@@ -28,8 +28,6 @@ export class GeminiAdapter implements LlmAdapter {
     totalTokens: number;
     promptTimeMs: number;
     completionTimeMs: number;
-    promptTokensPerSecond: number;
-    completionTokensPerSecond: number;
   }> {
     const url = `https://generativelanguage.googleapis.com/v1beta/models/${this.model}:generateContent?key=${this.apiKey}`;
 
@@ -72,21 +70,14 @@ export class GeminiAdapter implements LlmAdapter {
 
     const promptTokens = json.usageMetadata.promptTokenCount;
     const completionTokens = json.usageMetadata.candidatesTokenCount;
-    const totalTokens = json.usageMetadata.totalTokenCount;
-
-    // Gemini doesn't provide timing breakdown, so we estimate
-    const promptTimeMs = totalTimeMs * 0.3; // rough estimate
-    const completionTimeMs = totalTimeMs * 0.7; // rough estimate
 
     return {
       content,
       promptTokens,
       completionTokens,
-      totalTokens,
-      promptTimeMs,
-      completionTimeMs,
-      promptTokensPerSecond: promptTokens / (promptTimeMs / 1000),
-      completionTokensPerSecond: completionTokens / (completionTimeMs / 1000),
+      totalTokens: json.usageMetadata.totalTokenCount,
+      promptTimeMs: totalTimeMs * 0.3,
+      completionTimeMs: totalTimeMs * 0.7,
     };
   }
 }

data-pipeline/utils/llm-adapters/openai-compatible.ts

@@ -7,12 +7,7 @@ interface OpenAiResponse {
     completion_tokens: number;
     total_tokens: number;
   };
-  timings: {
-    prompt_ms: number;
-    predicted_ms: number;
-    prompt_per_second: number;
-    predicted_per_second: number;
-  };
+  timings?: { prompt_ms: number; predicted_ms: number };
 }
 
 export class OpenAiCompatibleAdapter implements LlmAdapter {
@@ -36,8 +31,6 @@ export class OpenAiCompatibleAdapter implements LlmAdapter {
     totalTokens: number;
     promptTimeMs: number;
     completionTimeMs: number;
-    promptTokensPerSecond: number;
-    completionTokensPerSecond: number;
   }> {
     const payload: Record<string, unknown> = {
       messages: [
@@ -61,12 +54,16 @@ export class OpenAiCompatibleAdapter implements LlmAdapter {
       headers["Authorization"] = `Bearer ${this.apiKey}`;
     }
 
+    const startTime = Date.now();
+
     const response = await fetch(this.url, {
       method: "POST",
       headers,
       body: JSON.stringify(payload),
     });
 
+    const totalTimeMs = Date.now() - startTime;
+
     if (!response.ok) {
       throw new Error(`LLM server responded with status: ${response.status}`);
     }
@@ -78,15 +75,19 @@ export class OpenAiCompatibleAdapter implements LlmAdapter {
       throw new Error("LLM response content is empty");
     }
 
+    const promptTokens = json.usage.prompt_tokens;
+    const completionTokens = json.usage.completion_tokens;
+
+    const promptTimeMs = json.timings?.prompt_ms ?? totalTimeMs * 0.3;
+    const completionTimeMs = json.timings?.predicted_ms ?? totalTimeMs * 0.7;
+
     return {
       content,
-      promptTokens: json.usage.prompt_tokens,
-      completionTokens: json.usage.completion_tokens,
+      promptTokens,
+      completionTokens,
       totalTokens: json.usage.total_tokens,
-      promptTimeMs: json.timings.prompt_ms,
-      completionTimeMs: json.timings.predicted_ms,
-      promptTokensPerSecond: json.timings.prompt_per_second,
-      completionTokensPerSecond: json.timings.predicted_per_second,
+      promptTimeMs,
+      completionTimeMs,
     };
   }
 }

data-pipeline/utils/llm-adapters/types.ts

@@ -9,7 +9,5 @@ export interface LlmAdapter {
     totalTokens: number;
     promptTimeMs: number;
     completionTimeMs: number;
-    promptTokensPerSecond: number;
-    completionTokensPerSecond: number;
   }>;
 }

data-pipeline/utils/merge-enriched-data.ts

@@ -1,3 +1,5 @@
+import { LLM_CONFIG } from "../config/llm.js";
+
 export type Language = "en" | "de" | "it" | "es" | "fr";
 export type Pos = "noun" | "verb" | "adjective" | "adverb";
 export type Gender = "masculine" | "feminine" | "neuter" | null;
@@ -24,46 +26,19 @@ export interface EnrichedSense {
   };
 }
 
-const LANG_MAP: Record<string, Language> = {
-  english: "en",
-  italian: "it",
-  german: "de",
-  french: "fr",
-  spanish: "es",
-};
-
-const POS_MAP: Record<string, Pos> = {
-  nouns: "noun",
-  verbs: "verb",
-  adverbs: "adverb",
-  adjectives: "adjective",
-};
-
 /**
  * Merges skeleton data with enriched LLM senses into the final pipeline output.
  */
 export function mergeEnrichedData(
   word: string,
-  rawLanguage: string,
-  rawPos: string,
   senses: EnrichedSense[],
 ): Record<string, unknown> {
-  const language = LANG_MAP[rawLanguage] || (rawLanguage as Language);
-  const pos = POS_MAP[rawPos] || (rawPos as Pos);
-
-  const fixedSenses = senses.map((sense, index) => ({
-    ...sense,
-    id: `${word}:${language}:${pos}:${index}`,
-    language,
-    pos,
-  }));
-
   return {
     word,
-    language,
-    pos,
-    senses: fixedSenses,
+    language: senses[0]?.language ?? "en",
+    pos: senses[0]?.pos ?? "noun",
+    senses,
     enrichedAt: new Date().toISOString(),
-    model: "qwen3.5-4b-q4_k_m",
+    model: LLM_CONFIG.model ?? "unknown",
   };
 }

data-pipeline/utils/pipeline-timer.ts

@@ -4,8 +4,6 @@ interface LlmMetrics {
   totalTokens: number;
   promptTimeMs: number;
   completionTimeMs: number;
-  promptTokensPerSecond: number;
-  completionTokensPerSecond: number;
 }
 
 interface PipelineMetrics {