diff --git a/data-pipeline/pipeline.ts b/data-pipeline/pipeline.ts index f660478..7f28677 100644 --- a/data-pipeline/pipeline.ts +++ b/data-pipeline/pipeline.ts @@ -118,6 +118,7 @@ async function main() { totalTokens: result.metrics.totalTokens, promptTimeMs: result.metrics.promptTimeMs, completionTimeMs: result.metrics.completionTimeMs, + totalTimeMs: result.metrics.totalTimeMs, }); } diff --git a/data-pipeline/utils/enrich-word.ts b/data-pipeline/utils/enrich-word.ts index 5a1bde5..f5536fb 100644 --- a/data-pipeline/utils/enrich-word.ts +++ b/data-pipeline/utils/enrich-word.ts @@ -9,8 +9,9 @@ interface LlmResponse { promptTokens: number; completionTokens: number; totalTokens: number; - promptTimeMs: number; - completionTimeMs: number; + promptTimeMs: number | null; + completionTimeMs: number | null; + totalTimeMs: number; } export interface EnrichmentResult { @@ -19,8 +20,9 @@ export interface EnrichmentResult { promptTokens: number; completionTokens: number; totalTokens: number; - promptTimeMs: number; - completionTimeMs: number; + promptTimeMs: number | null; + completionTimeMs: number | null; + totalTimeMs: number; }; } @@ -54,6 +56,60 @@ function sanitizeLlmOutput(raw: string): string { return cleaned.slice(start, end + 1); } +function validateSense(item: unknown, word: string, index: number): void { + if (typeof item !== "object" || item === null || Array.isArray(item)) { + throw new Error(`Sense ${index} for "${word}" is not an object`); + } + + const sense = item as Record; + + if (typeof sense["sense"] !== "string" || !sense["sense"]) { + throw new Error(`Sense ${index} for "${word}": missing or invalid "sense"`); + } + if (typeof sense["example"] !== "string" || !sense["example"]) { + throw new Error( + `Sense ${index} for "${word}": missing or invalid "example"`, + ); + } + if ( + !["easy", "medium", "hard"].includes(sense["difficulty_level"] as string) + ) { + throw new Error(`Sense ${index} for "${word}": invalid "difficulty_level"`); + } + if ( + typeof sense["translations"] !== "object" || + sense["translations"] === null + ) { + throw new Error(`Sense ${index} for "${word}": missing "translations"`); + } + + const trans = sense["translations"] as Record; + for (const lang of ["de", "it", "es", "fr"]) { + if (!Array.isArray(trans[lang])) { + throw new Error( + `Sense ${index} for "${word}": missing or invalid "${lang}" translations`, + ); + } + for (let j = 0; j < (trans[lang] as unknown[]).length; j++) { + const t = (trans[lang] as unknown[])[j] as Record; + if (typeof t["word"] !== "string" || !t["word"]) { + throw new Error( + `Sense ${index} for "${word}": ${lang}[${j}] missing "word"`, + ); + } + if ( + !["masculine", "feminine", "neuter", null].includes( + t["gender"] as string | null, + ) + ) { + throw new Error( + `Sense ${index} for "${word}": ${lang}[${j}] invalid "gender"`, + ); + } + } + } +} + /** * Parses the LLM response string into a JavaScript object. * Throws if the response is not valid JSON or not an object with expected keys. @@ -86,6 +142,12 @@ export function parseLlmResponse( if (!Array.isArray(obj[word]) || (obj[word] as unknown[]).length === 0) { throw new Error(`LLM output for "${word}" is not a non-empty array`); } + + // Validate each sense in the array + const senses = obj[word] as unknown[]; + for (let i = 0; i < senses.length; i++) { + validateSense(senses[i], word, i); + } } return obj; @@ -145,6 +207,7 @@ export async function enrichWord( totalTokens: llmResponse.totalTokens, promptTimeMs: llmResponse.promptTimeMs, completionTimeMs: llmResponse.completionTimeMs, + totalTimeMs: llmResponse.totalTimeMs, }, }; } @@ -207,10 +270,13 @@ export async function enrichWordWithRetry( totalTokens: leftResult.metrics.totalTokens + rightResult.metrics.totalTokens, promptTimeMs: - leftResult.metrics.promptTimeMs + rightResult.metrics.promptTimeMs, + (leftResult.metrics.promptTimeMs ?? 0) + + (rightResult.metrics.promptTimeMs ?? 0), completionTimeMs: - leftResult.metrics.completionTimeMs + - rightResult.metrics.completionTimeMs, + (leftResult.metrics.completionTimeMs ?? 0) + + (rightResult.metrics.completionTimeMs ?? 0), + totalTimeMs: + leftResult.metrics.totalTimeMs + rightResult.metrics.totalTimeMs, }; return { results: merged, metrics: mergedMetrics }; diff --git a/data-pipeline/utils/llm-adapters/gemini.ts b/data-pipeline/utils/llm-adapters/gemini.ts index 3a6060d..39008ca 100644 --- a/data-pipeline/utils/llm-adapters/gemini.ts +++ b/data-pipeline/utils/llm-adapters/gemini.ts @@ -26,8 +26,9 @@ export class GeminiAdapter implements LlmAdapter { promptTokens: number; completionTokens: number; totalTokens: number; - promptTimeMs: number; - completionTimeMs: number; + promptTimeMs: number | null; + completionTimeMs: number | null; + totalTimeMs: number; }> { const url = `https://generativelanguage.googleapis.com/v1beta/models/${this.model}:generateContent?key=${this.apiKey}`; @@ -72,8 +73,9 @@ export class GeminiAdapter implements LlmAdapter { promptTokens, completionTokens, totalTokens: json.usageMetadata.totalTokenCount, - promptTimeMs: totalTimeMs * 0.3, - completionTimeMs: totalTimeMs * 0.7, + promptTimeMs: null, + completionTimeMs: null, + totalTimeMs, }; } } diff --git a/data-pipeline/utils/llm-adapters/openai-compatible.ts b/data-pipeline/utils/llm-adapters/openai-compatible.ts index 14f0859..aa95aef 100644 --- a/data-pipeline/utils/llm-adapters/openai-compatible.ts +++ b/data-pipeline/utils/llm-adapters/openai-compatible.ts @@ -29,8 +29,9 @@ export class OpenAiCompatibleAdapter implements LlmAdapter { promptTokens: number; completionTokens: number; totalTokens: number; - promptTimeMs: number; - completionTimeMs: number; + promptTimeMs: number | null; + completionTimeMs: number | null; + totalTimeMs: number; }> { const payload: Record = { messages: [ @@ -78,16 +79,14 @@ export class OpenAiCompatibleAdapter implements LlmAdapter { const promptTokens = json.usage.prompt_tokens; const completionTokens = json.usage.completion_tokens; - const promptTimeMs = json.timings?.prompt_ms ?? totalTimeMs * 0.3; - const completionTimeMs = json.timings?.predicted_ms ?? totalTimeMs * 0.7; - return { content, promptTokens, completionTokens, totalTokens: json.usage.total_tokens, - promptTimeMs, - completionTimeMs, + promptTimeMs: json.timings?.prompt_ms ?? null, + completionTimeMs: json.timings?.predicted_ms ?? null, + totalTimeMs, }; } } diff --git a/data-pipeline/utils/llm-adapters/types.ts b/data-pipeline/utils/llm-adapters/types.ts index ae38a8b..c73c0b4 100644 --- a/data-pipeline/utils/llm-adapters/types.ts +++ b/data-pipeline/utils/llm-adapters/types.ts @@ -7,7 +7,8 @@ export interface LlmAdapter { promptTokens: number; completionTokens: number; totalTokens: number; - promptTimeMs: number; - completionTimeMs: number; + promptTimeMs: number | null; + completionTimeMs: number | null; + totalTimeMs: number; }>; } diff --git a/data-pipeline/utils/pipeline-timer.ts b/data-pipeline/utils/pipeline-timer.ts index 1eecfb0..8ba2d0d 100644 --- a/data-pipeline/utils/pipeline-timer.ts +++ b/data-pipeline/utils/pipeline-timer.ts @@ -2,8 +2,9 @@ interface LlmMetrics { promptTokens: number; completionTokens: number; totalTokens: number; - promptTimeMs: number; - completionTimeMs: number; + promptTimeMs: number | null; + completionTimeMs: number | null; + totalTimeMs: number; } interface PipelineMetrics { @@ -18,6 +19,7 @@ interface PipelineMetrics { totalTokens: number; totalPromptTimeMs: number; totalCompletionTimeMs: number; + totalTimeMs: number; currentWordStartTime?: Date; } @@ -40,6 +42,7 @@ export class PipelineTimer { totalTokens: 0, totalPromptTimeMs: 0, totalCompletionTimeMs: 0, + totalTimeMs: 0, }; } @@ -59,8 +62,13 @@ export class PipelineTimer { this.metrics.totalPromptTokens += llmMetrics.promptTokens; this.metrics.totalCompletionTokens += llmMetrics.completionTokens; this.metrics.totalTokens += llmMetrics.totalTokens; - this.metrics.totalPromptTimeMs += llmMetrics.promptTimeMs; - this.metrics.totalCompletionTimeMs += llmMetrics.completionTimeMs; + if (llmMetrics.promptTimeMs !== null) { + this.metrics.totalPromptTimeMs += llmMetrics.promptTimeMs; + } + if (llmMetrics.completionTimeMs !== null) { + this.metrics.totalCompletionTimeMs += llmMetrics.completionTimeMs; + } + this.metrics.totalTimeMs += llmMetrics.totalTimeMs; } } @@ -125,22 +133,40 @@ export class PipelineTimer { 0, ) : "0"; + const avgTotalTimeMs = + this.metrics.llmCalls > 0 + ? (this.metrics.totalTimeMs / this.metrics.llmCalls).toFixed(0) + : "0"; + + const unifiedThroughput = + this.metrics.totalTimeMs > 0 + ? ( + this.metrics.totalTokens / + (this.metrics.totalTimeMs / 1000) + ).toFixed(1) + : "N/A"; + + const hasDetailedTimings = + this.metrics.totalPromptTimeMs > 0 || + this.metrics.totalCompletionTimeMs > 0; + const avgPromptSpeed = this.metrics.totalPromptTimeMs > 0 ? ( this.metrics.totalPromptTokens / (this.metrics.totalPromptTimeMs / 1000) ).toFixed(1) - : "0"; + : "N/A"; + const avgCompletionSpeed = this.metrics.totalCompletionTimeMs > 0 ? ( this.metrics.totalCompletionTokens / (this.metrics.totalCompletionTimeMs / 1000) ).toFixed(1) - : "0"; + : "N/A"; - return [ + const lines = [ `⏱️ Pipeline Summary`, ` Duration: ${durationSec}s`, ` Processed: ${this.metrics.wordsProcessed}`, @@ -153,8 +179,20 @@ export class PipelineTimer { ` Calls: ${this.metrics.llmCalls}`, ` Avg prompt tokens: ${avgPromptTokens}`, ` Avg completion tokens: ${avgCompletionTokens}`, - ` Avg prompt speed: ${avgPromptSpeed} tok/s`, - ` Avg completion speed: ${avgCompletionSpeed} tok/s`, - ].join("\n"); + ` Avg total tokens: ${avgPromptTokens + avgCompletionTokens}`, + ` Avg total request time: ${avgTotalTimeMs}ms`, + ` Avg throughput: ${unifiedThroughput} tok/s`, + ]; + + if (hasDetailedTimings) { + lines.push( + ``, + ` [Local breakdown]`, + ` Avg prompt speed: ${avgPromptSpeed} tok/s`, + ` Avg completion speed: ${avgCompletionSpeed} tok/s`, + ); + } + + return lines.join("\n"); } }