interface LlmMetrics { promptTokens: number; completionTokens: number; totalTokens: number; promptTimeMs: number | null; completionTimeMs: number | null; totalTimeMs: number; } interface PipelineMetrics { startTime: Date; endTime?: Date; wordsProcessed: number; wordsSkipped: number; wordsFailed: number; llmCalls: number; totalPromptTokens: number; totalCompletionTokens: number; totalTokens: number; totalPromptTimeMs: number; totalCompletionTimeMs: number; totalTimeMs: number; currentWordStartTime?: Date; } /** * Simple timer and metrics tracker for the pipeline. * Tracks both pipeline throughput and LLM performance. */ export class PipelineTimer { private metrics: PipelineMetrics; constructor() { this.metrics = { startTime: new Date(), wordsProcessed: 0, wordsSkipped: 0, wordsFailed: 0, llmCalls: 0, totalPromptTokens: 0, totalCompletionTokens: 0, totalTokens: 0, totalPromptTimeMs: 0, totalCompletionTimeMs: 0, totalTimeMs: 0, }; } startWord(): void { this.metrics.currentWordStartTime = new Date(); } getWordDurationMs(): number { if (!this.metrics.currentWordStartTime) return 0; return new Date().getTime() - this.metrics.currentWordStartTime.getTime(); } recordProcessed(llmMetrics?: LlmMetrics): void { this.metrics.wordsProcessed++; if (llmMetrics) { this.metrics.llmCalls++; this.metrics.totalPromptTokens += llmMetrics.promptTokens; this.metrics.totalCompletionTokens += llmMetrics.completionTokens; this.metrics.totalTokens += llmMetrics.totalTokens; if (llmMetrics.promptTimeMs !== null) { this.metrics.totalPromptTimeMs += llmMetrics.promptTimeMs; } if (llmMetrics.completionTimeMs !== null) { this.metrics.totalCompletionTimeMs += llmMetrics.completionTimeMs; } this.metrics.totalTimeMs += llmMetrics.totalTimeMs; } } recordSkipped(): void { this.metrics.wordsSkipped++; } recordFailed(): void { this.metrics.wordsFailed++; } stop(): void { this.metrics.endTime = new Date(); } getWordTiming(): string { const durationMs = this.getWordDurationMs(); const durationSec = (durationMs / 1000).toFixed(1); return `⏱️ Word took ${durationSec}s`; } getEta(totalWords: number): string { const processed = this.metrics.wordsProcessed; const remaining = totalWords - processed - this.metrics.wordsSkipped; if (processed === 0 || remaining <= 0) return "ETA: calculating..."; const elapsedMs = new Date().getTime() - this.metrics.startTime.getTime(); const avgMsPerWord = elapsedMs / processed; const etaMs = avgMsPerWord * remaining; const etaMin = Math.round(etaMs / 60000); const etaHour = (etaMs / 3600000).toFixed(1); if (etaMin < 60) { return `ETA: ${etaMin} min`; } return `ETA: ${etaHour} hours`; } getSummary(): string { const end = this.metrics.endTime || new Date(); const durationMs = end.getTime() - this.metrics.startTime.getTime(); const durationSec = (durationMs / 1000).toFixed(1); const total = this.metrics.wordsProcessed + this.metrics.wordsSkipped + this.metrics.wordsFailed; const throughput = this.metrics.wordsProcessed > 0 ? (this.metrics.wordsProcessed / (durationMs / 1000)).toFixed(2) : "0"; const avgPromptTokens = this.metrics.llmCalls > 0 ? (this.metrics.totalPromptTokens / this.metrics.llmCalls).toFixed(0) : "0"; const avgCompletionTokens = this.metrics.llmCalls > 0 ? (this.metrics.totalCompletionTokens / this.metrics.llmCalls).toFixed( 0, ) : "0"; const avgTotalTimeMs = this.metrics.llmCalls > 0 ? (this.metrics.totalTimeMs / this.metrics.llmCalls).toFixed(0) : "0"; const unifiedThroughput = this.metrics.totalTimeMs > 0 ? ( this.metrics.totalTokens / (this.metrics.totalTimeMs / 1000) ).toFixed(1) : "N/A"; const hasDetailedTimings = this.metrics.totalPromptTimeMs > 0 || this.metrics.totalCompletionTimeMs > 0; const avgPromptSpeed = this.metrics.totalPromptTimeMs > 0 ? ( this.metrics.totalPromptTokens / (this.metrics.totalPromptTimeMs / 1000) ).toFixed(1) : "N/A"; const avgCompletionSpeed = this.metrics.totalCompletionTimeMs > 0 ? ( this.metrics.totalCompletionTokens / (this.metrics.totalCompletionTimeMs / 1000) ).toFixed(1) : "N/A"; const lines = [ `⏱️ Pipeline Summary`, ` Duration: ${durationSec}s`, ` Processed: ${this.metrics.wordsProcessed}`, ` Skipped: ${this.metrics.wordsSkipped}`, ` Failed: ${this.metrics.wordsFailed}`, ` Total: ${total}`, ` Throughput: ${throughput} words/sec`, ``, `🤖 LLM Metrics`, ` Calls: ${this.metrics.llmCalls}`, ` Avg prompt tokens: ${avgPromptTokens}`, ` Avg completion tokens: ${avgCompletionTokens}`, ` Avg total tokens: ${avgPromptTokens + avgCompletionTokens}`, ` Avg total request time: ${avgTotalTimeMs}ms`, ` Avg throughput: ${unifiedThroughput} tok/s`, ]; if (hasDetailedTimings) { lines.push( ``, ` [Local breakdown]`, ` Avg prompt speed: ${avgPromptSpeed} tok/s`, ` Avg completion speed: ${avgCompletionSpeed} tok/s`, ); } return lines.join("\n"); } }