wip

data-pipeline/utils/pipeline-timer.ts

@@ -0,0 +1,162 @@
+interface LlmMetrics {
+  promptTokens: number;
+  completionTokens: number;
+  totalTokens: number;
+  promptTimeMs: number;
+  completionTimeMs: number;
+  promptTokensPerSecond: number;
+  completionTokensPerSecond: number;
+}
+
+interface PipelineMetrics {
+  startTime: Date;
+  endTime?: Date;
+  wordsProcessed: number;
+  wordsSkipped: number;
+  wordsFailed: number;
+  llmCalls: number;
+  totalPromptTokens: number;
+  totalCompletionTokens: number;
+  totalTokens: number;
+  totalPromptTimeMs: number;
+  totalCompletionTimeMs: number;
+  currentWordStartTime?: Date;
+}
+
+/**
+ * Simple timer and metrics tracker for the pipeline.
+ * Tracks both pipeline throughput and LLM performance.
+ */
+export class PipelineTimer {
+  private metrics: PipelineMetrics;
+
+  constructor() {
+    this.metrics = {
+      startTime: new Date(),
+      wordsProcessed: 0,
+      wordsSkipped: 0,
+      wordsFailed: 0,
+      llmCalls: 0,
+      totalPromptTokens: 0,
+      totalCompletionTokens: 0,
+      totalTokens: 0,
+      totalPromptTimeMs: 0,
+      totalCompletionTimeMs: 0,
+    };
+  }
+
+  startWord(): void {
+    this.metrics.currentWordStartTime = new Date();
+  }
+
+  getWordDurationMs(): number {
+    if (!this.metrics.currentWordStartTime) return 0;
+    return new Date().getTime() - this.metrics.currentWordStartTime.getTime();
+  }
+
+  recordProcessed(llmMetrics?: LlmMetrics): void {
+    this.metrics.wordsProcessed++;
+    if (llmMetrics) {
+      this.metrics.llmCalls++;
+      this.metrics.totalPromptTokens += llmMetrics.promptTokens;
+      this.metrics.totalCompletionTokens += llmMetrics.completionTokens;
+      this.metrics.totalTokens += llmMetrics.totalTokens;
+      this.metrics.totalPromptTimeMs += llmMetrics.promptTimeMs;
+      this.metrics.totalCompletionTimeMs += llmMetrics.completionTimeMs;
+    }
+  }
+
+  recordSkipped(): void {
+    this.metrics.wordsSkipped++;
+  }
+
+  recordFailed(): void {
+    this.metrics.wordsFailed++;
+  }
+
+  stop(): void {
+    this.metrics.endTime = new Date();
+  }
+
+  getWordTiming(): string {
+    const durationMs = this.getWordDurationMs();
+    const durationSec = (durationMs / 1000).toFixed(1);
+    return `⏱️  Word took ${durationSec}s`;
+  }
+
+  getEta(totalWords: number): string {
+    const processed = this.metrics.wordsProcessed;
+    const remaining = totalWords - processed - this.metrics.wordsSkipped;
+
+    if (processed === 0 || remaining <= 0) return "ETA: calculating...";
+
+    const elapsedMs = new Date().getTime() - this.metrics.startTime.getTime();
+    const avgMsPerWord = elapsedMs / processed;
+    const etaMs = avgMsPerWord * remaining;
+
+    const etaMin = Math.round(etaMs / 60000);
+    const etaHour = (etaMs / 3600000).toFixed(1);
+
+    if (etaMin < 60) {
+      return `ETA: ${etaMin} min`;
+    }
+    return `ETA: ${etaHour} hours`;
+  }
+
+  getSummary(): string {
+    const end = this.metrics.endTime || new Date();
+    const durationMs = end.getTime() - this.metrics.startTime.getTime();
+    const durationSec = (durationMs / 1000).toFixed(1);
+
+    const total =
+      this.metrics.wordsProcessed +
+      this.metrics.wordsSkipped +
+      this.metrics.wordsFailed;
+    const throughput =
+      this.metrics.wordsProcessed > 0
+        ? (this.metrics.wordsProcessed / (durationMs / 1000)).toFixed(2)
+        : "0";
+
+    const avgPromptTokens =
+      this.metrics.llmCalls > 0
+        ? (this.metrics.totalPromptTokens / this.metrics.llmCalls).toFixed(0)
+        : "0";
+    const avgCompletionTokens =
+      this.metrics.llmCalls > 0
+        ? (this.metrics.totalCompletionTokens / this.metrics.llmCalls).toFixed(
+            0,
+          )
+        : "0";
+    const avgPromptSpeed =
+      this.metrics.totalPromptTimeMs > 0
+        ? (
+            this.metrics.totalPromptTokens /
+            (this.metrics.totalPromptTimeMs / 1000)
+          ).toFixed(1)
+        : "0";
+    const avgCompletionSpeed =
+      this.metrics.totalCompletionTimeMs > 0
+        ? (
+            this.metrics.totalCompletionTokens /
+            (this.metrics.totalCompletionTimeMs / 1000)
+          ).toFixed(1)
+        : "0";
+
+    return [
+      `⏱️  Pipeline Summary`,
+      `   Duration: ${durationSec}s`,
+      `   Processed: ${this.metrics.wordsProcessed}`,
+      `   Skipped: ${this.metrics.wordsSkipped}`,
+      `   Failed: ${this.metrics.wordsFailed}`,
+      `   Total: ${total}`,
+      `   Throughput: ${throughput} words/sec`,
+      ``,
+      `🤖 LLM Metrics`,
+      `   Calls: ${this.metrics.llmCalls}`,
+      `   Avg prompt tokens: ${avgPromptTokens}`,
+      `   Avg completion tokens: ${avgCompletionTokens}`,
+      `   Avg prompt speed: ${avgPromptSpeed} tok/s`,
+      `   Avg completion speed: ${avgCompletionSpeed} tok/s`,
+    ].join("\n");
+  }
+}