lila/data-pipeline/utils/pipeline-timer.ts

interface LlmMetrics {
  promptTokens: number;
  completionTokens: number;
  totalTokens: number;
  promptTimeMs: number | null;
  completionTimeMs: number | null;
  totalTimeMs: number;
}

interface PipelineMetrics {
  startTime: Date;
  endTime?: Date;
  wordsProcessed: number;
  wordsSkipped: number;
  wordsFailed: number;
  llmCalls: number;
  totalPromptTokens: number;
  totalCompletionTokens: number;
  totalTokens: number;
  totalPromptTimeMs: number;
  totalCompletionTimeMs: number;
  totalTimeMs: number;
  currentWordStartTime?: Date;
}

/**
 * Simple timer and metrics tracker for the pipeline.
 * Tracks both pipeline throughput and LLM performance.
 */
export class PipelineTimer {
  private metrics: PipelineMetrics;

  constructor() {
    this.metrics = {
      startTime: new Date(),
      wordsProcessed: 0,
      wordsSkipped: 0,
      wordsFailed: 0,
      llmCalls: 0,
      totalPromptTokens: 0,
      totalCompletionTokens: 0,
      totalTokens: 0,
      totalPromptTimeMs: 0,
      totalCompletionTimeMs: 0,
      totalTimeMs: 0,
    };
  }

  startWord(): void {
    this.metrics.currentWordStartTime = new Date();
  }

  getWordDurationMs(): number {
    if (!this.metrics.currentWordStartTime) return 0;
    return new Date().getTime() - this.metrics.currentWordStartTime.getTime();
  }

  recordProcessed(llmMetrics?: LlmMetrics): void {
    this.metrics.wordsProcessed++;
    if (llmMetrics) {
      this.metrics.llmCalls++;
      this.metrics.totalPromptTokens += llmMetrics.promptTokens;
      this.metrics.totalCompletionTokens += llmMetrics.completionTokens;
      this.metrics.totalTokens += llmMetrics.totalTokens;
      if (llmMetrics.promptTimeMs !== null) {
        this.metrics.totalPromptTimeMs += llmMetrics.promptTimeMs;
      }
      if (llmMetrics.completionTimeMs !== null) {
        this.metrics.totalCompletionTimeMs += llmMetrics.completionTimeMs;
      }
      this.metrics.totalTimeMs += llmMetrics.totalTimeMs;
    }
  }

  recordSkipped(): void {
    this.metrics.wordsSkipped++;
  }

  recordFailed(): void {
    this.metrics.wordsFailed++;
  }

  stop(): void {
    this.metrics.endTime = new Date();
  }

  getWordTiming(): string {
    const durationMs = this.getWordDurationMs();
    const durationSec = (durationMs / 1000).toFixed(1);
    return `⏱️  Word took ${durationSec}s`;
  }

  getEta(totalWords: number): string {
    const processed = this.metrics.wordsProcessed;
    const remaining = totalWords - processed - this.metrics.wordsSkipped;

    if (processed === 0 || remaining <= 0) return "ETA: calculating...";

    const elapsedMs = new Date().getTime() - this.metrics.startTime.getTime();
    const avgMsPerWord = elapsedMs / processed;
    const etaMs = avgMsPerWord * remaining;

    const etaMin = Math.round(etaMs / 60000);
    const etaHour = (etaMs / 3600000).toFixed(1);

    if (etaMin < 60) {
      return `ETA: ${etaMin} min`;
    }
    return `ETA: ${etaHour} hours`;
  }

  getSummary(): string {
    const end = this.metrics.endTime || new Date();
    const durationMs = end.getTime() - this.metrics.startTime.getTime();
    const durationSec = (durationMs / 1000).toFixed(1);

    const total =
      this.metrics.wordsProcessed +
      this.metrics.wordsSkipped +
      this.metrics.wordsFailed;
    const throughput =
      this.metrics.wordsProcessed > 0
        ? (this.metrics.wordsProcessed / (durationMs / 1000)).toFixed(2)
        : "0";

    const avgPromptTokens =
      this.metrics.llmCalls > 0
        ? (this.metrics.totalPromptTokens / this.metrics.llmCalls).toFixed(0)
        : "0";
    const avgCompletionTokens =
      this.metrics.llmCalls > 0
        ? (this.metrics.totalCompletionTokens / this.metrics.llmCalls).toFixed(
            0,
          )
        : "0";
    const avgTotalTimeMs =
      this.metrics.llmCalls > 0
        ? (this.metrics.totalTimeMs / this.metrics.llmCalls).toFixed(0)
        : "0";

    const unifiedThroughput =
      this.metrics.totalTimeMs > 0
        ? (
            this.metrics.totalTokens /
            (this.metrics.totalTimeMs / 1000)
          ).toFixed(1)
        : "N/A";

    const hasDetailedTimings =
      this.metrics.totalPromptTimeMs > 0 ||
      this.metrics.totalCompletionTimeMs > 0;

    const avgPromptSpeed =
      this.metrics.totalPromptTimeMs > 0
        ? (
            this.metrics.totalPromptTokens /
            (this.metrics.totalPromptTimeMs / 1000)
          ).toFixed(1)
        : "N/A";

    const avgCompletionSpeed =
      this.metrics.totalCompletionTimeMs > 0
        ? (
            this.metrics.totalCompletionTokens /
            (this.metrics.totalCompletionTimeMs / 1000)
          ).toFixed(1)
        : "N/A";

    const lines = [
      `⏱️  Pipeline Summary`,
      `   Duration: ${durationSec}s`,
      `   Processed: ${this.metrics.wordsProcessed}`,
      `   Skipped: ${this.metrics.wordsSkipped}`,
      `   Failed: ${this.metrics.wordsFailed}`,
      `   Total: ${total}`,
      `   Throughput: ${throughput} words/sec`,
      ``,
      `🤖 LLM Metrics`,
      `   Calls: ${this.metrics.llmCalls}`,
      `   Avg prompt tokens: ${avgPromptTokens}`,
      `   Avg completion tokens: ${avgCompletionTokens}`,
      `   Avg total tokens: ${avgPromptTokens + avgCompletionTokens}`,
      `   Avg total request time: ${avgTotalTimeMs}ms`,
      `   Avg throughput: ${unifiedThroughput} tok/s`,
    ];

    if (hasDetailedTimings) {
      lines.push(
        ``,
        `   [Local breakdown]`,
        `   Avg prompt speed: ${avgPromptSpeed} tok/s`,
        `   Avg completion speed: ${avgCompletionSpeed} tok/s`,
      );
    }

    return lines.join("\n");
  }
}