wip

data-pipeline/.pipeline-config.json

@@ -0,0 +1 @@
+{ "provider": "local", "model": "local-model", "batchSize": 4, "maxRetries": 3 }

data-pipeline/config/batch.ts

@@ -1 +1,2 @@
-export const BATCH_CONFIG = { size: 4, maxRetries: 3 } as const;
+// Runtime-populated by pipeline.ts after CLI initialization
+export const BATCH_CONFIG = { size: 4, maxRetries: 3 };

data-pipeline/config/llm.ts

@@ -1,5 +1,10 @@
+import type { OnlineProvider } from "./providers.js";
+
+export type LlmProvider = "local" | OnlineProvider;
+
+// Runtime-populated by pipeline.ts after CLI initialization
 export const LLM_CONFIG = {
-  provider: "local" as "local" | "openrouter" | "deepseek" | "gemini",
+  provider: "local" as LlmProvider,
   url: "http://127.0.0.1:8080/v1/chat/completions",
   model: undefined as string | undefined,
-} as const;
+};

data-pipeline/config/providers.ts

@@ -0,0 +1,58 @@
+export type ProviderMeta = {
+  name: string;
+  envVar: string;
+  url: string;
+  requiresKey: boolean;
+  models: string[];
+};
+
+// 1. Explicitly define the literal union
+export type OnlineProvider = "gemini" | "deepseek" | "openrouter" | "groq";
+
+// 2. Use the union to type the Record
+export const ONLINE_PROVIDERS: Record<OnlineProvider, ProviderMeta> = {
+  gemini: {
+    name: "Gemini",
+    envVar: "GEMINI_API_KEY",
+    url: "https://generativelanguage.googleapis.com/v1beta",
+    requiresKey: true,
+    models: ["gemini-2.5-flash", "gemini-2.5-pro"],
+  },
+  deepseek: {
+    name: "DeepSeek",
+    envVar: "DEEPSEEK_API_KEY",
+    url: "https://api.deepseek.com/v1/chat/completions",
+    requiresKey: true,
+    models: ["deepseek-chat", "deepseek-reasoner"],
+  },
+  openrouter: {
+    name: "OpenRouter",
+    envVar: "OPENROUTER_API_KEY",
+    url: "https://openrouter.ai/api/v1/chat/completions",
+    requiresKey: true,
+    models: [
+      "openai/gpt-oss-120b:free",
+      "google/gemma-4-31b-it:free",
+      "qwen/qwen3-next-80b-a3b-instruct:free",
+      "meta-llama/llama-3.3-70b-instruct:free",
+      "anthropic/claude-sonnet-4",
+      "google/gemini-2.5-flash",
+      "deepseek/deepseek-chat-v3",
+    ],
+  },
+  groq: {
+    name: "Groq",
+    envVar: "GROQ_API_KEY",
+    url: "https://api.groq.com/openai/v1/chat/completions",
+    requiresKey: true,
+    models: ["llama-3.3-70b-versatile", "gemma2-9b-it", "mixtral-8x7b-32768"],
+  },
+};
+
+export const LOCAL_PROVIDER: ProviderMeta = {
+  name: "Local (llama.cpp / ollama / lm-studio)",
+  envVar: "",
+  url: "http://127.0.0.1:8080/v1/chat/completions",
+  requiresKey: false,
+  models: [],
+};

data-pipeline/pipeline.ts

@@ -12,10 +12,33 @@ import { deleteFileIfExists } from "./utils/delete-file.js";
 import { PipelineTimer } from "./utils/pipeline-timer.js";
 import { ProgressTracker } from "./utils/progress-tracker.js";
 import { verifyEnrichedFile } from "./utils/verify-enriched-file.js";
+import { runCli } from "./utils/cli.js";
+import type { PipelineConfig } from "./utils/cli.js";
+import { LLM_CONFIG } from "./config/llm.js";
 import { BATCH_CONFIG } from "./config/batch.js";
 
+// Runtime config accessor for other modules
+let RUNTIME_CONFIG: PipelineConfig;
+
+export function getRuntimeConfig(): PipelineConfig {
+  return RUNTIME_CONFIG;
+}
+
 async function main() {
+  // ── Interactive CLI ──────────────────────────────────────────────────────
+  RUNTIME_CONFIG = await runCli();
+
+  // Populate shared config objects so existing imports keep working
+  LLM_CONFIG.provider = RUNTIME_CONFIG.provider;
+  LLM_CONFIG.url = RUNTIME_CONFIG.url;
+  LLM_CONFIG.model = RUNTIME_CONFIG.model;
+  BATCH_CONFIG.size = RUNTIME_CONFIG.batchSize;
+  BATCH_CONFIG.maxRetries = RUNTIME_CONFIG.maxRetries;
+
   console.log("Starting data pipeline...\n");
+  console.log(`Provider: ${RUNTIME_CONFIG.provider}`);
+  console.log(`Model:    ${RUNTIME_CONFIG.model ?? "(none)"}`);
+  console.log(`Batch:    ${RUNTIME_CONFIG.batchSize} words/call\n`);
 
   const timer = new PipelineTimer();
 

data-pipeline/utils/cli.ts

@@ -0,0 +1,321 @@
+import { createInterface } from "node:readline";
+import { existsSync, readFileSync, writeFileSync } from "node:fs";
+import { join } from "node:path";
+import {
+  ONLINE_PROVIDERS,
+  LOCAL_PROVIDER,
+  type OnlineProvider,
+} from "../config/providers.js";
+import type { LlmProvider } from "../config/llm.js";
+
+// ── Types ──────────────────────────────────────────────────────────────────
+
+export interface PipelineConfig {
+  provider: LlmProvider;
+  url: string;
+  model: string | undefined;
+  batchSize: number;
+  maxRetries: number;
+}
+
+interface SavedConfig {
+  provider: PipelineConfig["provider"];
+  model: string;
+  batchSize: number;
+  maxRetries: number;
+}
+
+// ── Helpers ────────────────────────────────────────────────────────────────
+
+function getConfigPath(): string {
+  return join(import.meta.dirname, "..", ".pipeline-config.json");
+}
+
+function loadLastConfig(): SavedConfig | null {
+  const path = getConfigPath();
+  if (!existsSync(path)) return null;
+  try {
+    const raw = readFileSync(path, "utf-8");
+    return JSON.parse(raw) as SavedConfig;
+  } catch {
+    return null;
+  }
+}
+
+function saveConfig(config: SavedConfig): void {
+  writeFileSync(getConfigPath(), JSON.stringify(config, null, 2));
+}
+
+function ask(
+  rl: ReturnType<typeof createInterface>,
+  prompt: string,
+): Promise<string> {
+  return new Promise((resolve) => {
+    rl.question(prompt, resolve);
+  });
+}
+
+function printLine(char = "─", length = 50): void {
+  console.log(char.repeat(length));
+}
+
+function formatProviderLabel(p: LlmProvider): string {
+  const meta = p === "local" ? LOCAL_PROVIDER : ONLINE_PROVIDERS[p];
+  return meta ? meta.name : p;
+}
+
+// ── Validation ─────────────────────────────────────────────────────────────
+
+function validateBatchSize(input: string): number {
+  const n = parseInt(input.trim(), 10);
+  if (Number.isNaN(n) || n < 1 || n > 20) {
+    throw new Error("Batch size must be an integer between 1 and 20");
+  }
+  return n;
+}
+
+function checkApiKey(provider: OnlineProvider): void {
+  const meta = ONLINE_PROVIDERS[provider];
+  if (!meta) return;
+  const key = process.env[meta.envVar];
+  if (!key) {
+    console.error(`\n ❌ Missing API key: ${meta.envVar} is not set.`);
+    console.error(`Export it before running the pipeline:`);
+    console.error(`export ${meta.envVar}=your_key_here\n`);
+    process.exit(1);
+  }
+}
+
+// ── Prompt flows ────────────────────────────────────────────────────────────
+
+async function promptProviderType(
+  rl: ReturnType<typeof createInterface>,
+): Promise<"local" | "online"> {
+  console.log("\nSelect provider type:");
+  console.log("  [1] Local (llama.cpp, ollama, lm-studio, etc.)");
+  console.log("  [2] Online API (Gemini, DeepSeek, OpenRouter, Groq)");
+  while (true) {
+    const choice = (await ask(rl, "Choice [1/2]: ")).trim();
+    if (choice === "1") return "local";
+    if (choice === "2") return "online";
+    console.log("  Invalid choice. Enter 1 or 2.");
+  }
+}
+
+async function promptOnlineProvider(
+  rl: ReturnType<typeof createInterface>,
+): Promise<OnlineProvider> {
+  console.log("\nSelect online provider:");
+  const entries = Object.entries(ONLINE_PROVIDERS);
+  entries.forEach(([_key, meta], i) => {
+    const hasKey = process.env[meta.envVar] ? "✓" : "✗";
+    console.log(`  [${i + 1}] ${meta.name} (${hasKey} ${meta.envVar})`);
+  });
+  while (true) {
+    const choice = (await ask(rl, `Choice [1-${entries.length}]: `)).trim();
+    const idx = parseInt(choice, 10) - 1;
+    if (idx >= 0 && idx < entries.length) {
+      const entry = entries[idx]!;
+      // Object.entries returns string keys, so we must cast it
+      const provider = entry[0] as OnlineProvider;
+      checkApiKey(provider);
+      return provider;
+    }
+    console.log(`  Invalid choice. Enter 1-${entries.length}.`);
+  }
+}
+
+async function promptModel(
+  rl: ReturnType<typeof createInterface>,
+  provider: LlmProvider,
+): Promise<string> {
+  if (provider === "local") {
+    console.log("\nLocal provider selected.");
+    console.log("  Using: http://127.0.0.1:8080/v1/chat/completions");
+    const model = (
+      await ask(rl, "Model name (optional, press Enter to skip): ")
+    ).trim();
+    return model || "local-model";
+  }
+
+  // TypeScript automatically narrows `provider` to `OnlineProvider` here
+  const meta = ONLINE_PROVIDERS[provider];
+  if (!meta) {
+    throw new Error(`Unknown provider: ${provider}`);
+  }
+
+  console.log(`\nSelect model for ${meta.name}:`);
+  meta.models.forEach((m, i) => console.log(`  [${i + 1}] ${m}`));
+  console.log(`  [${meta.models.length + 1}] Other (type manually)`);
+
+  while (true) {
+    const choice = (
+      await ask(rl, `Choice [1-${meta.models.length + 1}]: `)
+    ).trim();
+    const idx = parseInt(choice, 10) - 1;
+
+    if (idx >= 0 && idx < meta.models.length) {
+      return meta.models[idx]!;
+    }
+    if (idx === meta.models.length) {
+      const custom = (await ask(rl, "Enter model name: ")).trim();
+      if (custom) return custom;
+      console.log("  Model name cannot be empty.");
+      continue;
+    }
+    console.log(`  Invalid choice. Enter 1-${meta.models.length + 1}.`);
+  }
+}
+
+async function promptBatchSize(
+  rl: ReturnType<typeof createInterface>,
+): Promise<number> {
+  console.log("\nBatch size: how many words to enrich per LLM call.");
+  console.log("  Recommended: 2–6 for complex languages, 4–8 for simple.");
+  while (true) {
+    const input = (await ask(rl, "Batch size [1-20, default 4]: ")).trim();
+    if (!input) return 4;
+    try {
+      return validateBatchSize(input);
+    } catch (err) {
+      console.log(`  ${(err as Error).message}`);
+    }
+  }
+}
+
+async function promptConfirm(
+  rl: ReturnType<typeof createInterface>,
+  config: PipelineConfig,
+): Promise<boolean> {
+  console.log("\n");
+  printLine();
+  console.log("  CONFIGURATION SUMMARY");
+  printLine();
+  console.log(`  Provider: ${formatProviderLabel(config.provider)}`);
+  console.log(`  URL: ${config.url}`);
+  console.log(`  Model: ${config.model ?? "(none)"}`);
+  console.log(`  Batch: ${config.batchSize} words/call`);
+  console.log(`  Retries: ${config.maxRetries}`);
+  printLine();
+
+  const answer = (await ask(rl, "\nProceed with this configuration? [Y/n]: "))
+    .trim()
+    .toLowerCase();
+  return answer === "" || answer === "y" || answer === "yes";
+}
+
+// ── Main export ────────────────────────────────────────────────────────────
+
+export async function runCli(): Promise<PipelineConfig> {
+  const rl = createInterface({ input: process.stdin, output: process.stdout });
+
+  try {
+    const lastConfig = loadLastConfig();
+
+    // ── Startup menu ─────────────────────────────────────────────────────────
+    console.log("\n");
+    printLine("═", 50);
+    console.log("  PIPELINE CONFIGURATION");
+    printLine("═", 50);
+
+    if (lastConfig) {
+      console.log("\nLast used configuration:");
+      console.log(`  Provider: ${formatProviderLabel(lastConfig.provider)}`);
+      console.log(`  Model:    ${lastConfig.model}`);
+      console.log(`  Batch:    ${lastConfig.batchSize}`);
+    } else {
+      console.log("\nNo previous configuration found.");
+    }
+
+    console.log(
+      "\n[1] Use last config" +
+        (lastConfig ? "" : " (not available)") +
+        "\n[2] Configure new run",
+    );
+
+    let useLast = false;
+    if (lastConfig) {
+      while (true) {
+        const choice = (await ask(rl, "Choice [1/2]: ")).trim();
+        if (choice === "1") {
+          useLast = true;
+          break;
+        }
+        if (choice === "2") break;
+        console.log("  Invalid choice. Enter 1 or 2.");
+      }
+    } else {
+      // No last config, auto-select new run
+      console.log("Auto-selecting: Configure new run");
+      await ask(rl, "Press Enter to continue...");
+    }
+
+    // ── Build config ────────────────────────────────────────────────────────
+    let config: PipelineConfig;
+
+    if (useLast && lastConfig) {
+      // Re-validate API key before reusing
+      if (lastConfig.provider !== "local") {
+        checkApiKey(lastConfig.provider);
+      }
+      const meta =
+        lastConfig.provider === "local"
+          ? LOCAL_PROVIDER
+          : ONLINE_PROVIDERS[lastConfig.provider];
+
+      config = {
+        provider: lastConfig.provider,
+        url: meta?.url ?? LOCAL_PROVIDER.url,
+        model: lastConfig.model,
+        batchSize: lastConfig.batchSize,
+        maxRetries: lastConfig.maxRetries,
+      };
+    } else {
+      // New run flow
+      const providerType = await promptProviderType(rl);
+
+      let provider: LlmProvider;
+      let url: string;
+
+      if (providerType === "local") {
+        provider = "local";
+        url = LOCAL_PROVIDER.url;
+      } else {
+        provider = await promptOnlineProvider(rl);
+        url = ONLINE_PROVIDERS[provider].url;
+      }
+
+      const model = await promptModel(rl, provider);
+      const batchSize = await promptBatchSize(rl);
+
+      config = {
+        provider: provider,
+        url,
+        model: model || undefined,
+        batchSize,
+        maxRetries: 3,
+      };
+
+      // Confirm before saving
+      const confirmed = await promptConfirm(rl, config);
+      if (!confirmed) {
+        console.log("\n ❌ Configuration cancelled. Exiting.\n");
+        process.exit(0);
+      }
+
+      // Save for next time
+      saveConfig({
+        provider: config.provider,
+        model: config.model ?? "",
+        batchSize: config.batchSize,
+        maxRetries: config.maxRetries,
+      });
+      console.log("\n ✓ Configuration saved to .pipeline-config.json");
+    }
+
+    console.log("\n");
+    return config;
+  } finally {
+    rl.close();
+  }
+}

data-pipeline/utils/llm-adapters/factory.ts

@@ -6,7 +6,11 @@ import type { LlmAdapter } from "./types.js";
 export function createAdapter(): LlmAdapter {
   switch (LLM_CONFIG.provider) {
     case "local":
-      return new OpenAiCompatibleAdapter(LLM_CONFIG.url);
+      return new OpenAiCompatibleAdapter(
+        LLM_CONFIG.url,
+        undefined,
+        LLM_CONFIG.model,
+      );
     case "openrouter":
       return new OpenAiCompatibleAdapter(
         LLM_CONFIG.url,
@@ -19,6 +23,12 @@ export function createAdapter(): LlmAdapter {
         process.env["DEEPSEEK_API_KEY"],
         LLM_CONFIG.model,
       );
+    case "groq":
+      return new OpenAiCompatibleAdapter(
+        LLM_CONFIG.url,
+        process.env["GROQ_API_KEY"],
+        LLM_CONFIG.model,
+      );
     case "gemini": {
       const apiKey = process.env["GEMINI_API_KEY"];
       if (!apiKey) throw new Error("GEMINI_API_KEY env var not set");
@@ -26,5 +36,7 @@ export function createAdapter(): LlmAdapter {
         throw new Error("LLM_CONFIG.model required for gemini");
       return new GeminiAdapter(apiKey, LLM_CONFIG.model);
     }
+    default:
+      throw new Error(`Unknown provider: ${LLM_CONFIG.provider as string}`);
   }
 }