From 0ae3b9f686023ac4b4650eaf4193d38ea4c22105 Mon Sep 17 00:00:00 2001 From: lila Date: Sat, 18 Jul 2026 15:06:25 +0200 Subject: [PATCH] refactor: remove CLI, local LLM support, and unused configs --- data-pipeline/.pipeline-config.json | 1 - data-pipeline/config/batch.ts | 2 - data-pipeline/config/llm.ts | 10 - data-pipeline/config/providers.ts | 58 ---- data-pipeline/utils/check-llm-server.ts | 49 --- data-pipeline/utils/cli.ts | 321 ------------------ data-pipeline/utils/llm-adapters/factory.ts | 42 --- .../utils/llm-adapters/openai-compatible.ts | 92 ----- 8 files changed, 575 deletions(-) delete mode 100644 data-pipeline/.pipeline-config.json delete mode 100644 data-pipeline/config/batch.ts delete mode 100644 data-pipeline/config/llm.ts delete mode 100644 data-pipeline/config/providers.ts delete mode 100644 data-pipeline/utils/check-llm-server.ts delete mode 100644 data-pipeline/utils/cli.ts delete mode 100644 data-pipeline/utils/llm-adapters/factory.ts delete mode 100644 data-pipeline/utils/llm-adapters/openai-compatible.ts diff --git a/data-pipeline/.pipeline-config.json b/data-pipeline/.pipeline-config.json deleted file mode 100644 index 6b4d734..0000000 --- a/data-pipeline/.pipeline-config.json +++ /dev/null @@ -1 +0,0 @@ -{ "provider": "local", "model": "local-model", "batchSize": 1, "maxRetries": 3 } diff --git a/data-pipeline/config/batch.ts b/data-pipeline/config/batch.ts deleted file mode 100644 index cdd1d1a..0000000 --- a/data-pipeline/config/batch.ts +++ /dev/null @@ -1,2 +0,0 @@ -// Runtime-populated by pipeline.ts after CLI initialization -export const BATCH_CONFIG = { size: 4, maxRetries: 3 }; diff --git a/data-pipeline/config/llm.ts b/data-pipeline/config/llm.ts deleted file mode 100644 index 9036310..0000000 --- a/data-pipeline/config/llm.ts +++ /dev/null @@ -1,10 +0,0 @@ -import type { OnlineProvider } from "./providers.js"; - -export type LlmProvider = "local" | OnlineProvider; - -// Runtime-populated by pipeline.ts after CLI initialization -export const LLM_CONFIG = { - provider: "local" as LlmProvider, - url: "http://127.0.0.1:8080/v1/chat/completions", - model: undefined as string | undefined, -}; diff --git a/data-pipeline/config/providers.ts b/data-pipeline/config/providers.ts deleted file mode 100644 index a99a0e4..0000000 --- a/data-pipeline/config/providers.ts +++ /dev/null @@ -1,58 +0,0 @@ -export type ProviderMeta = { - name: string; - envVar: string; - url: string; - requiresKey: boolean; - models: string[]; -}; - -// 1. Explicitly define the literal union -export type OnlineProvider = "gemini" | "deepseek" | "openrouter" | "groq"; - -// 2. Use the union to type the Record -export const ONLINE_PROVIDERS: Record = { - gemini: { - name: "Gemini", - envVar: "GEMINI_API_KEY", - url: "https://generativelanguage.googleapis.com/v1beta", - requiresKey: true, - models: ["gemini-2.5-flash", "gemini-2.5-pro"], - }, - deepseek: { - name: "DeepSeek", - envVar: "DEEPSEEK_API_KEY", - url: "https://api.deepseek.com/v1/chat/completions", - requiresKey: true, - models: ["deepseek-chat", "deepseek-reasoner"], - }, - openrouter: { - name: "OpenRouter", - envVar: "OPENROUTER_API_KEY", - url: "https://openrouter.ai/api/v1/chat/completions", - requiresKey: true, - models: [ - "openai/gpt-oss-120b:free", - "google/gemma-4-31b-it:free", - "qwen/qwen3-next-80b-a3b-instruct:free", - "meta-llama/llama-3.3-70b-instruct:free", - "anthropic/claude-sonnet-4", - "google/gemini-2.5-flash", - "deepseek/deepseek-chat-v3", - ], - }, - groq: { - name: "Groq", - envVar: "GROQ_API_KEY", - url: "https://api.groq.com/openai/v1/chat/completions", - requiresKey: true, - models: ["llama-3.3-70b-versatile", "gemma2-9b-it", "mixtral-8x7b-32768"], - }, -}; - -export const LOCAL_PROVIDER: ProviderMeta = { - name: "Local (llama.cpp / ollama / lm-studio)", - envVar: "", - url: "http://127.0.0.1:8080/v1/chat/completions", - requiresKey: false, - models: [], -}; diff --git a/data-pipeline/utils/check-llm-server.ts b/data-pipeline/utils/check-llm-server.ts deleted file mode 100644 index b3f4f96..0000000 --- a/data-pipeline/utils/check-llm-server.ts +++ /dev/null @@ -1,49 +0,0 @@ -import { LLM_CONFIG } from "../config/llm.js"; - -/** - * Pings the local llama.cpp server to ensure it's up, running, and has a model loaded. - * If the server is offline or still loading, it terminates the pipeline gracefully. - * Skipped entirely when using a cloud provider. - */ -export async function checkLlmServer( - url = "http://127.0.0.1:8080/health", -): Promise { - if (LLM_CONFIG.provider !== "local") { - console.log("🌐 Using cloud provider — skipping local health check."); - return; - } - - try { - const response = await fetch(url); - - // llama.cpp returns a 503 status if the server is up but the model weights are still loading - if (response.status === 503) { - throw new Error( - "Local AI engine is starting up, but the model is still loading into memory. " + - "Please wait a minute for the weights to load, then run the pipeline again.", - ); - } - - // Parse the JSON health response (expected: { status: "ok" }) - const data = (await response.json()) as { status?: string }; - - if (response.ok && data.status === "ok") { - console.log("🟢 Local AI engine is connected and ready for inference!"); - return; - } - - // Catch-all for unexpected active server responses - throw new Error( - `Unknown response from local AI engine health check (Status: ${response.status}).`, - ); - } catch (error: unknown) { - if (error instanceof Error && error.message.includes("Local AI engine")) { - throw error; // Re-throw our own errors - } - throw new Error( - `Could not connect to the local AI engine at ${url}. ` + - "Make sure your './llama-server' command is actively running in another terminal tab.", - { cause: error }, - ); - } -} diff --git a/data-pipeline/utils/cli.ts b/data-pipeline/utils/cli.ts deleted file mode 100644 index 9a17e75..0000000 --- a/data-pipeline/utils/cli.ts +++ /dev/null @@ -1,321 +0,0 @@ -import { createInterface } from "node:readline"; -import { existsSync, readFileSync, writeFileSync } from "node:fs"; -import { join } from "node:path"; -import { - ONLINE_PROVIDERS, - LOCAL_PROVIDER, - type OnlineProvider, -} from "../config/providers.js"; -import type { LlmProvider } from "../config/llm.js"; - -// ── Types ────────────────────────────────────────────────────────────────── - -export interface PipelineConfig { - provider: LlmProvider; - url: string; - model: string | undefined; - batchSize: number; - maxRetries: number; -} - -interface SavedConfig { - provider: PipelineConfig["provider"]; - model: string; - batchSize: number; - maxRetries: number; -} - -// ── Helpers ──────────────────────────────────────────────────────────────── - -function getConfigPath(): string { - return join(import.meta.dirname, "..", ".pipeline-config.json"); -} - -function loadLastConfig(): SavedConfig | null { - const path = getConfigPath(); - if (!existsSync(path)) return null; - try { - const raw = readFileSync(path, "utf-8"); - return JSON.parse(raw) as SavedConfig; - } catch { - return null; - } -} - -function saveConfig(config: SavedConfig): void { - writeFileSync(getConfigPath(), JSON.stringify(config, null, 2)); -} - -function ask( - rl: ReturnType, - prompt: string, -): Promise { - return new Promise((resolve) => { - rl.question(prompt, resolve); - }); -} - -function printLine(char = "─", length = 50): void { - console.log(char.repeat(length)); -} - -function formatProviderLabel(p: LlmProvider): string { - const meta = p === "local" ? LOCAL_PROVIDER : ONLINE_PROVIDERS[p]; - return meta ? meta.name : p; -} - -// ── Validation ───────────────────────────────────────────────────────────── - -function validateBatchSize(input: string): number { - const n = parseInt(input.trim(), 10); - if (Number.isNaN(n) || n < 1 || n > 20) { - throw new Error("Batch size must be an integer between 1 and 20"); - } - return n; -} - -function checkApiKey(provider: OnlineProvider): void { - const meta = ONLINE_PROVIDERS[provider]; - if (!meta) return; - const key = process.env[meta.envVar]; - if (!key) { - console.error(`\n ❌ Missing API key: ${meta.envVar} is not set.`); - console.error(`Export it before running the pipeline:`); - console.error(`export ${meta.envVar}=your_key_here\n`); - process.exit(1); - } -} - -// ── Prompt flows ──────────────────────────────────────────────────────────── - -async function promptProviderType( - rl: ReturnType, -): Promise<"local" | "online"> { - console.log("\nSelect provider type:"); - console.log(" [1] Local (llama.cpp, ollama, lm-studio, etc.)"); - console.log(" [2] Online API (Gemini, DeepSeek, OpenRouter, Groq)"); - while (true) { - const choice = (await ask(rl, "Choice [1/2]: ")).trim(); - if (choice === "1") return "local"; - if (choice === "2") return "online"; - console.log(" Invalid choice. Enter 1 or 2."); - } -} - -async function promptOnlineProvider( - rl: ReturnType, -): Promise { - console.log("\nSelect online provider:"); - const entries = Object.entries(ONLINE_PROVIDERS); - entries.forEach(([_key, meta], i) => { - const hasKey = process.env[meta.envVar] ? "✓" : "✗"; - console.log(` [${i + 1}] ${meta.name} (${hasKey} ${meta.envVar})`); - }); - while (true) { - const choice = (await ask(rl, `Choice [1-${entries.length}]: `)).trim(); - const idx = parseInt(choice, 10) - 1; - if (idx >= 0 && idx < entries.length) { - const entry = entries[idx]!; - // Object.entries returns string keys, so we must cast it - const provider = entry[0] as OnlineProvider; - checkApiKey(provider); - return provider; - } - console.log(` Invalid choice. Enter 1-${entries.length}.`); - } -} - -async function promptModel( - rl: ReturnType, - provider: LlmProvider, -): Promise { - if (provider === "local") { - console.log("\nLocal provider selected."); - console.log(" Using: http://127.0.0.1:8080/v1/chat/completions"); - const model = ( - await ask(rl, "Model name (optional, press Enter to skip): ") - ).trim(); - return model || "local-model"; - } - - // TypeScript automatically narrows `provider` to `OnlineProvider` here - const meta = ONLINE_PROVIDERS[provider]; - if (!meta) { - throw new Error(`Unknown provider: ${provider}`); - } - - console.log(`\nSelect model for ${meta.name}:`); - meta.models.forEach((m, i) => console.log(` [${i + 1}] ${m}`)); - console.log(` [${meta.models.length + 1}] Other (type manually)`); - - while (true) { - const choice = ( - await ask(rl, `Choice [1-${meta.models.length + 1}]: `) - ).trim(); - const idx = parseInt(choice, 10) - 1; - - if (idx >= 0 && idx < meta.models.length) { - return meta.models[idx]!; - } - if (idx === meta.models.length) { - const custom = (await ask(rl, "Enter model name: ")).trim(); - if (custom) return custom; - console.log(" Model name cannot be empty."); - continue; - } - console.log(` Invalid choice. Enter 1-${meta.models.length + 1}.`); - } -} - -async function promptBatchSize( - rl: ReturnType, -): Promise { - console.log("\nBatch size: how many words to enrich per LLM call."); - console.log(" Recommended: 2–6 for complex languages, 4–8 for simple."); - while (true) { - const input = (await ask(rl, "Batch size [1-20, default 4]: ")).trim(); - if (!input) return 4; - try { - return validateBatchSize(input); - } catch (err) { - console.log(` ${(err as Error).message}`); - } - } -} - -async function promptConfirm( - rl: ReturnType, - config: PipelineConfig, -): Promise { - console.log("\n"); - printLine(); - console.log(" CONFIGURATION SUMMARY"); - printLine(); - console.log(` Provider: ${formatProviderLabel(config.provider)}`); - console.log(` URL: ${config.url}`); - console.log(` Model: ${config.model ?? "(none)"}`); - console.log(` Batch: ${config.batchSize} words/call`); - console.log(` Retries: ${config.maxRetries}`); - printLine(); - - const answer = (await ask(rl, "\nProceed with this configuration? [Y/n]: ")) - .trim() - .toLowerCase(); - return answer === "" || answer === "y" || answer === "yes"; -} - -// ── Main export ──────────────────────────────────────────────────────────── - -export async function runCli(): Promise { - const rl = createInterface({ input: process.stdin, output: process.stdout }); - - try { - const lastConfig = loadLastConfig(); - - // ── Startup menu ───────────────────────────────────────────────────────── - console.log("\n"); - printLine("═", 50); - console.log(" PIPELINE CONFIGURATION"); - printLine("═", 50); - - if (lastConfig) { - console.log("\nLast used configuration:"); - console.log(` Provider: ${formatProviderLabel(lastConfig.provider)}`); - console.log(` Model: ${lastConfig.model}`); - console.log(` Batch: ${lastConfig.batchSize}`); - } else { - console.log("\nNo previous configuration found."); - } - - console.log( - "\n[1] Use last config" + - (lastConfig ? "" : " (not available)") + - "\n[2] Configure new run", - ); - - let useLast = false; - if (lastConfig) { - while (true) { - const choice = (await ask(rl, "Choice [1/2]: ")).trim(); - if (choice === "1") { - useLast = true; - break; - } - if (choice === "2") break; - console.log(" Invalid choice. Enter 1 or 2."); - } - } else { - // No last config, auto-select new run - console.log("Auto-selecting: Configure new run"); - await ask(rl, "Press Enter to continue..."); - } - - // ── Build config ──────────────────────────────────────────────────────── - let config: PipelineConfig; - - if (useLast && lastConfig) { - // Re-validate API key before reusing - if (lastConfig.provider !== "local") { - checkApiKey(lastConfig.provider); - } - const meta = - lastConfig.provider === "local" - ? LOCAL_PROVIDER - : ONLINE_PROVIDERS[lastConfig.provider]; - - config = { - provider: lastConfig.provider, - url: meta?.url ?? LOCAL_PROVIDER.url, - model: lastConfig.model, - batchSize: lastConfig.batchSize, - maxRetries: lastConfig.maxRetries, - }; - } else { - // New run flow - const providerType = await promptProviderType(rl); - - let provider: LlmProvider; - let url: string; - - if (providerType === "local") { - provider = "local"; - url = LOCAL_PROVIDER.url; - } else { - provider = await promptOnlineProvider(rl); - url = ONLINE_PROVIDERS[provider].url; - } - - const model = await promptModel(rl, provider); - const batchSize = await promptBatchSize(rl); - - config = { - provider: provider, - url, - model: model || undefined, - batchSize, - maxRetries: 3, - }; - - // Confirm before saving - const confirmed = await promptConfirm(rl, config); - if (!confirmed) { - console.log("\n ❌ Configuration cancelled. Exiting.\n"); - process.exit(0); - } - - // Save for next time - saveConfig({ - provider: config.provider, - model: config.model ?? "", - batchSize: config.batchSize, - maxRetries: config.maxRetries, - }); - console.log("\n ✓ Configuration saved to .pipeline-config.json"); - } - - console.log("\n"); - return config; - } finally { - rl.close(); - } -} diff --git a/data-pipeline/utils/llm-adapters/factory.ts b/data-pipeline/utils/llm-adapters/factory.ts deleted file mode 100644 index f24a1c8..0000000 --- a/data-pipeline/utils/llm-adapters/factory.ts +++ /dev/null @@ -1,42 +0,0 @@ -import { LLM_CONFIG } from "../../config/llm.js"; -import { OpenAiCompatibleAdapter } from "./openai-compatible.js"; -import { GeminiAdapter } from "./gemini.js"; -import type { LlmAdapter } from "./types.js"; - -export function createAdapter(): LlmAdapter { - switch (LLM_CONFIG.provider) { - case "local": - return new OpenAiCompatibleAdapter( - LLM_CONFIG.url, - undefined, - LLM_CONFIG.model, - ); - case "openrouter": - return new OpenAiCompatibleAdapter( - LLM_CONFIG.url, - process.env["OPENROUTER_API_KEY"], - LLM_CONFIG.model, - ); - case "deepseek": - return new OpenAiCompatibleAdapter( - LLM_CONFIG.url, - process.env["DEEPSEEK_API_KEY"], - LLM_CONFIG.model, - ); - case "groq": - return new OpenAiCompatibleAdapter( - LLM_CONFIG.url, - process.env["GROQ_API_KEY"], - LLM_CONFIG.model, - ); - case "gemini": { - const apiKey = process.env["GEMINI_API_KEY"]; - if (!apiKey) throw new Error("GEMINI_API_KEY env var not set"); - if (!LLM_CONFIG.model) - throw new Error("LLM_CONFIG.model required for gemini"); - return new GeminiAdapter(apiKey, LLM_CONFIG.model); - } - default: - throw new Error(`Unknown provider: ${LLM_CONFIG.provider as string}`); - } -} diff --git a/data-pipeline/utils/llm-adapters/openai-compatible.ts b/data-pipeline/utils/llm-adapters/openai-compatible.ts deleted file mode 100644 index aa95aef..0000000 --- a/data-pipeline/utils/llm-adapters/openai-compatible.ts +++ /dev/null @@ -1,92 +0,0 @@ -import type { LlmAdapter } from "./types.js"; - -interface OpenAiResponse { - choices: Array<{ message: { content: string } }>; - usage: { - prompt_tokens: number; - completion_tokens: number; - total_tokens: number; - }; - timings?: { prompt_ms: number; predicted_ms: number }; -} - -export class OpenAiCompatibleAdapter implements LlmAdapter { - private url: string; - private apiKey: string | undefined; - private model: string | undefined; - - constructor(url: string, apiKey?: string, model?: string) { - this.url = url; - this.apiKey = apiKey; - this.model = model; - } - - async call( - words: string[], - systemPrompt: string, - ): Promise<{ - content: string; - promptTokens: number; - completionTokens: number; - totalTokens: number; - promptTimeMs: number | null; - completionTimeMs: number | null; - totalTimeMs: number; - }> { - const payload: Record = { - messages: [ - { role: "system", content: systemPrompt }, - { role: "user", content: JSON.stringify(words) }, - ], - temperature: 0.1, - top_p: 0.9, - max_tokens: Math.ceil(words.length * 250 * 1.2), - }; - - if (this.model) { - payload["model"] = this.model; - } - - const headers: Record = { - "Content-Type": "application/json", - }; - - if (this.apiKey) { - headers["Authorization"] = `Bearer ${this.apiKey}`; - } - - const startTime = Date.now(); - - const response = await fetch(this.url, { - method: "POST", - headers, - body: JSON.stringify(payload), - }); - - const totalTimeMs = Date.now() - startTime; - - if (!response.ok) { - throw new Error(`LLM server responded with status: ${response.status}`); - } - - const json = (await response.json()) as OpenAiResponse; - - const content = json.choices[0]?.message?.content; - if (!content) { - throw new Error("LLM response content is empty"); - } - - const promptTokens = json.usage.prompt_tokens; - const completionTokens = json.usage.completion_tokens; - - return { - content, - promptTokens, - completionTokens, - totalTokens: json.usage.total_tokens, - promptTimeMs: json.timings?.prompt_ms ?? null, - completionTimeMs: json.timings?.predicted_ms ?? null, - totalTimeMs, - }; - } -}