wip
This commit is contained in:
lila
parent
8e8484f875
commit
4fb1550e7d
8 changed files with 2412 additions and 4 deletions
321
data-pipeline/utils/cli.ts
Normal file
321
data-pipeline/utils/cli.ts
Normal file
|
|
@ -0,0 +1,321 @@
|
|||
import { createInterface } from "node:readline";
|
||||
import { existsSync, readFileSync, writeFileSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import {
|
||||
ONLINE_PROVIDERS,
|
||||
LOCAL_PROVIDER,
|
||||
type OnlineProvider,
|
||||
} from "../config/providers.js";
|
||||
import type { LlmProvider } from "../config/llm.js";
|
||||
|
||||
// ── Types ──────────────────────────────────────────────────────────────────
|
||||
|
||||
export interface PipelineConfig {
|
||||
provider: LlmProvider;
|
||||
url: string;
|
||||
model: string | undefined;
|
||||
batchSize: number;
|
||||
maxRetries: number;
|
||||
}
|
||||
|
||||
interface SavedConfig {
|
||||
provider: PipelineConfig["provider"];
|
||||
model: string;
|
||||
batchSize: number;
|
||||
maxRetries: number;
|
||||
}
|
||||
|
||||
// ── Helpers ────────────────────────────────────────────────────────────────
|
||||
|
||||
function getConfigPath(): string {
|
||||
return join(import.meta.dirname, "..", ".pipeline-config.json");
|
||||
}
|
||||
|
||||
function loadLastConfig(): SavedConfig | null {
|
||||
const path = getConfigPath();
|
||||
if (!existsSync(path)) return null;
|
||||
try {
|
||||
const raw = readFileSync(path, "utf-8");
|
||||
return JSON.parse(raw) as SavedConfig;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
function saveConfig(config: SavedConfig): void {
|
||||
writeFileSync(getConfigPath(), JSON.stringify(config, null, 2));
|
||||
}
|
||||
|
||||
function ask(
|
||||
rl: ReturnType<typeof createInterface>,
|
||||
prompt: string,
|
||||
): Promise<string> {
|
||||
return new Promise((resolve) => {
|
||||
rl.question(prompt, resolve);
|
||||
});
|
||||
}
|
||||
|
||||
function printLine(char = "─", length = 50): void {
|
||||
console.log(char.repeat(length));
|
||||
}
|
||||
|
||||
function formatProviderLabel(p: LlmProvider): string {
|
||||
const meta = p === "local" ? LOCAL_PROVIDER : ONLINE_PROVIDERS[p];
|
||||
return meta ? meta.name : p;
|
||||
}
|
||||
|
||||
// ── Validation ─────────────────────────────────────────────────────────────
|
||||
|
||||
function validateBatchSize(input: string): number {
|
||||
const n = parseInt(input.trim(), 10);
|
||||
if (Number.isNaN(n) || n < 1 || n > 20) {
|
||||
throw new Error("Batch size must be an integer between 1 and 20");
|
||||
}
|
||||
return n;
|
||||
}
|
||||
|
||||
function checkApiKey(provider: OnlineProvider): void {
|
||||
const meta = ONLINE_PROVIDERS[provider];
|
||||
if (!meta) return;
|
||||
const key = process.env[meta.envVar];
|
||||
if (!key) {
|
||||
console.error(`\n ❌ Missing API key: ${meta.envVar} is not set.`);
|
||||
console.error(`Export it before running the pipeline:`);
|
||||
console.error(`export ${meta.envVar}=your_key_here\n`);
|
||||
process.exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
// ── Prompt flows ────────────────────────────────────────────────────────────
|
||||
|
||||
async function promptProviderType(
|
||||
rl: ReturnType<typeof createInterface>,
|
||||
): Promise<"local" | "online"> {
|
||||
console.log("\nSelect provider type:");
|
||||
console.log(" [1] Local (llama.cpp, ollama, lm-studio, etc.)");
|
||||
console.log(" [2] Online API (Gemini, DeepSeek, OpenRouter, Groq)");
|
||||
while (true) {
|
||||
const choice = (await ask(rl, "Choice [1/2]: ")).trim();
|
||||
if (choice === "1") return "local";
|
||||
if (choice === "2") return "online";
|
||||
console.log(" Invalid choice. Enter 1 or 2.");
|
||||
}
|
||||
}
|
||||
|
||||
async function promptOnlineProvider(
|
||||
rl: ReturnType<typeof createInterface>,
|
||||
): Promise<OnlineProvider> {
|
||||
console.log("\nSelect online provider:");
|
||||
const entries = Object.entries(ONLINE_PROVIDERS);
|
||||
entries.forEach(([_key, meta], i) => {
|
||||
const hasKey = process.env[meta.envVar] ? "✓" : "✗";
|
||||
console.log(` [${i + 1}] ${meta.name} (${hasKey} ${meta.envVar})`);
|
||||
});
|
||||
while (true) {
|
||||
const choice = (await ask(rl, `Choice [1-${entries.length}]: `)).trim();
|
||||
const idx = parseInt(choice, 10) - 1;
|
||||
if (idx >= 0 && idx < entries.length) {
|
||||
const entry = entries[idx]!;
|
||||
// Object.entries returns string keys, so we must cast it
|
||||
const provider = entry[0] as OnlineProvider;
|
||||
checkApiKey(provider);
|
||||
return provider;
|
||||
}
|
||||
console.log(` Invalid choice. Enter 1-${entries.length}.`);
|
||||
}
|
||||
}
|
||||
|
||||
async function promptModel(
|
||||
rl: ReturnType<typeof createInterface>,
|
||||
provider: LlmProvider,
|
||||
): Promise<string> {
|
||||
if (provider === "local") {
|
||||
console.log("\nLocal provider selected.");
|
||||
console.log(" Using: http://127.0.0.1:8080/v1/chat/completions");
|
||||
const model = (
|
||||
await ask(rl, "Model name (optional, press Enter to skip): ")
|
||||
).trim();
|
||||
return model || "local-model";
|
||||
}
|
||||
|
||||
// TypeScript automatically narrows `provider` to `OnlineProvider` here
|
||||
const meta = ONLINE_PROVIDERS[provider];
|
||||
if (!meta) {
|
||||
throw new Error(`Unknown provider: ${provider}`);
|
||||
}
|
||||
|
||||
console.log(`\nSelect model for ${meta.name}:`);
|
||||
meta.models.forEach((m, i) => console.log(` [${i + 1}] ${m}`));
|
||||
console.log(` [${meta.models.length + 1}] Other (type manually)`);
|
||||
|
||||
while (true) {
|
||||
const choice = (
|
||||
await ask(rl, `Choice [1-${meta.models.length + 1}]: `)
|
||||
).trim();
|
||||
const idx = parseInt(choice, 10) - 1;
|
||||
|
||||
if (idx >= 0 && idx < meta.models.length) {
|
||||
return meta.models[idx]!;
|
||||
}
|
||||
if (idx === meta.models.length) {
|
||||
const custom = (await ask(rl, "Enter model name: ")).trim();
|
||||
if (custom) return custom;
|
||||
console.log(" Model name cannot be empty.");
|
||||
continue;
|
||||
}
|
||||
console.log(` Invalid choice. Enter 1-${meta.models.length + 1}.`);
|
||||
}
|
||||
}
|
||||
|
||||
async function promptBatchSize(
|
||||
rl: ReturnType<typeof createInterface>,
|
||||
): Promise<number> {
|
||||
console.log("\nBatch size: how many words to enrich per LLM call.");
|
||||
console.log(" Recommended: 2–6 for complex languages, 4–8 for simple.");
|
||||
while (true) {
|
||||
const input = (await ask(rl, "Batch size [1-20, default 4]: ")).trim();
|
||||
if (!input) return 4;
|
||||
try {
|
||||
return validateBatchSize(input);
|
||||
} catch (err) {
|
||||
console.log(` ${(err as Error).message}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function promptConfirm(
|
||||
rl: ReturnType<typeof createInterface>,
|
||||
config: PipelineConfig,
|
||||
): Promise<boolean> {
|
||||
console.log("\n");
|
||||
printLine();
|
||||
console.log(" CONFIGURATION SUMMARY");
|
||||
printLine();
|
||||
console.log(` Provider: ${formatProviderLabel(config.provider)}`);
|
||||
console.log(` URL: ${config.url}`);
|
||||
console.log(` Model: ${config.model ?? "(none)"}`);
|
||||
console.log(` Batch: ${config.batchSize} words/call`);
|
||||
console.log(` Retries: ${config.maxRetries}`);
|
||||
printLine();
|
||||
|
||||
const answer = (await ask(rl, "\nProceed with this configuration? [Y/n]: "))
|
||||
.trim()
|
||||
.toLowerCase();
|
||||
return answer === "" || answer === "y" || answer === "yes";
|
||||
}
|
||||
|
||||
// ── Main export ────────────────────────────────────────────────────────────
|
||||
|
||||
export async function runCli(): Promise<PipelineConfig> {
|
||||
const rl = createInterface({ input: process.stdin, output: process.stdout });
|
||||
|
||||
try {
|
||||
const lastConfig = loadLastConfig();
|
||||
|
||||
// ── Startup menu ─────────────────────────────────────────────────────────
|
||||
console.log("\n");
|
||||
printLine("═", 50);
|
||||
console.log(" PIPELINE CONFIGURATION");
|
||||
printLine("═", 50);
|
||||
|
||||
if (lastConfig) {
|
||||
console.log("\nLast used configuration:");
|
||||
console.log(` Provider: ${formatProviderLabel(lastConfig.provider)}`);
|
||||
console.log(` Model: ${lastConfig.model}`);
|
||||
console.log(` Batch: ${lastConfig.batchSize}`);
|
||||
} else {
|
||||
console.log("\nNo previous configuration found.");
|
||||
}
|
||||
|
||||
console.log(
|
||||
"\n[1] Use last config" +
|
||||
(lastConfig ? "" : " (not available)") +
|
||||
"\n[2] Configure new run",
|
||||
);
|
||||
|
||||
let useLast = false;
|
||||
if (lastConfig) {
|
||||
while (true) {
|
||||
const choice = (await ask(rl, "Choice [1/2]: ")).trim();
|
||||
if (choice === "1") {
|
||||
useLast = true;
|
||||
break;
|
||||
}
|
||||
if (choice === "2") break;
|
||||
console.log(" Invalid choice. Enter 1 or 2.");
|
||||
}
|
||||
} else {
|
||||
// No last config, auto-select new run
|
||||
console.log("Auto-selecting: Configure new run");
|
||||
await ask(rl, "Press Enter to continue...");
|
||||
}
|
||||
|
||||
// ── Build config ────────────────────────────────────────────────────────
|
||||
let config: PipelineConfig;
|
||||
|
||||
if (useLast && lastConfig) {
|
||||
// Re-validate API key before reusing
|
||||
if (lastConfig.provider !== "local") {
|
||||
checkApiKey(lastConfig.provider);
|
||||
}
|
||||
const meta =
|
||||
lastConfig.provider === "local"
|
||||
? LOCAL_PROVIDER
|
||||
: ONLINE_PROVIDERS[lastConfig.provider];
|
||||
|
||||
config = {
|
||||
provider: lastConfig.provider,
|
||||
url: meta?.url ?? LOCAL_PROVIDER.url,
|
||||
model: lastConfig.model,
|
||||
batchSize: lastConfig.batchSize,
|
||||
maxRetries: lastConfig.maxRetries,
|
||||
};
|
||||
} else {
|
||||
// New run flow
|
||||
const providerType = await promptProviderType(rl);
|
||||
|
||||
let provider: LlmProvider;
|
||||
let url: string;
|
||||
|
||||
if (providerType === "local") {
|
||||
provider = "local";
|
||||
url = LOCAL_PROVIDER.url;
|
||||
} else {
|
||||
provider = await promptOnlineProvider(rl);
|
||||
url = ONLINE_PROVIDERS[provider].url;
|
||||
}
|
||||
|
||||
const model = await promptModel(rl, provider);
|
||||
const batchSize = await promptBatchSize(rl);
|
||||
|
||||
config = {
|
||||
provider: provider,
|
||||
url,
|
||||
model: model || undefined,
|
||||
batchSize,
|
||||
maxRetries: 3,
|
||||
};
|
||||
|
||||
// Confirm before saving
|
||||
const confirmed = await promptConfirm(rl, config);
|
||||
if (!confirmed) {
|
||||
console.log("\n ❌ Configuration cancelled. Exiting.\n");
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
// Save for next time
|
||||
saveConfig({
|
||||
provider: config.provider,
|
||||
model: config.model ?? "",
|
||||
batchSize: config.batchSize,
|
||||
maxRetries: config.maxRetries,
|
||||
});
|
||||
console.log("\n ✓ Configuration saved to .pipeline-config.json");
|
||||
}
|
||||
|
||||
console.log("\n");
|
||||
return config;
|
||||
} finally {
|
||||
rl.close();
|
||||
}
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue