forgejo-lila/lila
1
0
Fork 0
Code Issues 50 Pull requests Projects 1 Releases Packages Wiki Activity Actions

wip

Browse source
This commit is contained in:
lila 2026-07-06 17:12:25 +02:00
parent 8e8484f875
commit 4fb1550e7d
8 changed files with 2412 additions and 4 deletions
Download patch file Download diff file Expand all files Collapse all files

23
data-pipeline/pipeline.ts
View file

@ -12,10 +12,33 @@ import { deleteFileIfExists } from "./utils/delete-file.js";
import { PipelineTimer } from "./utils/pipeline-timer.js";
import { ProgressTracker } from "./utils/progress-tracker.js";
import { verifyEnrichedFile } from "./utils/verify-enriched-file.js";
import { runCli } from "./utils/cli.js";
import type { PipelineConfig } from "./utils/cli.js";
import { LLM_CONFIG } from "./config/llm.js";
import { BATCH_CONFIG } from "./config/batch.js";
// Runtime config accessor for other modules
let RUNTIME_CONFIG: PipelineConfig;
export function getRuntimeConfig(): PipelineConfig {
return RUNTIME_CONFIG;
}
async function main() {
// ── Interactive CLI ──────────────────────────────────────────────────────
RUNTIME_CONFIG = await runCli();
// Populate shared config objects so existing imports keep working
LLM_CONFIG.provider = RUNTIME_CONFIG.provider;
LLM_CONFIG.url = RUNTIME_CONFIG.url;
LLM_CONFIG.model = RUNTIME_CONFIG.model;
BATCH_CONFIG.size = RUNTIME_CONFIG.batchSize;
BATCH_CONFIG.maxRetries = RUNTIME_CONFIG.maxRetries;
console.log("Starting data pipeline...\n");
console.log(`Provider: ${RUNTIME_CONFIG.provider}`);
console.log(`Model: ${RUNTIME_CONFIG.model ?? "(none)"}`);
console.log(`Batch: ${RUNTIME_CONFIG.batchSize} words/call\n`);
const timer = new PipelineTimer();