diff --git a/data-pipeline/pipeline.ts b/data-pipeline/pipeline.ts deleted file mode 100644 index cd2abc3..0000000 --- a/data-pipeline/pipeline.ts +++ /dev/null @@ -1,187 +0,0 @@ -import { isWordProcessed } from "./utils/check-if-json-exists.js"; -import { createBaseJson } from "./utils/create-base-json.js"; -import { ensureOutputFolders } from "./utils/create-output-dirs.js"; -import { scanSourceData } from "./utils/scanning-source-files.js"; -import { createLineReader } from "./utils/create-line-reader.js"; -import { checkLlmServer } from "./utils/check-llm-server.js"; -import { getWordFilePath } from "./utils/get-word-file-path.js"; -import { mergeEnrichedData } from "./utils/merge-enriched-data.js"; -import { enrichWordWithRetry } from "./utils/enrich-word.js"; -import { writeJsonFile } from "./utils/write-json-file.js"; -import { deleteFileIfExists } from "./utils/delete-file.js"; -import { PipelineTimer } from "./utils/pipeline-timer.js"; -import { ProgressTracker } from "./utils/progress-tracker.js"; -import { verifyEnrichedFile } from "./utils/verify-enriched-file.js"; -import { runCli } from "./utils/cli.js"; -import type { PipelineConfig } from "./utils/cli.js"; -import { LLM_CONFIG } from "./config/llm.js"; -import { BATCH_CONFIG } from "./config/batch.js"; - -// Runtime config accessor for other modules -let RUNTIME_CONFIG: PipelineConfig; - -export function getRuntimeConfig(): PipelineConfig { - return RUNTIME_CONFIG; -} - -async function main() { - // ── Interactive CLI ────────────────────────────────────────────────────── - RUNTIME_CONFIG = await runCli(); - - // Populate shared config objects so existing imports keep working - LLM_CONFIG.provider = RUNTIME_CONFIG.provider; - LLM_CONFIG.url = RUNTIME_CONFIG.url; - LLM_CONFIG.model = RUNTIME_CONFIG.model; - BATCH_CONFIG.size = RUNTIME_CONFIG.batchSize; - BATCH_CONFIG.maxRetries = RUNTIME_CONFIG.maxRetries; - - console.log("Starting data pipeline...\n"); - console.log(`Provider: ${RUNTIME_CONFIG.provider}`); - console.log(`Model: ${RUNTIME_CONFIG.model ?? "(none)"}`); - console.log(`Batch: ${RUNTIME_CONFIG.batchSize} words/call\n`); - - const timer = new PipelineTimer(); - - // step 1: scanning for source files - console.log("\n step 1: scanning the source files..."); - const wordlists = scanSourceData(import.meta.dirname); - - // step 2: ensuring output folders exist - console.log("\n step 2: creating necessary output folders..."); - ensureOutputFolders(wordlists); - - // step 3: check to verify the local AI engine is ready before touching anything - console.log("\n step 3: verifying local AI engine status..."); - try { - await checkLlmServer(); - } catch (error: unknown) { - const message = error instanceof Error ? error.message : String(error); - console.error(`\n ❌ ${message}`); - process.exit(1); - } - - // Step 4: Loop through the wordlists array - console.log("\n step 4: looping through the wordlists..."); - - for (const wordlist of wordlists) { - console.log( - `\nReading list: [${wordlist.language.toUpperCase()}] -> [${wordlist.pos.toUpperCase()}]`, - ); - - const rl = createLineReader(wordlist.sourcePath); - - // Collect words and count them - const words: string[] = []; - for await (const line of rl) { - const word = line.trim().toLowerCase(); - if (word) words.push(word); - } - - // Filter out already-processed words - const unprocessedWords = words.filter( - (word) => !isWordProcessed(word, wordlist.outputDir), - ); - - const skippedCount = words.length - unprocessedWords.length; - if (skippedCount > 0) { - console.log(` Skipped ${skippedCount} already-processed words`); - } - - const progress = new ProgressTracker(unprocessedWords.length); - - // Step 5: Process in batches - for (let i = 0; i < unprocessedWords.length; i += BATCH_CONFIG.size) { - const batch = unprocessedWords.slice(i, i + BATCH_CONFIG.size); - const batchNum = Math.floor(i / BATCH_CONFIG.size) + 1; - const totalBatches = Math.ceil( - unprocessedWords.length / BATCH_CONFIG.size, - ); - const batchLabel = `Batch ${batchNum}/${totalBatches}`; - - console.log(`\n ${batchLabel}: [${batch.join(", ")}]`); - - // Create skeletons for all words in batch - for (const word of batch) { - createBaseJson( - word, - wordlist.outputDir, - wordlist.language, - wordlist.pos, - ); - } - - timer.startWord(); - - try { - // Step 6: enrich batch with senses (with retry/split) - const result = await enrichWordWithRetry( - batch, - wordlist.language, - wordlist.pos, - ); - - // Step 7: write each word's result - for (const [word, senses] of result.results) { - progress.next(); - console.log( - ` ${progress.format(`Enriched and saved: ${word}.json`)}`, - ); - - const targetFilePath = getWordFilePath(word, wordlist.outputDir); - const enrichedData = mergeEnrichedData(word, senses); - - writeJsonFile(targetFilePath, enrichedData); - - // Verify the generated file - const verification = verifyEnrichedFile(targetFilePath); - if (!verification.valid) { - console.error(` Warning: Schema violations in ${word}.json:`); - for (const error of verification.errors) { - console.error(` - ${error}`); - } - } - - timer.recordProcessed({ - promptTokens: result.metrics.promptTokens, - completionTokens: result.metrics.completionTokens, - totalTokens: result.metrics.totalTokens, - promptTimeMs: result.metrics.promptTimeMs, - completionTimeMs: result.metrics.completionTimeMs, - totalTimeMs: result.metrics.totalTimeMs, - }); - } - - console.log(` ${timer.getWordTiming()}`); - // Show ETA every 5 batches or on the last batch - if (batchNum % 5 === 0 || batchNum === totalBatches) { - console.log(` 📊 ${timer.getEta(unprocessedWords.length)}`); - } - } catch (error: unknown) { - const errorMessage = - error instanceof Error ? error.message : String(error); - console.error( - ` Failed to enrich batch [${batch.join(", ")}]: ${errorMessage}`, - ); - - // Cleanup: delete any partially-written files for the failed batch - for (const word of batch) { - const targetFilePath = getWordFilePath(word, wordlist.outputDir); - deleteFileIfExists(targetFilePath); - console.log(` Removed incomplete file: ${word}.json`); - progress.recordFailed(); - } - - timer.recordFailed(); - } - } - } - - timer.stop(); - console.log("\n" + timer.getSummary()); - console.log("\nGlobal data pipeline run completed successfully."); -} - -// Fire the orchestrator block -main().catch((err) => { - console.error("Critical unexpected pipeline failure:", err); -}); diff --git a/data-pipeline/utils/check-if-json-exists.ts b/data-pipeline/utils/check-if-json-exists.ts deleted file mode 100644 index 00ab535..0000000 --- a/data-pipeline/utils/check-if-json-exists.ts +++ /dev/null @@ -1,23 +0,0 @@ -import fs from "fs"; -import path from "path"; - -/** - * Checks if a JSON file for the given word exists AND contains enriched data. - * Returns false for skeleton files (missing senses array). - */ -export function isWordProcessed(word: string, outputDir: string): boolean { - const targetFilePath = path.join(outputDir, `${word}.json`); - - if (!fs.existsSync(targetFilePath)) { - return false; - } - - try { - const content = fs.readFileSync(targetFilePath, "utf-8"); - const data = JSON.parse(content) as Record; - return Array.isArray(data["senses"]) && data["senses"].length > 0; - } catch (_error: unknown) { - // Corrupted file => treat as not processed - return false; - } -} diff --git a/data-pipeline/utils/create-base-json.ts b/data-pipeline/utils/create-base-json.ts deleted file mode 100644 index 6d7d057..0000000 --- a/data-pipeline/utils/create-base-json.ts +++ /dev/null @@ -1,27 +0,0 @@ -import fs from "fs"; -import path from "path"; -import { LANG_MAP, POS_MAP } from "../config/constants.js"; - -/** - * Creates the base JSON file with word, language, and pos. - * No logging — the orchestrator handles all console output. - */ -export function createBaseJson( - word: string, - outputDir: string, - rawLanguage: string, - rawPos: string, -): void { - const targetFilePath = path.join(outputDir, `${word}.json`); - - const dbLanguage = LANG_MAP[rawLanguage] || rawLanguage; - const dbPos = POS_MAP[rawPos] || rawPos; - - const initialData = { word, language: dbLanguage, pos: dbPos }; - - fs.writeFileSync( - targetFilePath, - JSON.stringify(initialData, null, 2), - "utf-8", - ); -} diff --git a/data-pipeline/utils/delete-file.ts b/data-pipeline/utils/delete-file.ts deleted file mode 100644 index 3c545ce..0000000 --- a/data-pipeline/utils/delete-file.ts +++ /dev/null @@ -1,10 +0,0 @@ -import fs from "fs"; - -/** - * Deletes a file if it exists. Silently ignores missing files. - */ -export function deleteFileIfExists(filePath: string): void { - if (fs.existsSync(filePath)) { - fs.unlinkSync(filePath); - } -} diff --git a/data-pipeline/utils/enrich-word.ts b/data-pipeline/utils/enrich-word.ts deleted file mode 100644 index f5536fb..0000000 --- a/data-pipeline/utils/enrich-word.ts +++ /dev/null @@ -1,284 +0,0 @@ -import { buildSystemPrompt } from "../config/prompt.js"; -import { createAdapter } from "./llm-adapters/factory.js"; -import { BATCH_CONFIG } from "../config/batch.js"; -import type { Language, Pos, EnrichedSense } from "./merge-enriched-data.js"; -import { LANG_MAP, POS_MAP, ALL_LANGUAGES } from "../config/constants.js"; - -interface LlmResponse { - content: string; - promptTokens: number; - completionTokens: number; - totalTokens: number; - promptTimeMs: number | null; - completionTimeMs: number | null; - totalTimeMs: number; -} - -export interface EnrichmentResult { - results: Map; - metrics: { - promptTokens: number; - completionTokens: number; - totalTokens: number; - promptTimeMs: number | null; - completionTimeMs: number | null; - totalTimeMs: number; - }; -} - -/** - * Calls the LLM with the enrichment prompt. - * Returns the response content and timing metrics. - */ -async function callLlm( - words: string[], - rawLanguage: string, - rawPos: string, -): Promise { - const adapter = createAdapter(); - const sourceCode = LANG_MAP[rawLanguage] || rawLanguage; - const targetLanguages = ALL_LANGUAGES.filter((lang) => lang !== sourceCode); - const prompt = buildSystemPrompt(rawLanguage, rawPos, targetLanguages); - return adapter.call(words, prompt); -} - -/** - * Strips markdown code blocks and extracts the JSON object from raw LLM output. - * Throws if no valid JSON object braces are found. - */ -function sanitizeLlmOutput(raw: string): string { - const cleaned = raw.replace(/```json\s*/g, "").replace(/```\s*$/g, ""); - const start = cleaned.indexOf("{"); - const end = cleaned.lastIndexOf("}"); - if (start === -1 || end === -1) { - throw new Error("No JSON object found in LLM output"); - } - return cleaned.slice(start, end + 1); -} - -function validateSense(item: unknown, word: string, index: number): void { - if (typeof item !== "object" || item === null || Array.isArray(item)) { - throw new Error(`Sense ${index} for "${word}" is not an object`); - } - - const sense = item as Record; - - if (typeof sense["sense"] !== "string" || !sense["sense"]) { - throw new Error(`Sense ${index} for "${word}": missing or invalid "sense"`); - } - if (typeof sense["example"] !== "string" || !sense["example"]) { - throw new Error( - `Sense ${index} for "${word}": missing or invalid "example"`, - ); - } - if ( - !["easy", "medium", "hard"].includes(sense["difficulty_level"] as string) - ) { - throw new Error(`Sense ${index} for "${word}": invalid "difficulty_level"`); - } - if ( - typeof sense["translations"] !== "object" || - sense["translations"] === null - ) { - throw new Error(`Sense ${index} for "${word}": missing "translations"`); - } - - const trans = sense["translations"] as Record; - for (const lang of ["de", "it", "es", "fr"]) { - if (!Array.isArray(trans[lang])) { - throw new Error( - `Sense ${index} for "${word}": missing or invalid "${lang}" translations`, - ); - } - for (let j = 0; j < (trans[lang] as unknown[]).length; j++) { - const t = (trans[lang] as unknown[])[j] as Record; - if (typeof t["word"] !== "string" || !t["word"]) { - throw new Error( - `Sense ${index} for "${word}": ${lang}[${j}] missing "word"`, - ); - } - if ( - !["masculine", "feminine", "neuter", null].includes( - t["gender"] as string | null, - ) - ) { - throw new Error( - `Sense ${index} for "${word}": ${lang}[${j}] invalid "gender"`, - ); - } - } - } -} - -/** - * Parses the LLM response string into a JavaScript object. - * Throws if the response is not valid JSON or not an object with expected keys. - */ -export function parseLlmResponse( - rawJson: string, - expectedWords: string[], -): Record { - let parsed: unknown; - - try { - const sanitized = sanitizeLlmOutput(rawJson); - parsed = JSON.parse(sanitized); - } catch (error: unknown) { - throw new Error(`Failed to parse LLM output as JSON: ${rawJson}`, { - cause: error, - }); - } - - if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) { - throw new Error("LLM output is not a JSON object"); - } - - const obj = parsed as Record; - - for (const word of expectedWords) { - if (!(word in obj)) { - throw new Error(`Missing key in LLM output: "${word}"`); - } - if (!Array.isArray(obj[word]) || (obj[word] as unknown[]).length === 0) { - throw new Error(`LLM output for "${word}" is not a non-empty array`); - } - - // Validate each sense in the array - const senses = obj[word] as unknown[]; - for (let i = 0; i < senses.length; i++) { - validateSense(senses[i], word, i); - } - } - - return obj; -} - -/** - * Takes parsed LLM output and builds final enriched objects with composite IDs. - */ -export function buildEnrichedData( - parsed: Record, - rawLanguage: string, - rawPos: string, -): Map { - const language = (LANG_MAP[rawLanguage] || rawLanguage) as Language; - const pos = (POS_MAP[rawPos] || rawPos) as Pos; - const results = new Map(); - - for (const [word, sensesArray] of Object.entries(parsed)) { - const senses = (sensesArray as unknown[]).map((item, index) => { - const sense = item as Omit< - EnrichedSense, - "id" | "word" | "language" | "pos" - >; - - return { - id: `${word}:${language}:${pos}:${index}`, - word, - language, - pos, - ...sense, - } as EnrichedSense; - }); - - results.set(word, senses); - } - - return results; -} - -/** - * Enriches a batch of words by calling the LLM, parsing the response, and building final data. - */ -export async function enrichWord( - words: string[], - rawLanguage: string, - rawPos: string, -): Promise { - const llmResponse = await callLlm(words, rawLanguage, rawPos); - const parsed = parseLlmResponse(llmResponse.content, words); - const results = buildEnrichedData(parsed, rawLanguage, rawPos); - - return { - results, - metrics: { - promptTokens: llmResponse.promptTokens, - completionTokens: llmResponse.completionTokens, - totalTokens: llmResponse.totalTokens, - promptTimeMs: llmResponse.promptTimeMs, - completionTimeMs: llmResponse.completionTimeMs, - totalTimeMs: llmResponse.totalTimeMs, - }, - }; -} - -/** - * Enriches a batch of words with retry and split-on-failure logic. - * Retries up to BATCH_CONFIG.maxRetries times, then splits batch in half and retries each half. - * Continues splitting until batch size is 1, then throws if still failing. - */ -export async function enrichWordWithRetry( - words: string[], - rawLanguage: string, - rawPos: string, - attempt: number = 1, -): Promise { - try { - return await enrichWord(words, rawLanguage, rawPos); - } catch (error: unknown) { - const errorMessage = error instanceof Error ? error.message : String(error); - - if (words.length === 1) { - throw new Error( - `Failed to enrich word "${words[0]}" after ${attempt} attempts: ${errorMessage}`, - { cause: error }, - ); - } - - if (attempt < BATCH_CONFIG.maxRetries) { - console.log( - ` Retry ${attempt}/${BATCH_CONFIG.maxRetries} for batch [${words.join(", ")}]: ${errorMessage}`, - ); - return enrichWordWithRetry(words, rawLanguage, rawPos, attempt + 1); - } - - // Max retries reached — split and retry - console.log( - ` Splitting batch [${words.join(", ")}] after ${BATCH_CONFIG.maxRetries} failed attempts`, - ); - - const half = Math.ceil(words.length / 2); - const left = words.slice(0, half); - const right = words.slice(half); - - const leftResult = await enrichWordWithRetry(left, rawLanguage, rawPos, 1); - const rightResult = await enrichWordWithRetry( - right, - rawLanguage, - rawPos, - 1, - ); - - // Merge results - const merged = new Map([...leftResult.results, ...rightResult.results]); - const mergedMetrics = { - promptTokens: - leftResult.metrics.promptTokens + rightResult.metrics.promptTokens, - completionTokens: - leftResult.metrics.completionTokens + - rightResult.metrics.completionTokens, - totalTokens: - leftResult.metrics.totalTokens + rightResult.metrics.totalTokens, - promptTimeMs: - (leftResult.metrics.promptTimeMs ?? 0) + - (rightResult.metrics.promptTimeMs ?? 0), - completionTimeMs: - (leftResult.metrics.completionTimeMs ?? 0) + - (rightResult.metrics.completionTimeMs ?? 0), - totalTimeMs: - leftResult.metrics.totalTimeMs + rightResult.metrics.totalTimeMs, - }; - - return { results: merged, metrics: mergedMetrics }; - } -} diff --git a/data-pipeline/utils/llm-adapters/gemini.ts b/data-pipeline/utils/llm-adapters/gemini.ts deleted file mode 100644 index 39008ca..0000000 --- a/data-pipeline/utils/llm-adapters/gemini.ts +++ /dev/null @@ -1,81 +0,0 @@ -import type { LlmAdapter } from "./types.js"; - -interface GeminiResponse { - candidates: Array<{ content: { parts: Array<{ text: string }> } }>; - usageMetadata: { - promptTokenCount: number; - candidatesTokenCount: number; - totalTokenCount: number; - }; -} - -export class GeminiAdapter implements LlmAdapter { - private apiKey: string; - private model: string; - - constructor(apiKey: string, model: string) { - this.apiKey = apiKey; - this.model = model; - } - - async call( - words: string[], - systemPrompt: string, - ): Promise<{ - content: string; - promptTokens: number; - completionTokens: number; - totalTokens: number; - promptTimeMs: number | null; - completionTimeMs: number | null; - totalTimeMs: number; - }> { - const url = `https://generativelanguage.googleapis.com/v1beta/models/${this.model}:generateContent?key=${this.apiKey}`; - - const payload = { - systemInstruction: { parts: [{ text: systemPrompt }] }, - contents: [ - { role: "user", parts: [{ text: "Words: " + JSON.stringify(words) }] }, - ], - generationConfig: { - temperature: 0.1, - topP: 0.9, - maxOutputTokens: Math.ceil(words.length * 250 * 1.2), - }, - }; - - const startTime = Date.now(); - - const response = await fetch(url, { - method: "POST", - headers: { "Content-Type": "application/json" }, - body: JSON.stringify(payload), - }); - - const totalTimeMs = Date.now() - startTime; - - if (!response.ok) { - throw new Error(`Gemini API responded with status: ${response.status}`); - } - - const json = (await response.json()) as GeminiResponse; - - const content = json.candidates[0]?.content?.parts[0]?.text; - if (!content) { - throw new Error("Gemini response content is empty"); - } - - const promptTokens = json.usageMetadata.promptTokenCount; - const completionTokens = json.usageMetadata.candidatesTokenCount; - - return { - content, - promptTokens, - completionTokens, - totalTokens: json.usageMetadata.totalTokenCount, - promptTimeMs: null, - completionTimeMs: null, - totalTimeMs, - }; - } -} diff --git a/data-pipeline/utils/llm-adapters/types.ts b/data-pipeline/utils/llm-adapters/types.ts deleted file mode 100644 index c73c0b4..0000000 --- a/data-pipeline/utils/llm-adapters/types.ts +++ /dev/null @@ -1,14 +0,0 @@ -export interface LlmAdapter { - call( - words: string[], - systemPrompt: string, - ): Promise<{ - content: string; - promptTokens: number; - completionTokens: number; - totalTokens: number; - promptTimeMs: number | null; - completionTimeMs: number | null; - totalTimeMs: number; - }>; -} diff --git a/data-pipeline/utils/merge-enriched-data.ts b/data-pipeline/utils/merge-enriched-data.ts deleted file mode 100644 index ad722a4..0000000 --- a/data-pipeline/utils/merge-enriched-data.ts +++ /dev/null @@ -1,44 +0,0 @@ -import { LLM_CONFIG } from "../config/llm.js"; - -export type Language = "en" | "de" | "it" | "es" | "fr"; -export type Pos = "noun" | "verb" | "adjective" | "adverb"; -export type Gender = "masculine" | "feminine" | "neuter" | null; -export type Difficulty = "easy" | "medium" | "hard"; - -export interface Translation { - word: string; - gender: Gender; -} - -export interface EnrichedSense { - id: string; - word: string; - language: Language; - pos: Pos; - sense: string; - example: string; - difficulty_level: Difficulty; - translations: { - de: Translation[]; - it: Translation[]; - es: Translation[]; - fr: Translation[]; - }; -} - -/** - * Merges skeleton data with enriched LLM senses into the final pipeline output. - */ -export function mergeEnrichedData( - word: string, - senses: EnrichedSense[], -): Record { - return { - word, - language: senses[0]?.language ?? "en", - pos: senses[0]?.pos ?? "noun", - senses, - enrichedAt: new Date().toISOString(), - model: LLM_CONFIG.model ?? "unknown", - }; -} diff --git a/data-pipeline/utils/pipeline-timer.ts b/data-pipeline/utils/pipeline-timer.ts deleted file mode 100644 index 8ba2d0d..0000000 --- a/data-pipeline/utils/pipeline-timer.ts +++ /dev/null @@ -1,198 +0,0 @@ -interface LlmMetrics { - promptTokens: number; - completionTokens: number; - totalTokens: number; - promptTimeMs: number | null; - completionTimeMs: number | null; - totalTimeMs: number; -} - -interface PipelineMetrics { - startTime: Date; - endTime?: Date; - wordsProcessed: number; - wordsSkipped: number; - wordsFailed: number; - llmCalls: number; - totalPromptTokens: number; - totalCompletionTokens: number; - totalTokens: number; - totalPromptTimeMs: number; - totalCompletionTimeMs: number; - totalTimeMs: number; - currentWordStartTime?: Date; -} - -/** - * Simple timer and metrics tracker for the pipeline. - * Tracks both pipeline throughput and LLM performance. - */ -export class PipelineTimer { - private metrics: PipelineMetrics; - - constructor() { - this.metrics = { - startTime: new Date(), - wordsProcessed: 0, - wordsSkipped: 0, - wordsFailed: 0, - llmCalls: 0, - totalPromptTokens: 0, - totalCompletionTokens: 0, - totalTokens: 0, - totalPromptTimeMs: 0, - totalCompletionTimeMs: 0, - totalTimeMs: 0, - }; - } - - startWord(): void { - this.metrics.currentWordStartTime = new Date(); - } - - getWordDurationMs(): number { - if (!this.metrics.currentWordStartTime) return 0; - return new Date().getTime() - this.metrics.currentWordStartTime.getTime(); - } - - recordProcessed(llmMetrics?: LlmMetrics): void { - this.metrics.wordsProcessed++; - if (llmMetrics) { - this.metrics.llmCalls++; - this.metrics.totalPromptTokens += llmMetrics.promptTokens; - this.metrics.totalCompletionTokens += llmMetrics.completionTokens; - this.metrics.totalTokens += llmMetrics.totalTokens; - if (llmMetrics.promptTimeMs !== null) { - this.metrics.totalPromptTimeMs += llmMetrics.promptTimeMs; - } - if (llmMetrics.completionTimeMs !== null) { - this.metrics.totalCompletionTimeMs += llmMetrics.completionTimeMs; - } - this.metrics.totalTimeMs += llmMetrics.totalTimeMs; - } - } - - recordSkipped(): void { - this.metrics.wordsSkipped++; - } - - recordFailed(): void { - this.metrics.wordsFailed++; - } - - stop(): void { - this.metrics.endTime = new Date(); - } - - getWordTiming(): string { - const durationMs = this.getWordDurationMs(); - const durationSec = (durationMs / 1000).toFixed(1); - return `⏱️ Word took ${durationSec}s`; - } - - getEta(totalWords: number): string { - const processed = this.metrics.wordsProcessed; - const remaining = totalWords - processed - this.metrics.wordsSkipped; - - if (processed === 0 || remaining <= 0) return "ETA: calculating..."; - - const elapsedMs = new Date().getTime() - this.metrics.startTime.getTime(); - const avgMsPerWord = elapsedMs / processed; - const etaMs = avgMsPerWord * remaining; - - const etaMin = Math.round(etaMs / 60000); - const etaHour = (etaMs / 3600000).toFixed(1); - - if (etaMin < 60) { - return `ETA: ${etaMin} min`; - } - return `ETA: ${etaHour} hours`; - } - - getSummary(): string { - const end = this.metrics.endTime || new Date(); - const durationMs = end.getTime() - this.metrics.startTime.getTime(); - const durationSec = (durationMs / 1000).toFixed(1); - - const total = - this.metrics.wordsProcessed + - this.metrics.wordsSkipped + - this.metrics.wordsFailed; - const throughput = - this.metrics.wordsProcessed > 0 - ? (this.metrics.wordsProcessed / (durationMs / 1000)).toFixed(2) - : "0"; - - const avgPromptTokens = - this.metrics.llmCalls > 0 - ? (this.metrics.totalPromptTokens / this.metrics.llmCalls).toFixed(0) - : "0"; - const avgCompletionTokens = - this.metrics.llmCalls > 0 - ? (this.metrics.totalCompletionTokens / this.metrics.llmCalls).toFixed( - 0, - ) - : "0"; - const avgTotalTimeMs = - this.metrics.llmCalls > 0 - ? (this.metrics.totalTimeMs / this.metrics.llmCalls).toFixed(0) - : "0"; - - const unifiedThroughput = - this.metrics.totalTimeMs > 0 - ? ( - this.metrics.totalTokens / - (this.metrics.totalTimeMs / 1000) - ).toFixed(1) - : "N/A"; - - const hasDetailedTimings = - this.metrics.totalPromptTimeMs > 0 || - this.metrics.totalCompletionTimeMs > 0; - - const avgPromptSpeed = - this.metrics.totalPromptTimeMs > 0 - ? ( - this.metrics.totalPromptTokens / - (this.metrics.totalPromptTimeMs / 1000) - ).toFixed(1) - : "N/A"; - - const avgCompletionSpeed = - this.metrics.totalCompletionTimeMs > 0 - ? ( - this.metrics.totalCompletionTokens / - (this.metrics.totalCompletionTimeMs / 1000) - ).toFixed(1) - : "N/A"; - - const lines = [ - `⏱️ Pipeline Summary`, - ` Duration: ${durationSec}s`, - ` Processed: ${this.metrics.wordsProcessed}`, - ` Skipped: ${this.metrics.wordsSkipped}`, - ` Failed: ${this.metrics.wordsFailed}`, - ` Total: ${total}`, - ` Throughput: ${throughput} words/sec`, - ``, - `🤖 LLM Metrics`, - ` Calls: ${this.metrics.llmCalls}`, - ` Avg prompt tokens: ${avgPromptTokens}`, - ` Avg completion tokens: ${avgCompletionTokens}`, - ` Avg total tokens: ${avgPromptTokens + avgCompletionTokens}`, - ` Avg total request time: ${avgTotalTimeMs}ms`, - ` Avg throughput: ${unifiedThroughput} tok/s`, - ]; - - if (hasDetailedTimings) { - lines.push( - ``, - ` [Local breakdown]`, - ` Avg prompt speed: ${avgPromptSpeed} tok/s`, - ` Avg completion speed: ${avgCompletionSpeed} tok/s`, - ); - } - - return lines.join("\n"); - } -} diff --git a/data-pipeline/utils/progress-tracker.ts b/data-pipeline/utils/progress-tracker.ts deleted file mode 100644 index d232918..0000000 --- a/data-pipeline/utils/progress-tracker.ts +++ /dev/null @@ -1,27 +0,0 @@ -/** - * Simple progress tracker for pipeline execution. - */ -export class ProgressTracker { - private current: number; - private failed: number; - private total: number; - - constructor(total: number) { - this.current = 0; - this.failed = 0; - this.total = total; - } - - next(): number { - this.current++; - return this.current; - } - - recordFailed(): void { - this.failed++; - } - - format(label: string): string { - return `[${this.current}/${this.total}] (${this.failed} failed) ${label}`; - } -} diff --git a/data-pipeline/utils/verify-enriched-file.ts b/data-pipeline/utils/verify-enriched-file.ts deleted file mode 100644 index bbe0560..0000000 --- a/data-pipeline/utils/verify-enriched-file.ts +++ /dev/null @@ -1,96 +0,0 @@ -import fs from "fs"; - -interface VerificationResult { - valid: boolean; - errors: string[]; -} - -/** - * Verifies that an enriched JSON file matches the expected schema. - * Returns detailed error messages for any violations. - */ -export function verifyEnrichedFile(filePath: string): VerificationResult { - const errors: string[] = []; - - if (!fs.existsSync(filePath)) { - return { valid: false, errors: ["File does not exist"] }; - } - - let data: unknown; - try { - data = JSON.parse(fs.readFileSync(filePath, "utf-8")); - } catch (_error: unknown) { - return { valid: false, errors: ["Invalid JSON syntax"] }; - } - - if (typeof data !== "object" || data === null || Array.isArray(data)) { - return { valid: false, errors: ["Root must be an object"] }; - } - - const obj = data as Record; - - // Required top-level fields - const requiredFields = ["word", "language", "pos", "senses"]; - for (const field of requiredFields) { - if (!(field in obj)) { - errors.push(`Missing required field: "${field}"`); - } - } - - // Validate senses array - if (!Array.isArray(obj["senses"])) { - errors.push('"senses" must be an array'); - } else if (obj["senses"].length === 0) { - errors.push('"senses" array cannot be empty'); - } else { - for (let i = 0; i < obj["senses"].length; i++) { - const sense = obj["senses"][i] as Record; - const sensePrefix = `senses[${i}]`; - - if (!sense["sense"] || typeof sense["sense"] !== "string") { - errors.push(`${sensePrefix}: missing or invalid "sense"`); - } - if (!sense["example"] || typeof sense["example"] !== "string") { - errors.push(`${sensePrefix}: missing or invalid "example"`); - } - if ( - !["easy", "medium", "hard"].includes( - sense["difficulty_level"] as string, - ) - ) { - errors.push(`${sensePrefix}: invalid "difficulty_level"`); - } - if (!sense["translations"] || typeof sense["translations"] !== "object") { - errors.push(`${sensePrefix}: missing "translations"`); - } else { - const trans = sense["translations"] as Record; - for (const lang of ["de", "it", "es", "fr"]) { - if (!Array.isArray(trans[lang])) { - errors.push( - `${sensePrefix}: missing or invalid "${lang}" translations`, - ); - } else { - for (let j = 0; j < (trans[lang] as unknown[]).length; j++) { - const t = (trans[lang] as unknown[])[j] as Record< - string, - unknown - >; - if (!t["word"] || typeof t["word"] !== "string") { - errors.push(`${sensePrefix}.${lang}[${j}]: missing "word"`); - } - if ( - !["masculine", "feminine", "neuter", null].includes( - t["gender"] as string | null, - ) - ) { - errors.push(`${sensePrefix}.${lang}[${j}]: invalid "gender"`); - } - } - } - } - } - } - } - - return { valid: errors.length === 0, errors }; -}