refactor: gut old pipeline architecture for complete rewrite
This commit is contained in:
lila
parent
0ae3b9f686
commit
597083e1fd
11 changed files with 0 additions and 991 deletions
|
|
@ -1,187 +0,0 @@
|
|||
import { isWordProcessed } from "./utils/check-if-json-exists.js";
|
||||
import { createBaseJson } from "./utils/create-base-json.js";
|
||||
import { ensureOutputFolders } from "./utils/create-output-dirs.js";
|
||||
import { scanSourceData } from "./utils/scanning-source-files.js";
|
||||
import { createLineReader } from "./utils/create-line-reader.js";
|
||||
import { checkLlmServer } from "./utils/check-llm-server.js";
|
||||
import { getWordFilePath } from "./utils/get-word-file-path.js";
|
||||
import { mergeEnrichedData } from "./utils/merge-enriched-data.js";
|
||||
import { enrichWordWithRetry } from "./utils/enrich-word.js";
|
||||
import { writeJsonFile } from "./utils/write-json-file.js";
|
||||
import { deleteFileIfExists } from "./utils/delete-file.js";
|
||||
import { PipelineTimer } from "./utils/pipeline-timer.js";
|
||||
import { ProgressTracker } from "./utils/progress-tracker.js";
|
||||
import { verifyEnrichedFile } from "./utils/verify-enriched-file.js";
|
||||
import { runCli } from "./utils/cli.js";
|
||||
import type { PipelineConfig } from "./utils/cli.js";
|
||||
import { LLM_CONFIG } from "./config/llm.js";
|
||||
import { BATCH_CONFIG } from "./config/batch.js";
|
||||
|
||||
// Runtime config accessor for other modules
|
||||
let RUNTIME_CONFIG: PipelineConfig;
|
||||
|
||||
export function getRuntimeConfig(): PipelineConfig {
|
||||
return RUNTIME_CONFIG;
|
||||
}
|
||||
|
||||
async function main() {
|
||||
// ── Interactive CLI ──────────────────────────────────────────────────────
|
||||
RUNTIME_CONFIG = await runCli();
|
||||
|
||||
// Populate shared config objects so existing imports keep working
|
||||
LLM_CONFIG.provider = RUNTIME_CONFIG.provider;
|
||||
LLM_CONFIG.url = RUNTIME_CONFIG.url;
|
||||
LLM_CONFIG.model = RUNTIME_CONFIG.model;
|
||||
BATCH_CONFIG.size = RUNTIME_CONFIG.batchSize;
|
||||
BATCH_CONFIG.maxRetries = RUNTIME_CONFIG.maxRetries;
|
||||
|
||||
console.log("Starting data pipeline...\n");
|
||||
console.log(`Provider: ${RUNTIME_CONFIG.provider}`);
|
||||
console.log(`Model: ${RUNTIME_CONFIG.model ?? "(none)"}`);
|
||||
console.log(`Batch: ${RUNTIME_CONFIG.batchSize} words/call\n`);
|
||||
|
||||
const timer = new PipelineTimer();
|
||||
|
||||
// step 1: scanning for source files
|
||||
console.log("\n step 1: scanning the source files...");
|
||||
const wordlists = scanSourceData(import.meta.dirname);
|
||||
|
||||
// step 2: ensuring output folders exist
|
||||
console.log("\n step 2: creating necessary output folders...");
|
||||
ensureOutputFolders(wordlists);
|
||||
|
||||
// step 3: check to verify the local AI engine is ready before touching anything
|
||||
console.log("\n step 3: verifying local AI engine status...");
|
||||
try {
|
||||
await checkLlmServer();
|
||||
} catch (error: unknown) {
|
||||
const message = error instanceof Error ? error.message : String(error);
|
||||
console.error(`\n ❌ ${message}`);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
// Step 4: Loop through the wordlists array
|
||||
console.log("\n step 4: looping through the wordlists...");
|
||||
|
||||
for (const wordlist of wordlists) {
|
||||
console.log(
|
||||
`\nReading list: [${wordlist.language.toUpperCase()}] -> [${wordlist.pos.toUpperCase()}]`,
|
||||
);
|
||||
|
||||
const rl = createLineReader(wordlist.sourcePath);
|
||||
|
||||
// Collect words and count them
|
||||
const words: string[] = [];
|
||||
for await (const line of rl) {
|
||||
const word = line.trim().toLowerCase();
|
||||
if (word) words.push(word);
|
||||
}
|
||||
|
||||
// Filter out already-processed words
|
||||
const unprocessedWords = words.filter(
|
||||
(word) => !isWordProcessed(word, wordlist.outputDir),
|
||||
);
|
||||
|
||||
const skippedCount = words.length - unprocessedWords.length;
|
||||
if (skippedCount > 0) {
|
||||
console.log(` Skipped ${skippedCount} already-processed words`);
|
||||
}
|
||||
|
||||
const progress = new ProgressTracker(unprocessedWords.length);
|
||||
|
||||
// Step 5: Process in batches
|
||||
for (let i = 0; i < unprocessedWords.length; i += BATCH_CONFIG.size) {
|
||||
const batch = unprocessedWords.slice(i, i + BATCH_CONFIG.size);
|
||||
const batchNum = Math.floor(i / BATCH_CONFIG.size) + 1;
|
||||
const totalBatches = Math.ceil(
|
||||
unprocessedWords.length / BATCH_CONFIG.size,
|
||||
);
|
||||
const batchLabel = `Batch ${batchNum}/${totalBatches}`;
|
||||
|
||||
console.log(`\n ${batchLabel}: [${batch.join(", ")}]`);
|
||||
|
||||
// Create skeletons for all words in batch
|
||||
for (const word of batch) {
|
||||
createBaseJson(
|
||||
word,
|
||||
wordlist.outputDir,
|
||||
wordlist.language,
|
||||
wordlist.pos,
|
||||
);
|
||||
}
|
||||
|
||||
timer.startWord();
|
||||
|
||||
try {
|
||||
// Step 6: enrich batch with senses (with retry/split)
|
||||
const result = await enrichWordWithRetry(
|
||||
batch,
|
||||
wordlist.language,
|
||||
wordlist.pos,
|
||||
);
|
||||
|
||||
// Step 7: write each word's result
|
||||
for (const [word, senses] of result.results) {
|
||||
progress.next();
|
||||
console.log(
|
||||
` ${progress.format(`Enriched and saved: ${word}.json`)}`,
|
||||
);
|
||||
|
||||
const targetFilePath = getWordFilePath(word, wordlist.outputDir);
|
||||
const enrichedData = mergeEnrichedData(word, senses);
|
||||
|
||||
writeJsonFile(targetFilePath, enrichedData);
|
||||
|
||||
// Verify the generated file
|
||||
const verification = verifyEnrichedFile(targetFilePath);
|
||||
if (!verification.valid) {
|
||||
console.error(` Warning: Schema violations in ${word}.json:`);
|
||||
for (const error of verification.errors) {
|
||||
console.error(` - ${error}`);
|
||||
}
|
||||
}
|
||||
|
||||
timer.recordProcessed({
|
||||
promptTokens: result.metrics.promptTokens,
|
||||
completionTokens: result.metrics.completionTokens,
|
||||
totalTokens: result.metrics.totalTokens,
|
||||
promptTimeMs: result.metrics.promptTimeMs,
|
||||
completionTimeMs: result.metrics.completionTimeMs,
|
||||
totalTimeMs: result.metrics.totalTimeMs,
|
||||
});
|
||||
}
|
||||
|
||||
console.log(` ${timer.getWordTiming()}`);
|
||||
// Show ETA every 5 batches or on the last batch
|
||||
if (batchNum % 5 === 0 || batchNum === totalBatches) {
|
||||
console.log(` 📊 ${timer.getEta(unprocessedWords.length)}`);
|
||||
}
|
||||
} catch (error: unknown) {
|
||||
const errorMessage =
|
||||
error instanceof Error ? error.message : String(error);
|
||||
console.error(
|
||||
` Failed to enrich batch [${batch.join(", ")}]: ${errorMessage}`,
|
||||
);
|
||||
|
||||
// Cleanup: delete any partially-written files for the failed batch
|
||||
for (const word of batch) {
|
||||
const targetFilePath = getWordFilePath(word, wordlist.outputDir);
|
||||
deleteFileIfExists(targetFilePath);
|
||||
console.log(` Removed incomplete file: ${word}.json`);
|
||||
progress.recordFailed();
|
||||
}
|
||||
|
||||
timer.recordFailed();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
timer.stop();
|
||||
console.log("\n" + timer.getSummary());
|
||||
console.log("\nGlobal data pipeline run completed successfully.");
|
||||
}
|
||||
|
||||
// Fire the orchestrator block
|
||||
main().catch((err) => {
|
||||
console.error("Critical unexpected pipeline failure:", err);
|
||||
});
|
||||
|
|
@ -1,23 +0,0 @@
|
|||
import fs from "fs";
|
||||
import path from "path";
|
||||
|
||||
/**
|
||||
* Checks if a JSON file for the given word exists AND contains enriched data.
|
||||
* Returns false for skeleton files (missing senses array).
|
||||
*/
|
||||
export function isWordProcessed(word: string, outputDir: string): boolean {
|
||||
const targetFilePath = path.join(outputDir, `${word}.json`);
|
||||
|
||||
if (!fs.existsSync(targetFilePath)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
try {
|
||||
const content = fs.readFileSync(targetFilePath, "utf-8");
|
||||
const data = JSON.parse(content) as Record<string, unknown>;
|
||||
return Array.isArray(data["senses"]) && data["senses"].length > 0;
|
||||
} catch (_error: unknown) {
|
||||
// Corrupted file => treat as not processed
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
|
@ -1,27 +0,0 @@
|
|||
import fs from "fs";
|
||||
import path from "path";
|
||||
import { LANG_MAP, POS_MAP } from "../config/constants.js";
|
||||
|
||||
/**
|
||||
* Creates the base JSON file with word, language, and pos.
|
||||
* No logging — the orchestrator handles all console output.
|
||||
*/
|
||||
export function createBaseJson(
|
||||
word: string,
|
||||
outputDir: string,
|
||||
rawLanguage: string,
|
||||
rawPos: string,
|
||||
): void {
|
||||
const targetFilePath = path.join(outputDir, `${word}.json`);
|
||||
|
||||
const dbLanguage = LANG_MAP[rawLanguage] || rawLanguage;
|
||||
const dbPos = POS_MAP[rawPos] || rawPos;
|
||||
|
||||
const initialData = { word, language: dbLanguage, pos: dbPos };
|
||||
|
||||
fs.writeFileSync(
|
||||
targetFilePath,
|
||||
JSON.stringify(initialData, null, 2),
|
||||
"utf-8",
|
||||
);
|
||||
}
|
||||
|
|
@ -1,10 +0,0 @@
|
|||
import fs from "fs";
|
||||
|
||||
/**
|
||||
* Deletes a file if it exists. Silently ignores missing files.
|
||||
*/
|
||||
export function deleteFileIfExists(filePath: string): void {
|
||||
if (fs.existsSync(filePath)) {
|
||||
fs.unlinkSync(filePath);
|
||||
}
|
||||
}
|
||||
|
|
@ -1,284 +0,0 @@
|
|||
import { buildSystemPrompt } from "../config/prompt.js";
|
||||
import { createAdapter } from "./llm-adapters/factory.js";
|
||||
import { BATCH_CONFIG } from "../config/batch.js";
|
||||
import type { Language, Pos, EnrichedSense } from "./merge-enriched-data.js";
|
||||
import { LANG_MAP, POS_MAP, ALL_LANGUAGES } from "../config/constants.js";
|
||||
|
||||
interface LlmResponse {
|
||||
content: string;
|
||||
promptTokens: number;
|
||||
completionTokens: number;
|
||||
totalTokens: number;
|
||||
promptTimeMs: number | null;
|
||||
completionTimeMs: number | null;
|
||||
totalTimeMs: number;
|
||||
}
|
||||
|
||||
export interface EnrichmentResult {
|
||||
results: Map<string, EnrichedSense[]>;
|
||||
metrics: {
|
||||
promptTokens: number;
|
||||
completionTokens: number;
|
||||
totalTokens: number;
|
||||
promptTimeMs: number | null;
|
||||
completionTimeMs: number | null;
|
||||
totalTimeMs: number;
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Calls the LLM with the enrichment prompt.
|
||||
* Returns the response content and timing metrics.
|
||||
*/
|
||||
async function callLlm(
|
||||
words: string[],
|
||||
rawLanguage: string,
|
||||
rawPos: string,
|
||||
): Promise<LlmResponse> {
|
||||
const adapter = createAdapter();
|
||||
const sourceCode = LANG_MAP[rawLanguage] || rawLanguage;
|
||||
const targetLanguages = ALL_LANGUAGES.filter((lang) => lang !== sourceCode);
|
||||
const prompt = buildSystemPrompt(rawLanguage, rawPos, targetLanguages);
|
||||
return adapter.call(words, prompt);
|
||||
}
|
||||
|
||||
/**
|
||||
* Strips markdown code blocks and extracts the JSON object from raw LLM output.
|
||||
* Throws if no valid JSON object braces are found.
|
||||
*/
|
||||
function sanitizeLlmOutput(raw: string): string {
|
||||
const cleaned = raw.replace(/```json\s*/g, "").replace(/```\s*$/g, "");
|
||||
const start = cleaned.indexOf("{");
|
||||
const end = cleaned.lastIndexOf("}");
|
||||
if (start === -1 || end === -1) {
|
||||
throw new Error("No JSON object found in LLM output");
|
||||
}
|
||||
return cleaned.slice(start, end + 1);
|
||||
}
|
||||
|
||||
function validateSense(item: unknown, word: string, index: number): void {
|
||||
if (typeof item !== "object" || item === null || Array.isArray(item)) {
|
||||
throw new Error(`Sense ${index} for "${word}" is not an object`);
|
||||
}
|
||||
|
||||
const sense = item as Record<string, unknown>;
|
||||
|
||||
if (typeof sense["sense"] !== "string" || !sense["sense"]) {
|
||||
throw new Error(`Sense ${index} for "${word}": missing or invalid "sense"`);
|
||||
}
|
||||
if (typeof sense["example"] !== "string" || !sense["example"]) {
|
||||
throw new Error(
|
||||
`Sense ${index} for "${word}": missing or invalid "example"`,
|
||||
);
|
||||
}
|
||||
if (
|
||||
!["easy", "medium", "hard"].includes(sense["difficulty_level"] as string)
|
||||
) {
|
||||
throw new Error(`Sense ${index} for "${word}": invalid "difficulty_level"`);
|
||||
}
|
||||
if (
|
||||
typeof sense["translations"] !== "object" ||
|
||||
sense["translations"] === null
|
||||
) {
|
||||
throw new Error(`Sense ${index} for "${word}": missing "translations"`);
|
||||
}
|
||||
|
||||
const trans = sense["translations"] as Record<string, unknown>;
|
||||
for (const lang of ["de", "it", "es", "fr"]) {
|
||||
if (!Array.isArray(trans[lang])) {
|
||||
throw new Error(
|
||||
`Sense ${index} for "${word}": missing or invalid "${lang}" translations`,
|
||||
);
|
||||
}
|
||||
for (let j = 0; j < (trans[lang] as unknown[]).length; j++) {
|
||||
const t = (trans[lang] as unknown[])[j] as Record<string, unknown>;
|
||||
if (typeof t["word"] !== "string" || !t["word"]) {
|
||||
throw new Error(
|
||||
`Sense ${index} for "${word}": ${lang}[${j}] missing "word"`,
|
||||
);
|
||||
}
|
||||
if (
|
||||
!["masculine", "feminine", "neuter", null].includes(
|
||||
t["gender"] as string | null,
|
||||
)
|
||||
) {
|
||||
throw new Error(
|
||||
`Sense ${index} for "${word}": ${lang}[${j}] invalid "gender"`,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Parses the LLM response string into a JavaScript object.
|
||||
* Throws if the response is not valid JSON or not an object with expected keys.
|
||||
*/
|
||||
export function parseLlmResponse(
|
||||
rawJson: string,
|
||||
expectedWords: string[],
|
||||
): Record<string, unknown> {
|
||||
let parsed: unknown;
|
||||
|
||||
try {
|
||||
const sanitized = sanitizeLlmOutput(rawJson);
|
||||
parsed = JSON.parse(sanitized);
|
||||
} catch (error: unknown) {
|
||||
throw new Error(`Failed to parse LLM output as JSON: ${rawJson}`, {
|
||||
cause: error,
|
||||
});
|
||||
}
|
||||
|
||||
if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) {
|
||||
throw new Error("LLM output is not a JSON object");
|
||||
}
|
||||
|
||||
const obj = parsed as Record<string, unknown>;
|
||||
|
||||
for (const word of expectedWords) {
|
||||
if (!(word in obj)) {
|
||||
throw new Error(`Missing key in LLM output: "${word}"`);
|
||||
}
|
||||
if (!Array.isArray(obj[word]) || (obj[word] as unknown[]).length === 0) {
|
||||
throw new Error(`LLM output for "${word}" is not a non-empty array`);
|
||||
}
|
||||
|
||||
// Validate each sense in the array
|
||||
const senses = obj[word] as unknown[];
|
||||
for (let i = 0; i < senses.length; i++) {
|
||||
validateSense(senses[i], word, i);
|
||||
}
|
||||
}
|
||||
|
||||
return obj;
|
||||
}
|
||||
|
||||
/**
|
||||
* Takes parsed LLM output and builds final enriched objects with composite IDs.
|
||||
*/
|
||||
export function buildEnrichedData(
|
||||
parsed: Record<string, unknown>,
|
||||
rawLanguage: string,
|
||||
rawPos: string,
|
||||
): Map<string, EnrichedSense[]> {
|
||||
const language = (LANG_MAP[rawLanguage] || rawLanguage) as Language;
|
||||
const pos = (POS_MAP[rawPos] || rawPos) as Pos;
|
||||
const results = new Map<string, EnrichedSense[]>();
|
||||
|
||||
for (const [word, sensesArray] of Object.entries(parsed)) {
|
||||
const senses = (sensesArray as unknown[]).map((item, index) => {
|
||||
const sense = item as Omit<
|
||||
EnrichedSense,
|
||||
"id" | "word" | "language" | "pos"
|
||||
>;
|
||||
|
||||
return {
|
||||
id: `${word}:${language}:${pos}:${index}`,
|
||||
word,
|
||||
language,
|
||||
pos,
|
||||
...sense,
|
||||
} as EnrichedSense;
|
||||
});
|
||||
|
||||
results.set(word, senses);
|
||||
}
|
||||
|
||||
return results;
|
||||
}
|
||||
|
||||
/**
|
||||
* Enriches a batch of words by calling the LLM, parsing the response, and building final data.
|
||||
*/
|
||||
export async function enrichWord(
|
||||
words: string[],
|
||||
rawLanguage: string,
|
||||
rawPos: string,
|
||||
): Promise<EnrichmentResult> {
|
||||
const llmResponse = await callLlm(words, rawLanguage, rawPos);
|
||||
const parsed = parseLlmResponse(llmResponse.content, words);
|
||||
const results = buildEnrichedData(parsed, rawLanguage, rawPos);
|
||||
|
||||
return {
|
||||
results,
|
||||
metrics: {
|
||||
promptTokens: llmResponse.promptTokens,
|
||||
completionTokens: llmResponse.completionTokens,
|
||||
totalTokens: llmResponse.totalTokens,
|
||||
promptTimeMs: llmResponse.promptTimeMs,
|
||||
completionTimeMs: llmResponse.completionTimeMs,
|
||||
totalTimeMs: llmResponse.totalTimeMs,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Enriches a batch of words with retry and split-on-failure logic.
|
||||
* Retries up to BATCH_CONFIG.maxRetries times, then splits batch in half and retries each half.
|
||||
* Continues splitting until batch size is 1, then throws if still failing.
|
||||
*/
|
||||
export async function enrichWordWithRetry(
|
||||
words: string[],
|
||||
rawLanguage: string,
|
||||
rawPos: string,
|
||||
attempt: number = 1,
|
||||
): Promise<EnrichmentResult> {
|
||||
try {
|
||||
return await enrichWord(words, rawLanguage, rawPos);
|
||||
} catch (error: unknown) {
|
||||
const errorMessage = error instanceof Error ? error.message : String(error);
|
||||
|
||||
if (words.length === 1) {
|
||||
throw new Error(
|
||||
`Failed to enrich word "${words[0]}" after ${attempt} attempts: ${errorMessage}`,
|
||||
{ cause: error },
|
||||
);
|
||||
}
|
||||
|
||||
if (attempt < BATCH_CONFIG.maxRetries) {
|
||||
console.log(
|
||||
` Retry ${attempt}/${BATCH_CONFIG.maxRetries} for batch [${words.join(", ")}]: ${errorMessage}`,
|
||||
);
|
||||
return enrichWordWithRetry(words, rawLanguage, rawPos, attempt + 1);
|
||||
}
|
||||
|
||||
// Max retries reached — split and retry
|
||||
console.log(
|
||||
` Splitting batch [${words.join(", ")}] after ${BATCH_CONFIG.maxRetries} failed attempts`,
|
||||
);
|
||||
|
||||
const half = Math.ceil(words.length / 2);
|
||||
const left = words.slice(0, half);
|
||||
const right = words.slice(half);
|
||||
|
||||
const leftResult = await enrichWordWithRetry(left, rawLanguage, rawPos, 1);
|
||||
const rightResult = await enrichWordWithRetry(
|
||||
right,
|
||||
rawLanguage,
|
||||
rawPos,
|
||||
1,
|
||||
);
|
||||
|
||||
// Merge results
|
||||
const merged = new Map([...leftResult.results, ...rightResult.results]);
|
||||
const mergedMetrics = {
|
||||
promptTokens:
|
||||
leftResult.metrics.promptTokens + rightResult.metrics.promptTokens,
|
||||
completionTokens:
|
||||
leftResult.metrics.completionTokens +
|
||||
rightResult.metrics.completionTokens,
|
||||
totalTokens:
|
||||
leftResult.metrics.totalTokens + rightResult.metrics.totalTokens,
|
||||
promptTimeMs:
|
||||
(leftResult.metrics.promptTimeMs ?? 0) +
|
||||
(rightResult.metrics.promptTimeMs ?? 0),
|
||||
completionTimeMs:
|
||||
(leftResult.metrics.completionTimeMs ?? 0) +
|
||||
(rightResult.metrics.completionTimeMs ?? 0),
|
||||
totalTimeMs:
|
||||
leftResult.metrics.totalTimeMs + rightResult.metrics.totalTimeMs,
|
||||
};
|
||||
|
||||
return { results: merged, metrics: mergedMetrics };
|
||||
}
|
||||
}
|
||||
|
|
@ -1,81 +0,0 @@
|
|||
import type { LlmAdapter } from "./types.js";
|
||||
|
||||
interface GeminiResponse {
|
||||
candidates: Array<{ content: { parts: Array<{ text: string }> } }>;
|
||||
usageMetadata: {
|
||||
promptTokenCount: number;
|
||||
candidatesTokenCount: number;
|
||||
totalTokenCount: number;
|
||||
};
|
||||
}
|
||||
|
||||
export class GeminiAdapter implements LlmAdapter {
|
||||
private apiKey: string;
|
||||
private model: string;
|
||||
|
||||
constructor(apiKey: string, model: string) {
|
||||
this.apiKey = apiKey;
|
||||
this.model = model;
|
||||
}
|
||||
|
||||
async call(
|
||||
words: string[],
|
||||
systemPrompt: string,
|
||||
): Promise<{
|
||||
content: string;
|
||||
promptTokens: number;
|
||||
completionTokens: number;
|
||||
totalTokens: number;
|
||||
promptTimeMs: number | null;
|
||||
completionTimeMs: number | null;
|
||||
totalTimeMs: number;
|
||||
}> {
|
||||
const url = `https://generativelanguage.googleapis.com/v1beta/models/${this.model}:generateContent?key=${this.apiKey}`;
|
||||
|
||||
const payload = {
|
||||
systemInstruction: { parts: [{ text: systemPrompt }] },
|
||||
contents: [
|
||||
{ role: "user", parts: [{ text: "Words: " + JSON.stringify(words) }] },
|
||||
],
|
||||
generationConfig: {
|
||||
temperature: 0.1,
|
||||
topP: 0.9,
|
||||
maxOutputTokens: Math.ceil(words.length * 250 * 1.2),
|
||||
},
|
||||
};
|
||||
|
||||
const startTime = Date.now();
|
||||
|
||||
const response = await fetch(url, {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify(payload),
|
||||
});
|
||||
|
||||
const totalTimeMs = Date.now() - startTime;
|
||||
|
||||
if (!response.ok) {
|
||||
throw new Error(`Gemini API responded with status: ${response.status}`);
|
||||
}
|
||||
|
||||
const json = (await response.json()) as GeminiResponse;
|
||||
|
||||
const content = json.candidates[0]?.content?.parts[0]?.text;
|
||||
if (!content) {
|
||||
throw new Error("Gemini response content is empty");
|
||||
}
|
||||
|
||||
const promptTokens = json.usageMetadata.promptTokenCount;
|
||||
const completionTokens = json.usageMetadata.candidatesTokenCount;
|
||||
|
||||
return {
|
||||
content,
|
||||
promptTokens,
|
||||
completionTokens,
|
||||
totalTokens: json.usageMetadata.totalTokenCount,
|
||||
promptTimeMs: null,
|
||||
completionTimeMs: null,
|
||||
totalTimeMs,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
|
@ -1,14 +0,0 @@
|
|||
export interface LlmAdapter {
|
||||
call(
|
||||
words: string[],
|
||||
systemPrompt: string,
|
||||
): Promise<{
|
||||
content: string;
|
||||
promptTokens: number;
|
||||
completionTokens: number;
|
||||
totalTokens: number;
|
||||
promptTimeMs: number | null;
|
||||
completionTimeMs: number | null;
|
||||
totalTimeMs: number;
|
||||
}>;
|
||||
}
|
||||
|
|
@ -1,44 +0,0 @@
|
|||
import { LLM_CONFIG } from "../config/llm.js";
|
||||
|
||||
export type Language = "en" | "de" | "it" | "es" | "fr";
|
||||
export type Pos = "noun" | "verb" | "adjective" | "adverb";
|
||||
export type Gender = "masculine" | "feminine" | "neuter" | null;
|
||||
export type Difficulty = "easy" | "medium" | "hard";
|
||||
|
||||
export interface Translation {
|
||||
word: string;
|
||||
gender: Gender;
|
||||
}
|
||||
|
||||
export interface EnrichedSense {
|
||||
id: string;
|
||||
word: string;
|
||||
language: Language;
|
||||
pos: Pos;
|
||||
sense: string;
|
||||
example: string;
|
||||
difficulty_level: Difficulty;
|
||||
translations: {
|
||||
de: Translation[];
|
||||
it: Translation[];
|
||||
es: Translation[];
|
||||
fr: Translation[];
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Merges skeleton data with enriched LLM senses into the final pipeline output.
|
||||
*/
|
||||
export function mergeEnrichedData(
|
||||
word: string,
|
||||
senses: EnrichedSense[],
|
||||
): Record<string, unknown> {
|
||||
return {
|
||||
word,
|
||||
language: senses[0]?.language ?? "en",
|
||||
pos: senses[0]?.pos ?? "noun",
|
||||
senses,
|
||||
enrichedAt: new Date().toISOString(),
|
||||
model: LLM_CONFIG.model ?? "unknown",
|
||||
};
|
||||
}
|
||||
|
|
@ -1,198 +0,0 @@
|
|||
interface LlmMetrics {
|
||||
promptTokens: number;
|
||||
completionTokens: number;
|
||||
totalTokens: number;
|
||||
promptTimeMs: number | null;
|
||||
completionTimeMs: number | null;
|
||||
totalTimeMs: number;
|
||||
}
|
||||
|
||||
interface PipelineMetrics {
|
||||
startTime: Date;
|
||||
endTime?: Date;
|
||||
wordsProcessed: number;
|
||||
wordsSkipped: number;
|
||||
wordsFailed: number;
|
||||
llmCalls: number;
|
||||
totalPromptTokens: number;
|
||||
totalCompletionTokens: number;
|
||||
totalTokens: number;
|
||||
totalPromptTimeMs: number;
|
||||
totalCompletionTimeMs: number;
|
||||
totalTimeMs: number;
|
||||
currentWordStartTime?: Date;
|
||||
}
|
||||
|
||||
/**
|
||||
* Simple timer and metrics tracker for the pipeline.
|
||||
* Tracks both pipeline throughput and LLM performance.
|
||||
*/
|
||||
export class PipelineTimer {
|
||||
private metrics: PipelineMetrics;
|
||||
|
||||
constructor() {
|
||||
this.metrics = {
|
||||
startTime: new Date(),
|
||||
wordsProcessed: 0,
|
||||
wordsSkipped: 0,
|
||||
wordsFailed: 0,
|
||||
llmCalls: 0,
|
||||
totalPromptTokens: 0,
|
||||
totalCompletionTokens: 0,
|
||||
totalTokens: 0,
|
||||
totalPromptTimeMs: 0,
|
||||
totalCompletionTimeMs: 0,
|
||||
totalTimeMs: 0,
|
||||
};
|
||||
}
|
||||
|
||||
startWord(): void {
|
||||
this.metrics.currentWordStartTime = new Date();
|
||||
}
|
||||
|
||||
getWordDurationMs(): number {
|
||||
if (!this.metrics.currentWordStartTime) return 0;
|
||||
return new Date().getTime() - this.metrics.currentWordStartTime.getTime();
|
||||
}
|
||||
|
||||
recordProcessed(llmMetrics?: LlmMetrics): void {
|
||||
this.metrics.wordsProcessed++;
|
||||
if (llmMetrics) {
|
||||
this.metrics.llmCalls++;
|
||||
this.metrics.totalPromptTokens += llmMetrics.promptTokens;
|
||||
this.metrics.totalCompletionTokens += llmMetrics.completionTokens;
|
||||
this.metrics.totalTokens += llmMetrics.totalTokens;
|
||||
if (llmMetrics.promptTimeMs !== null) {
|
||||
this.metrics.totalPromptTimeMs += llmMetrics.promptTimeMs;
|
||||
}
|
||||
if (llmMetrics.completionTimeMs !== null) {
|
||||
this.metrics.totalCompletionTimeMs += llmMetrics.completionTimeMs;
|
||||
}
|
||||
this.metrics.totalTimeMs += llmMetrics.totalTimeMs;
|
||||
}
|
||||
}
|
||||
|
||||
recordSkipped(): void {
|
||||
this.metrics.wordsSkipped++;
|
||||
}
|
||||
|
||||
recordFailed(): void {
|
||||
this.metrics.wordsFailed++;
|
||||
}
|
||||
|
||||
stop(): void {
|
||||
this.metrics.endTime = new Date();
|
||||
}
|
||||
|
||||
getWordTiming(): string {
|
||||
const durationMs = this.getWordDurationMs();
|
||||
const durationSec = (durationMs / 1000).toFixed(1);
|
||||
return `⏱️ Word took ${durationSec}s`;
|
||||
}
|
||||
|
||||
getEta(totalWords: number): string {
|
||||
const processed = this.metrics.wordsProcessed;
|
||||
const remaining = totalWords - processed - this.metrics.wordsSkipped;
|
||||
|
||||
if (processed === 0 || remaining <= 0) return "ETA: calculating...";
|
||||
|
||||
const elapsedMs = new Date().getTime() - this.metrics.startTime.getTime();
|
||||
const avgMsPerWord = elapsedMs / processed;
|
||||
const etaMs = avgMsPerWord * remaining;
|
||||
|
||||
const etaMin = Math.round(etaMs / 60000);
|
||||
const etaHour = (etaMs / 3600000).toFixed(1);
|
||||
|
||||
if (etaMin < 60) {
|
||||
return `ETA: ${etaMin} min`;
|
||||
}
|
||||
return `ETA: ${etaHour} hours`;
|
||||
}
|
||||
|
||||
getSummary(): string {
|
||||
const end = this.metrics.endTime || new Date();
|
||||
const durationMs = end.getTime() - this.metrics.startTime.getTime();
|
||||
const durationSec = (durationMs / 1000).toFixed(1);
|
||||
|
||||
const total =
|
||||
this.metrics.wordsProcessed +
|
||||
this.metrics.wordsSkipped +
|
||||
this.metrics.wordsFailed;
|
||||
const throughput =
|
||||
this.metrics.wordsProcessed > 0
|
||||
? (this.metrics.wordsProcessed / (durationMs / 1000)).toFixed(2)
|
||||
: "0";
|
||||
|
||||
const avgPromptTokens =
|
||||
this.metrics.llmCalls > 0
|
||||
? (this.metrics.totalPromptTokens / this.metrics.llmCalls).toFixed(0)
|
||||
: "0";
|
||||
const avgCompletionTokens =
|
||||
this.metrics.llmCalls > 0
|
||||
? (this.metrics.totalCompletionTokens / this.metrics.llmCalls).toFixed(
|
||||
0,
|
||||
)
|
||||
: "0";
|
||||
const avgTotalTimeMs =
|
||||
this.metrics.llmCalls > 0
|
||||
? (this.metrics.totalTimeMs / this.metrics.llmCalls).toFixed(0)
|
||||
: "0";
|
||||
|
||||
const unifiedThroughput =
|
||||
this.metrics.totalTimeMs > 0
|
||||
? (
|
||||
this.metrics.totalTokens /
|
||||
(this.metrics.totalTimeMs / 1000)
|
||||
).toFixed(1)
|
||||
: "N/A";
|
||||
|
||||
const hasDetailedTimings =
|
||||
this.metrics.totalPromptTimeMs > 0 ||
|
||||
this.metrics.totalCompletionTimeMs > 0;
|
||||
|
||||
const avgPromptSpeed =
|
||||
this.metrics.totalPromptTimeMs > 0
|
||||
? (
|
||||
this.metrics.totalPromptTokens /
|
||||
(this.metrics.totalPromptTimeMs / 1000)
|
||||
).toFixed(1)
|
||||
: "N/A";
|
||||
|
||||
const avgCompletionSpeed =
|
||||
this.metrics.totalCompletionTimeMs > 0
|
||||
? (
|
||||
this.metrics.totalCompletionTokens /
|
||||
(this.metrics.totalCompletionTimeMs / 1000)
|
||||
).toFixed(1)
|
||||
: "N/A";
|
||||
|
||||
const lines = [
|
||||
`⏱️ Pipeline Summary`,
|
||||
` Duration: ${durationSec}s`,
|
||||
` Processed: ${this.metrics.wordsProcessed}`,
|
||||
` Skipped: ${this.metrics.wordsSkipped}`,
|
||||
` Failed: ${this.metrics.wordsFailed}`,
|
||||
` Total: ${total}`,
|
||||
` Throughput: ${throughput} words/sec`,
|
||||
``,
|
||||
`🤖 LLM Metrics`,
|
||||
` Calls: ${this.metrics.llmCalls}`,
|
||||
` Avg prompt tokens: ${avgPromptTokens}`,
|
||||
` Avg completion tokens: ${avgCompletionTokens}`,
|
||||
` Avg total tokens: ${avgPromptTokens + avgCompletionTokens}`,
|
||||
` Avg total request time: ${avgTotalTimeMs}ms`,
|
||||
` Avg throughput: ${unifiedThroughput} tok/s`,
|
||||
];
|
||||
|
||||
if (hasDetailedTimings) {
|
||||
lines.push(
|
||||
``,
|
||||
` [Local breakdown]`,
|
||||
` Avg prompt speed: ${avgPromptSpeed} tok/s`,
|
||||
` Avg completion speed: ${avgCompletionSpeed} tok/s`,
|
||||
);
|
||||
}
|
||||
|
||||
return lines.join("\n");
|
||||
}
|
||||
}
|
||||
|
|
@ -1,27 +0,0 @@
|
|||
/**
|
||||
* Simple progress tracker for pipeline execution.
|
||||
*/
|
||||
export class ProgressTracker {
|
||||
private current: number;
|
||||
private failed: number;
|
||||
private total: number;
|
||||
|
||||
constructor(total: number) {
|
||||
this.current = 0;
|
||||
this.failed = 0;
|
||||
this.total = total;
|
||||
}
|
||||
|
||||
next(): number {
|
||||
this.current++;
|
||||
return this.current;
|
||||
}
|
||||
|
||||
recordFailed(): void {
|
||||
this.failed++;
|
||||
}
|
||||
|
||||
format(label: string): string {
|
||||
return `[${this.current}/${this.total}] (${this.failed} failed) ${label}`;
|
||||
}
|
||||
}
|
||||
|
|
@ -1,96 +0,0 @@
|
|||
import fs from "fs";
|
||||
|
||||
interface VerificationResult {
|
||||
valid: boolean;
|
||||
errors: string[];
|
||||
}
|
||||
|
||||
/**
|
||||
* Verifies that an enriched JSON file matches the expected schema.
|
||||
* Returns detailed error messages for any violations.
|
||||
*/
|
||||
export function verifyEnrichedFile(filePath: string): VerificationResult {
|
||||
const errors: string[] = [];
|
||||
|
||||
if (!fs.existsSync(filePath)) {
|
||||
return { valid: false, errors: ["File does not exist"] };
|
||||
}
|
||||
|
||||
let data: unknown;
|
||||
try {
|
||||
data = JSON.parse(fs.readFileSync(filePath, "utf-8"));
|
||||
} catch (_error: unknown) {
|
||||
return { valid: false, errors: ["Invalid JSON syntax"] };
|
||||
}
|
||||
|
||||
if (typeof data !== "object" || data === null || Array.isArray(data)) {
|
||||
return { valid: false, errors: ["Root must be an object"] };
|
||||
}
|
||||
|
||||
const obj = data as Record<string, unknown>;
|
||||
|
||||
// Required top-level fields
|
||||
const requiredFields = ["word", "language", "pos", "senses"];
|
||||
for (const field of requiredFields) {
|
||||
if (!(field in obj)) {
|
||||
errors.push(`Missing required field: "${field}"`);
|
||||
}
|
||||
}
|
||||
|
||||
// Validate senses array
|
||||
if (!Array.isArray(obj["senses"])) {
|
||||
errors.push('"senses" must be an array');
|
||||
} else if (obj["senses"].length === 0) {
|
||||
errors.push('"senses" array cannot be empty');
|
||||
} else {
|
||||
for (let i = 0; i < obj["senses"].length; i++) {
|
||||
const sense = obj["senses"][i] as Record<string, unknown>;
|
||||
const sensePrefix = `senses[${i}]`;
|
||||
|
||||
if (!sense["sense"] || typeof sense["sense"] !== "string") {
|
||||
errors.push(`${sensePrefix}: missing or invalid "sense"`);
|
||||
}
|
||||
if (!sense["example"] || typeof sense["example"] !== "string") {
|
||||
errors.push(`${sensePrefix}: missing or invalid "example"`);
|
||||
}
|
||||
if (
|
||||
!["easy", "medium", "hard"].includes(
|
||||
sense["difficulty_level"] as string,
|
||||
)
|
||||
) {
|
||||
errors.push(`${sensePrefix}: invalid "difficulty_level"`);
|
||||
}
|
||||
if (!sense["translations"] || typeof sense["translations"] !== "object") {
|
||||
errors.push(`${sensePrefix}: missing "translations"`);
|
||||
} else {
|
||||
const trans = sense["translations"] as Record<string, unknown>;
|
||||
for (const lang of ["de", "it", "es", "fr"]) {
|
||||
if (!Array.isArray(trans[lang])) {
|
||||
errors.push(
|
||||
`${sensePrefix}: missing or invalid "${lang}" translations`,
|
||||
);
|
||||
} else {
|
||||
for (let j = 0; j < (trans[lang] as unknown[]).length; j++) {
|
||||
const t = (trans[lang] as unknown[])[j] as Record<
|
||||
string,
|
||||
unknown
|
||||
>;
|
||||
if (!t["word"] || typeof t["word"] !== "string") {
|
||||
errors.push(`${sensePrefix}.${lang}[${j}]: missing "word"`);
|
||||
}
|
||||
if (
|
||||
!["masculine", "feminine", "neuter", null].includes(
|
||||
t["gender"] as string | null,
|
||||
)
|
||||
) {
|
||||
errors.push(`${sensePrefix}.${lang}[${j}]: invalid "gender"`);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return { valid: errors.length === 0, errors };
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue