forgejo-lila/lila
1
0
Fork 0
Code Issues 50 Pull requests Projects 1 Releases Packages Wiki Activity Actions

bugfixing #1

Browse source
This commit is contained in:
lila 2026-07-06 14:27:47 +02:00
parent afd28d934e
commit 2a6c56ed23
12 changed files with 300 additions and 269 deletions
Download patch file Download diff file Expand all files Collapse all files

25
data-pipeline/pipeline.ts
View file

@ -27,7 +27,7 @@ async function main() {
console.log("\n step 2: creating necessary output folders...");
ensureOutputFolders(wordlists);
// step 3: check to verify the AI engine is ready before touching anything
// step 3: check to verify the local AI engine is ready before touching anything
console.log("\n step 3: verifying local AI engine status...");
await checkLlmServer();
@ -99,12 +99,7 @@ async function main() {
);
const targetFilePath = getWordFilePath(word, wordlist.outputDir);
const enrichedData = mergeEnrichedData(
word,
wordlist.language,
wordlist.pos,
senses,
);
const enrichedData = mergeEnrichedData(word, senses);
writeJsonFile(targetFilePath, enrichedData);
@ -118,17 +113,19 @@ async function main() {
}
timer.recordProcessed({
promptTokens: result.metrics.promptTokens / batch.length,
completionTokens: result.metrics.completionTokens / batch.length,
totalTokens: result.metrics.totalTokens / batch.length,
promptTimeMs: result.metrics.promptTimeMs / batch.length,
completionTimeMs: result.metrics.completionTimeMs / batch.length,
promptTokensPerSecond: result.metrics.promptTokensPerSecond,
completionTokensPerSecond: result.metrics.completionTokensPerSecond,
promptTokens: result.metrics.promptTokens,
completionTokens: result.metrics.completionTokens,
totalTokens: result.metrics.totalTokens,
promptTimeMs: result.metrics.promptTimeMs,
completionTimeMs: result.metrics.completionTimeMs,
});
}
console.log(` ${timer.getWordTiming()}`);
// Show ETA every 5 batches or on the last batch
if (batchNum % 5 === 0 || batchNum === totalBatches) {
console.log(` 📊 ${timer.getEta(unprocessedWords.length)}`);
}
} catch (error: unknown) {
const errorMessage =
error instanceof Error ? error.message : String(error);