lila/data-pipeline/utils/merge-enriched-data.ts
2026-07-06 13:09:30 +02:00

69 lines
1.5 KiB
TypeScript

export type Language = "en" | "de" | "it" | "es" | "fr";
export type Pos = "noun" | "verb" | "adjective" | "adverb";
export type Gender = "masculine" | "feminine" | "neuter" | null;
export type Difficulty = "easy" | "medium" | "hard";
export interface Translation {
word: string;
gender: Gender;
}
export interface EnrichedSense {
id: string;
word: string;
language: Language;
pos: Pos;
sense: string;
example: string;
difficulty_level: Difficulty;
translations: {
de: Translation[];
it: Translation[];
es: Translation[];
fr: Translation[];
};
}
const LANG_MAP: Record<string, Language> = {
english: "en",
italian: "it",
german: "de",
french: "fr",
spanish: "es",
};
const POS_MAP: Record<string, Pos> = {
nouns: "noun",
verbs: "verb",
adverbs: "adverb",
adjectives: "adjective",
};
/**
* Merges skeleton data with enriched LLM senses into the final pipeline output.
*/
export function mergeEnrichedData(
word: string,
rawLanguage: string,
rawPos: string,
senses: EnrichedSense[],
): Record<string, unknown> {
const language = LANG_MAP[rawLanguage] || (rawLanguage as Language);
const pos = POS_MAP[rawPos] || (rawPos as Pos);
const fixedSenses = senses.map((sense, index) => ({
...sense,
id: `${word}:${language}:${pos}:${index}`,
language,
pos,
}));
return {
word,
language,
pos,
senses: fixedSenses,
enrichedAt: new Date().toISOString(),
model: "qwen3.5-4b-q4_k_m",
};
}