wip

data-pipeline/utils/merge-enriched-data.ts

@@ -0,0 +1,69 @@
+export type Language = "en" | "de" | "it" | "es" | "fr";
+export type Pos = "noun" | "verb" | "adjective" | "adverb";
+export type Gender = "masculine" | "feminine" | "neuter" | null;
+export type Difficulty = "easy" | "medium" | "hard";
+
+export interface Translation {
+  word: string;
+  gender: Gender;
+}
+
+export interface EnrichedSense {
+  id: string;
+  word: string;
+  language: Language;
+  pos: Pos;
+  sense: string;
+  example: string;
+  difficulty_level: Difficulty;
+  translations: {
+    de: Translation[];
+    it: Translation[];
+    es: Translation[];
+    fr: Translation[];
+  };
+}
+
+const LANG_MAP: Record<string, Language> = {
+  english: "en",
+  italian: "it",
+  german: "de",
+  french: "fr",
+  spanish: "es",
+};
+
+const POS_MAP: Record<string, Pos> = {
+  nouns: "noun",
+  verbs: "verb",
+  adverbs: "adverb",
+  adjectives: "adjective",
+};
+
+/**
+ * Merges skeleton data with enriched LLM senses into the final pipeline output.
+ */
+export function mergeEnrichedData(
+  word: string,
+  rawLanguage: string,
+  rawPos: string,
+  senses: EnrichedSense[],
+): Record<string, unknown> {
+  const language = LANG_MAP[rawLanguage] || (rawLanguage as Language);
+  const pos = POS_MAP[rawPos] || (rawPos as Pos);
+
+  const fixedSenses = senses.map((sense, index) => ({
+    ...sense,
+    id: `${word}:${language}:${pos}:${index}`,
+    language,
+    pos,
+  }));
+
+  return {
+    word,
+    language,
+    pos,
+    senses: fixedSenses,
+    enrichedAt: new Date().toISOString(),
+    model: "qwen3.5-4b-q4_k_m",
+  };
+}