From 88b16a1ed73d774ec6151406ddbaa24048c1e72b Mon Sep 17 00:00:00 2001 From: lila Date: Tue, 21 Jul 2026 17:45:10 +0200 Subject: [PATCH] removing not needed files --- data-pipeline/config/constants.ts | 16 - data-pipeline/config/prompt.ts | 43 --- data-pipeline/package.json | 5 - data-pipeline/pipeline.ts | 0 data-pipeline/source-data/french/adjectives | 20 + data-pipeline/source-data/german/verbs | 20 + data-pipeline/source-data/italian/adverbs | 20 + data-pipeline/source-data/spanish/nouns | 20 + data-pipeline/utils/create-line-reader.ts | 11 - data-pipeline/utils/create-output-dirs.ts | 20 - data-pipeline/utils/get-word-file-path.ts | 5 - data-pipeline/utils/scanning-source-files.ts | 61 ---- data-pipeline/utils/write-json-file.ts | 11 - documentation/pipeline/ENGLISH_NOUNS.md | 342 ------------------ .../pipeline/TRIAL_IMPLEMENTATION_ROADMAP.md | 41 --- 15 files changed, 80 insertions(+), 555 deletions(-) delete mode 100644 data-pipeline/config/constants.ts delete mode 100644 data-pipeline/config/prompt.ts create mode 100644 data-pipeline/pipeline.ts create mode 100644 data-pipeline/source-data/french/adjectives create mode 100644 data-pipeline/source-data/german/verbs create mode 100644 data-pipeline/source-data/italian/adverbs create mode 100644 data-pipeline/source-data/spanish/nouns delete mode 100644 data-pipeline/utils/create-line-reader.ts delete mode 100644 data-pipeline/utils/create-output-dirs.ts delete mode 100644 data-pipeline/utils/get-word-file-path.ts delete mode 100644 data-pipeline/utils/scanning-source-files.ts delete mode 100644 data-pipeline/utils/write-json-file.ts delete mode 100644 documentation/pipeline/ENGLISH_NOUNS.md delete mode 100644 documentation/pipeline/TRIAL_IMPLEMENTATION_ROADMAP.md diff --git a/data-pipeline/config/constants.ts b/data-pipeline/config/constants.ts deleted file mode 100644 index 7ffc146..0000000 --- a/data-pipeline/config/constants.ts +++ /dev/null @@ -1,16 +0,0 @@ -export const LANG_MAP: Record = { - english: "en", - italian: "it", - german: "de", - french: "fr", - spanish: "es", -}; - -export const POS_MAP: Record = { - nouns: "noun", - verbs: "verb", - adverbs: "adverb", - adjectives: "adjective", -}; - -export const ALL_LANGUAGES = ["en", "de", "it", "es", "fr"]; diff --git a/data-pipeline/config/prompt.ts b/data-pipeline/config/prompt.ts deleted file mode 100644 index 4c2d559..0000000 --- a/data-pipeline/config/prompt.ts +++ /dev/null @@ -1,43 +0,0 @@ -export function buildSystemPrompt( - sourceLanguage: string, - pos: string, - targetLanguages: string[], -): string { - return `You are a multilingual dictionary engine. Output ONLY a JSON object. No markdown, no explanations. - -For each ${sourceLanguage} ${pos} provided, generate 1-2 distinct senses. - -CEFR difficulty mapping: -- A1/A2 → easy -- B1/B2 → medium -- C1/C2 → hard - -Each sense must have: -- sense: student-friendly definition, max 15 words -- example: natural sentence using the word -- difficulty_level: easy, medium, or hard -- translations: object with keys ${targetLanguages.join(", ")}; each value is an array of {word, gender} where gender MUST be masculine, feminine, or neuter. Use null ONLY if the language has no grammatical gender for that word. - -Output format: JSON object where keys are the input words, values are arrays of sense objects. - -Example for ["house"]: -{ - "house": [ - { - "sense": "A building for human habitation.", - "example": "They bought a house in the city.", - "difficulty_level": "easy", - "translations": { - "de": [{"word": "Haus", "gender": "neuter"}], - "it": [{"word": "casa", "gender": "feminine"}], - "es": [{"word": "casa", "gender": "feminine"}], - "fr": [{"word": "maison", "gender": "feminine"}] - } - } - ] -} - -/no-think - -`; -} diff --git a/data-pipeline/package.json b/data-pipeline/package.json index b8004c3..515057a 100644 --- a/data-pipeline/package.json +++ b/data-pipeline/package.json @@ -4,11 +4,6 @@ "private": true, "type": "module", "scripts": { - "db:reset": "tsx db/reset.ts", - "extract": "tsx stage-1-extract/scripts/extract.ts", - "reverse-link": "tsx stage-2-reverse-link/scripts/reverse-link.ts", - "db:import": "tsx db/import.ts", - "db:init": "tsx db/init.ts", "test": "vitest run", "test:watch": "vitest", "pipeline:run": "tsx --env-file .env pipeline.ts" diff --git a/data-pipeline/pipeline.ts b/data-pipeline/pipeline.ts new file mode 100644 index 0000000..e69de29 diff --git a/data-pipeline/source-data/french/adjectives b/data-pipeline/source-data/french/adjectives new file mode 100644 index 0000000..a374b94 --- /dev/null +++ b/data-pipeline/source-data/french/adjectives @@ -0,0 +1,20 @@ +grand +petit +bon +mauvais +beau +nouveau +vieux +jeune +heureux +triste +fort +faible +rapide +lent +chaud +froid +facile +difficile +propre +sale diff --git a/data-pipeline/source-data/german/verbs b/data-pipeline/source-data/german/verbs new file mode 100644 index 0000000..84b9aea --- /dev/null +++ b/data-pipeline/source-data/german/verbs @@ -0,0 +1,20 @@ +sein +haben +werden +können +müssen +sagen +machen +geben +kommen +gehen +wissen +sehen +lassen +stehen +finden +bleiben +liegen +heißen +denken +nehmen diff --git a/data-pipeline/source-data/italian/adverbs b/data-pipeline/source-data/italian/adverbs new file mode 100644 index 0000000..e275515 --- /dev/null +++ b/data-pipeline/source-data/italian/adverbs @@ -0,0 +1,20 @@ +bene +male +sempre +mai +spesso +raramente +oggi +domani +ieri +qui +lì +molto +poco +troppo +abbastanza +velocemente +lentamente +insieme +forse +davvero diff --git a/data-pipeline/source-data/spanish/nouns b/data-pipeline/source-data/spanish/nouns new file mode 100644 index 0000000..8e9d205 --- /dev/null +++ b/data-pipeline/source-data/spanish/nouns @@ -0,0 +1,20 @@ +mesa +silla +coche +perro +gato +ventana +puerta +calle +plaza +mercado +parque +río +montaña +playa +sol +luna +estrella +cielo +tierra +árbol diff --git a/data-pipeline/utils/create-line-reader.ts b/data-pipeline/utils/create-line-reader.ts deleted file mode 100644 index c3b166a..0000000 --- a/data-pipeline/utils/create-line-reader.ts +++ /dev/null @@ -1,11 +0,0 @@ -import fs from "fs"; -import readline from "readline"; - -/** - * Creates a line-by-line reader stream for a given file path. - */ -export function createLineReader(sourcePath: string): readline.Interface { - const fileStream = fs.createReadStream(sourcePath, "utf-8"); - - return readline.createInterface({ input: fileStream, crlfDelay: Infinity }); -} diff --git a/data-pipeline/utils/create-output-dirs.ts b/data-pipeline/utils/create-output-dirs.ts deleted file mode 100644 index d86b0e7..0000000 --- a/data-pipeline/utils/create-output-dirs.ts +++ /dev/null @@ -1,20 +0,0 @@ -import fs from "fs"; -import type { Wordlist } from "./scanning-source-files.js"; - -/** - * Takes a list of scanned datasets and creates their output folders if missing. - */ -export function ensureOutputFolders(wordlists: Wordlist[]): void { - for (const wordlist of wordlists) { - if (!fs.existsSync(wordlist.outputDir)) { - fs.mkdirSync(wordlist.outputDir, { recursive: true }); - console.log( - `📁 Created target folder: worddata/${wordlist.language}/${wordlist.pos}`, - ); - } - } - - console.log( - "✅ All required output directories have been verified and created successfully.", - ); -} diff --git a/data-pipeline/utils/get-word-file-path.ts b/data-pipeline/utils/get-word-file-path.ts deleted file mode 100644 index 81ef2d3..0000000 --- a/data-pipeline/utils/get-word-file-path.ts +++ /dev/null @@ -1,5 +0,0 @@ -import path from "path"; - -export function getWordFilePath(word: string, outputDir: string): string { - return path.join(outputDir, `${word}.json`); -} diff --git a/data-pipeline/utils/scanning-source-files.ts b/data-pipeline/utils/scanning-source-files.ts deleted file mode 100644 index e9d05ca..0000000 --- a/data-pipeline/utils/scanning-source-files.ts +++ /dev/null @@ -1,61 +0,0 @@ -import fs from "fs"; -import path from "path"; - -// Define a simple shape for what a discovered dataset looks like -export interface Wordlist { - language: string; - pos: string; - sourcePath: string; - outputDir: string; -} - -/** - * Scans the source-data directory to find all available word lists. - */ -export function scanSourceData(baseDir: string): Wordlist[] { - const sourceBaseDir = path.join(baseDir, "source-data"); - const discoveredWordlists: Wordlist[] = []; - - // Safety check: if there's no source-data folder, return an empty array - if (!fs.existsSync(sourceBaseDir)) { - return discoveredWordlists; - } - - // 1. Read the language directories (e.g., ['english']) - const languages = fs.readdirSync(sourceBaseDir); - - for (const lang of languages) { - const langFolderPath = path.join(sourceBaseDir, lang); - - // Make sure it's a directory, not a stray file - if (!fs.statSync(langFolderPath).isDirectory()) continue; - - // 2. Read the files inside the language folder (e.g., ['nouns']) - const posFiles = fs.readdirSync(langFolderPath); - - for (const pos of posFiles) { - const fullSourcePath = path.join(langFolderPath, pos); - - // Make sure it's a file (like your extensionless "nouns" file) - if (!fs.statSync(fullSourcePath).isFile()) continue; - - // 3. Package everything into a flat item and add it to our array - discoveredWordlists.push({ - language: lang, - pos: pos, - sourcePath: fullSourcePath, - outputDir: path.join(baseDir, "worddata", lang, pos), - }); - } - } - - // show summary - console.log( - `✅ Scan complete! Found ${discoveredWordlists.length} wordlist(s):`, - ); - for (const list of discoveredWordlists) { - console.log(` • ${list.language.toUpperCase()} (${list.pos})`); - } - - return discoveredWordlists; -} diff --git a/data-pipeline/utils/write-json-file.ts b/data-pipeline/utils/write-json-file.ts deleted file mode 100644 index 29c5c99..0000000 --- a/data-pipeline/utils/write-json-file.ts +++ /dev/null @@ -1,11 +0,0 @@ -import fs from "fs"; - -/** - * Writes data as formatted JSON to a file path. - * Safely catches and re-throws file system errors. - */ -export function writeJsonFile(filePath: string, data: unknown): void { - const tempPath = `${filePath}.tmp`; - fs.writeFileSync(tempPath, JSON.stringify(data, null, 2), "utf-8"); - fs.renameSync(tempPath, filePath); -} diff --git a/documentation/pipeline/ENGLISH_NOUNS.md b/documentation/pipeline/ENGLISH_NOUNS.md deleted file mode 100644 index 6c23891..0000000 --- a/documentation/pipeline/ENGLISH_NOUNS.md +++ /dev/null @@ -1,342 +0,0 @@ -# english nouns - -## step 1a - -get freqency source list + verify lemmatization + confirm language/POS tagging - -example: - -house -bank -asdf - -## step 1b - -transform list into JSON (headword, language, POS) - -example: - -```json -{ - "headword": "house", - "language": "en", - "pos": "noun" -}, -{ - "headword": "bank", - "language": "en", - "pos": "noun" -}, -{ - "headword": "asdf", - "language": "en", - "pos": "noun" -} -``` - ---- - -## step 2a - -check existence in kaikki (eg. is there a english noun "asdf" in kaikki, if not put it separate list) - -example output: - -```json -{ - "headword": "house", - "language": "en", - "pos": "noun" -}, -{ - "headword": "bank", - "language": "en", - "pos": "noun" -} -``` - -miss list (triage queue, not trash! contains junk, slipped inflections, and real Kaikki gaps, gets handled separately, never auto-drop!) - -```json -{ "headword": "asdf", "language": "en", "pos": "noun" } -``` - -## step 2b - -for each sense, extract: glosses, translations (with gender), examples -id will be generated via: `headword:lang:pos:sense_index` - -words without glosses will be dropped! (put in special list) - -example output: - -```json -{ - "id": "house:en:noun:0", - "headword": "house", - "language": "en", - "pos": "noun", - "sense": ["A building for human habitation."], - "examples": ["They bought a house in the city."], - "translations": { - "de": [{ "word": "Haus", "gender": "neuter" }], - "it": [{ "word": "casa", "gender": "feminine" }], - "es": [{ "word": "casa", "gender": "feminine" }], - "fr": [{ "word": "maison", "gender": "feminine" }] - } -}, -{ - "id": "house:en:noun:1", - "headword": "house", - "language": "en", - "pos": "noun", - "sense": ["A noble family or lineage."], - "examples": ["The House of Tudor ruled England."], - "translations": { - "de": [ - { "word": "Adelsgeschlecht", "gender": "neuter" }, - { "word": "Haus", "gender": "neuter" } - ] - } -}, -{ - "id": "bank:en:noun:0", - "headword": "bank", - "language": "en", - "pos": "noun", - "sense": ["An institution where one can place and borrow money."], - "examples": ["She deposited her paycheck at the bank."], - "translations": { - "de": [{ "word": "Bank", "gender": "feminine" }], - "it": [{ "word": "banca", "gender": "feminine" }], - "es": [{ "word": "banco", "gender": "masculine" }], - "fr": [{ "word": "banque", "gender": "feminine" }] - } -}, -{ - "id": "bank:en:noun:1", - "headword": "bank", - "language": "en", - "pos": "noun", - "sense": ["The land alongside a river or lake."], - "examples": ["They picnicked on the bank of the river."], - "translations": { - "de": [{ "word": "Ufer", "gender": "neuter" }], - "it": [{ "word": "riva", "gender": "feminine" }], - "es": [{ "word": "orilla", "gender": "feminine" }], - "fr": [{ "word": "rive", "gender": "feminine" }] - } -}, -{ - "id": "bank:en:noun:2", - "headword": "bank", - "language": "en", - "pos": "noun", - "sense": ["A collection or store of something held in reserve."], - "examples": ["The hospital keeps a blood bank."], - "translations": { - "de": [{ "word": "Bank", "gender": "feminine" }] - } -} -``` - -## step 2c - -fill gaps Kaikki left (LLM, 3 models, generate-then-vote) - -takes 2b's partial cards. 3 different-family models (qwen, llama, gemma) generate, then vote. - -separate focused sub-passes, one field at a time — never combined: - -- **translations** → per missing language, generated with the gloss as context. generate → vote. no agreement → gap stays (cloud audit later) -- **examples** → for senses with no Kaikki example. generate → verify-vote ("is this a valid example of the gloss?"), since freeform sentences never exact-match. no agreement → no example (card still valid) no tiebreak: no agreement leaves the gap, never escalates to more models. vote records stored in pipeline.db for the cloud audit. -- **invariant**: the gloss is never LLM-generated — translations are filled, examples generated, difficulty graded, but the gloss must always be Kaikki's (no gloss → sense dropped in 2b) - -example input: - -```json -{ - "id": "harbor:en:noun:0", - "headword": "harbor", - "language": "en", - "pos": "noun", - "glosses": ["A sheltered area of water where ships can dock safely."], - "examples": [], - "translations": { "de": [{ "word": "Hafen", "gender": "masculine" }] } -} -``` - -next substage is the translation generation: - -```json -{ - "id": "harbor:en:noun:0", - "headword": "harbor", - "language": "en", - "pos": "noun", - "glosses": ["A sheltered area of water where ships can dock safely."], - "examples": [], - "translations": { - "de": [{ "word": "Hafen", "gender": "masculine" }], - "it": [{ "word": "porto", "gender": null }], - "es": [{ "word": "puerto", "gender": null }], - "fr": [{ "word": "port", "gender": null }] - } -} -``` - -next substage is the gender modification (by looking up kaikki for it/es/fr or corresponding wiktionary, fill only nulls, not touching existing genders): -**runs only after all genders are present** - -```json -{ - "id": "harbor:en:noun:0", - "headword": "harbor", - "language": "en", - "pos": "noun", - "glosses": ["A sheltered area of water where ships can dock safely."], - "examples": [], - "translations": { - "de": [{ "word": "Hafen", "gender": "masculine" }], - "it": [{ "word": "porto", "gender": "masculine" }], - "es": [{ "word": "puerto", "gender": "masculine" }], - "fr": [{ "word": "port", "gender": "masculine" }] - } -} -``` - -next substage is the example generation: - -```json -{ - "id": "harbor:en:noun:0", - "headword": "harbor", - "language": "en", - "pos": "noun", - "glosses": ["A sheltered area of water where ships can dock safely."], - "examples": ["The fishing boats returned to the harbor at dusk."], - "translations": { - "de": [{ "word": "Hafen", "gender": "masculine" }], - "it": [{ "word": "porto", "gender": "masculine" }], - "es": [{ "word": "puerto", "gender": "masculine" }], - "fr": [{ "word": "port", "gender": "masculine" }] - } -} -``` - ---- - -## step 3 - -adding difficulty level - -CEFR is mapped to three buckets: -A1/A2 → easy -B1/B2 → intermediate -C1/C2 → hard - -depending on number of senses per headword: - -- One sense → derive difficulty from the CEFRLex distribution (first CEFR level crossing a frequency threshold, mapped to a bucket, not the peak), for the four covered languages (en/de/es/fr). Deterministic, no LLM -- Multiple senses → the three local LLMs grade each sense (easy/intermediate/hard) from the gloss. CEFRLex not involved. -- No CEFRLex entry at all (Italian, or single-sense word that's missing) → LLMs grade from gloss - -to get coverage (not quality), 3 local llms are going to be used: gemma, qwen and llama - -on the llm votes: - -- **Majority agrees (3-0 or 2-1)** → ship the majority value. -- **3-way split (all three differ)** → no consensus → exclude the card to the review queue. Not shipped. - -example output: - -```json -{ - "id": "house:en:noun:0", - "headword": "house", - "language": "en", - "pos": "noun", - "difficulty_level": "easy", - "glosses": ["A building for human habitation."], - "examples": ["They bought a house in the city."], - "translations": { - "de": [{ "word": "Haus", "gender": "neuter" }], - "it": [{ "word": "casa", "gender": "feminine" }], - "es": [{ "word": "casa", "gender": "feminine" }], - "fr": [{ "word": "maison", "gender": "feminine" }] - } -}, -{ - "id": "house:en:noun:1", - "headword": "house", - "language": "en", - "pos": "noun", - "difficulty_level": "hard", - "glosses": ["A noble family or lineage."], - "examples": ["The House of Tudor ruled England."], - "translations": { - "de": [ - { "word": "Adelsgeschlecht", "gender": "neuter" }, - { "word": "Haus", "gender": "neuter" } - ] - } -} -``` - -**important**: `difficulty_level` is mandatory on every shipped card (the user sets difficulty before a game) - -This is coverage, not final quality: three local models make every shipped card's difficulty present and plausible, but not guaranteed correct - -no cefr list: needs to be graded by online llms (see notes) - ---- - -## coverage report (trial deliverable) - -- not a data stage => reads the finished cards and summarises them -- the trial (english + italian nouns, local models) runs steps 1a→3 then emits a report - -### pipeline health — did it run? - -- input lemmas → output cards (ratio; multi-sense makes cards > lemmas) -- dropped per stage + why: 2a misses (junk/inflection/gap), 2b no-gloss drops, 3 no-consensus exclusions -- parse failures / errors - -### coverage — how complete? - -- translation completeness: all 4 langs vs gaps, per language -- examples: from Kaikki vs LLM-generated vs none -- gender: translations still null after Wiktionary fill -- difficulty distribution per bucket (lopsided = broken CEFRLex threshold or bad grading) - -### local good enough? — model agreement - -- difficulty votes: 3-0 / 2-1 / 3-way split rates (high 3-way = locals can't do it → need cloud) -- translation gap-fill: agreement rate, gaps left unfilled -- → go/no-go on local-for-launch - -### rent vs API? — workload volume - -- total LLM calls across 2c + 3 (× per-token rate = API cost) -- cards needing LLM vs handled deterministically (english: high deterministic; italian: ~0, no CEFRLex) -- per-language call counts → scale english (low) and italian (high) to estimate the middle three - -### review backlog — what's deferred - -- miss-list size + composition, per language -- no-consensus exclusions (cloud-audit queue size) -- unfilled translation gaps (also cloud-audit work) - -**framing**: english = optimistic floor (rich Kaikki, CEFRLex exists, models strongest). -italian = pessimistic ceiling (no CEFRLex, thinner Kaikki). all five languages sit between. -read to decide, not to archive. - ---- - -Notes: - -- the miss list needs to get verified/re-worked later on -- the kaikki data contains ipas and links to audio files, add them later if needed, they are not needed now -- use online llms to set the difficulty(cefr) of the not shipped words -- add plurals from kaikki/wiktionary -- postgres sync to prod db diff --git a/documentation/pipeline/TRIAL_IMPLEMENTATION_ROADMAP.md b/documentation/pipeline/TRIAL_IMPLEMENTATION_ROADMAP.md deleted file mode 100644 index 6f0c7d2..0000000 --- a/documentation/pipeline/TRIAL_IMPLEMENTATION_ROADMAP.md +++ /dev/null @@ -1,41 +0,0 @@ -# trial run — implementation roadmap (pure vertical) - -goal: english + italian nouns through the full pipeline on local models, emit -coverage report. build a thin end-to-end slice first, then widen each pass. - -## phase 0 — foundation - -- confirm @lila/shared types (lang codes, POS) exist and match -- package.json deps for what phase 1 needs -- one hardcoded test word to carry through the slice (e.g. "house") - -## phase 1 — thin vertical slice (ONE word, 1a→3, crudest possible) - -goal: prove a single card can travel the whole pipeline and come out the end. -allowed to be ugly — hardcode, skip voting, one model, fake CEFRLex. - -- 1a: one lemma record, by hand or trivial read -- 2a: look it up in Kaikki, confirm it exists -- 2b: extract its senses → card(s) with gloss -- 2c: fill one missing translation with ONE local model, no voting -- 3: assign a difficulty crudely (even hardcoded "easy") -- OUT: one finished card. the pipeline has a shape. - -## phase 2 — widen: real deterministic front (1a–2b, all words) - -- 1a: real frequency list, verified -- 2a: real existence gate + miss-list triage -- 2b: real sense extraction, gender from tags, gloss-required drop -- still JSON output. inspect cards by hand. - -## phase 3 — widen: real LLM back (2c + 3) - -- storage seam: JSON → SQLite, schema, resumability -- 2c: real generate→vote, three families, all sub-passes -- 3: real CEFRLex single-sense + 3-model vote multi-sense + exclude-on-split - -## phase 4 — coverage report + runs - -- emit report -- run english, then italian -- decide: local good enough? rent vs API?