removing not needed files
This commit is contained in:
lila
parent
597083e1fd
commit
88b16a1ed7
15 changed files with 80 additions and 555 deletions
|
|
@ -1,16 +0,0 @@
|
||||||
export const LANG_MAP: Record<string, string> = {
|
|
||||||
english: "en",
|
|
||||||
italian: "it",
|
|
||||||
german: "de",
|
|
||||||
french: "fr",
|
|
||||||
spanish: "es",
|
|
||||||
};
|
|
||||||
|
|
||||||
export const POS_MAP: Record<string, string> = {
|
|
||||||
nouns: "noun",
|
|
||||||
verbs: "verb",
|
|
||||||
adverbs: "adverb",
|
|
||||||
adjectives: "adjective",
|
|
||||||
};
|
|
||||||
|
|
||||||
export const ALL_LANGUAGES = ["en", "de", "it", "es", "fr"];
|
|
||||||
|
|
@ -1,43 +0,0 @@
|
||||||
export function buildSystemPrompt(
|
|
||||||
sourceLanguage: string,
|
|
||||||
pos: string,
|
|
||||||
targetLanguages: string[],
|
|
||||||
): string {
|
|
||||||
return `You are a multilingual dictionary engine. Output ONLY a JSON object. No markdown, no explanations.
|
|
||||||
|
|
||||||
For each ${sourceLanguage} ${pos} provided, generate 1-2 distinct senses.
|
|
||||||
|
|
||||||
CEFR difficulty mapping:
|
|
||||||
- A1/A2 → easy
|
|
||||||
- B1/B2 → medium
|
|
||||||
- C1/C2 → hard
|
|
||||||
|
|
||||||
Each sense must have:
|
|
||||||
- sense: student-friendly definition, max 15 words
|
|
||||||
- example: natural sentence using the word
|
|
||||||
- difficulty_level: easy, medium, or hard
|
|
||||||
- translations: object with keys ${targetLanguages.join(", ")}; each value is an array of {word, gender} where gender MUST be masculine, feminine, or neuter. Use null ONLY if the language has no grammatical gender for that word.
|
|
||||||
|
|
||||||
Output format: JSON object where keys are the input words, values are arrays of sense objects.
|
|
||||||
|
|
||||||
Example for ["house"]:
|
|
||||||
{
|
|
||||||
"house": [
|
|
||||||
{
|
|
||||||
"sense": "A building for human habitation.",
|
|
||||||
"example": "They bought a house in the city.",
|
|
||||||
"difficulty_level": "easy",
|
|
||||||
"translations": {
|
|
||||||
"de": [{"word": "Haus", "gender": "neuter"}],
|
|
||||||
"it": [{"word": "casa", "gender": "feminine"}],
|
|
||||||
"es": [{"word": "casa", "gender": "feminine"}],
|
|
||||||
"fr": [{"word": "maison", "gender": "feminine"}]
|
|
||||||
}
|
|
||||||
}
|
|
||||||
]
|
|
||||||
}
|
|
||||||
|
|
||||||
/no-think
|
|
||||||
|
|
||||||
`;
|
|
||||||
}
|
|
||||||
|
|
@ -4,11 +4,6 @@
|
||||||
"private": true,
|
"private": true,
|
||||||
"type": "module",
|
"type": "module",
|
||||||
"scripts": {
|
"scripts": {
|
||||||
"db:reset": "tsx db/reset.ts",
|
|
||||||
"extract": "tsx stage-1-extract/scripts/extract.ts",
|
|
||||||
"reverse-link": "tsx stage-2-reverse-link/scripts/reverse-link.ts",
|
|
||||||
"db:import": "tsx db/import.ts",
|
|
||||||
"db:init": "tsx db/init.ts",
|
|
||||||
"test": "vitest run",
|
"test": "vitest run",
|
||||||
"test:watch": "vitest",
|
"test:watch": "vitest",
|
||||||
"pipeline:run": "tsx --env-file .env pipeline.ts"
|
"pipeline:run": "tsx --env-file .env pipeline.ts"
|
||||||
|
|
|
||||||
0
data-pipeline/pipeline.ts
Normal file
0
data-pipeline/pipeline.ts
Normal file
20
data-pipeline/source-data/french/adjectives
Normal file
20
data-pipeline/source-data/french/adjectives
Normal file
|
|
@ -0,0 +1,20 @@
|
||||||
|
grand
|
||||||
|
petit
|
||||||
|
bon
|
||||||
|
mauvais
|
||||||
|
beau
|
||||||
|
nouveau
|
||||||
|
vieux
|
||||||
|
jeune
|
||||||
|
heureux
|
||||||
|
triste
|
||||||
|
fort
|
||||||
|
faible
|
||||||
|
rapide
|
||||||
|
lent
|
||||||
|
chaud
|
||||||
|
froid
|
||||||
|
facile
|
||||||
|
difficile
|
||||||
|
propre
|
||||||
|
sale
|
||||||
20
data-pipeline/source-data/german/verbs
Normal file
20
data-pipeline/source-data/german/verbs
Normal file
|
|
@ -0,0 +1,20 @@
|
||||||
|
sein
|
||||||
|
haben
|
||||||
|
werden
|
||||||
|
können
|
||||||
|
müssen
|
||||||
|
sagen
|
||||||
|
machen
|
||||||
|
geben
|
||||||
|
kommen
|
||||||
|
gehen
|
||||||
|
wissen
|
||||||
|
sehen
|
||||||
|
lassen
|
||||||
|
stehen
|
||||||
|
finden
|
||||||
|
bleiben
|
||||||
|
liegen
|
||||||
|
heißen
|
||||||
|
denken
|
||||||
|
nehmen
|
||||||
20
data-pipeline/source-data/italian/adverbs
Normal file
20
data-pipeline/source-data/italian/adverbs
Normal file
|
|
@ -0,0 +1,20 @@
|
||||||
|
bene
|
||||||
|
male
|
||||||
|
sempre
|
||||||
|
mai
|
||||||
|
spesso
|
||||||
|
raramente
|
||||||
|
oggi
|
||||||
|
domani
|
||||||
|
ieri
|
||||||
|
qui
|
||||||
|
lì
|
||||||
|
molto
|
||||||
|
poco
|
||||||
|
troppo
|
||||||
|
abbastanza
|
||||||
|
velocemente
|
||||||
|
lentamente
|
||||||
|
insieme
|
||||||
|
forse
|
||||||
|
davvero
|
||||||
20
data-pipeline/source-data/spanish/nouns
Normal file
20
data-pipeline/source-data/spanish/nouns
Normal file
|
|
@ -0,0 +1,20 @@
|
||||||
|
mesa
|
||||||
|
silla
|
||||||
|
coche
|
||||||
|
perro
|
||||||
|
gato
|
||||||
|
ventana
|
||||||
|
puerta
|
||||||
|
calle
|
||||||
|
plaza
|
||||||
|
mercado
|
||||||
|
parque
|
||||||
|
río
|
||||||
|
montaña
|
||||||
|
playa
|
||||||
|
sol
|
||||||
|
luna
|
||||||
|
estrella
|
||||||
|
cielo
|
||||||
|
tierra
|
||||||
|
árbol
|
||||||
|
|
@ -1,11 +0,0 @@
|
||||||
import fs from "fs";
|
|
||||||
import readline from "readline";
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Creates a line-by-line reader stream for a given file path.
|
|
||||||
*/
|
|
||||||
export function createLineReader(sourcePath: string): readline.Interface {
|
|
||||||
const fileStream = fs.createReadStream(sourcePath, "utf-8");
|
|
||||||
|
|
||||||
return readline.createInterface({ input: fileStream, crlfDelay: Infinity });
|
|
||||||
}
|
|
||||||
|
|
@ -1,20 +0,0 @@
|
||||||
import fs from "fs";
|
|
||||||
import type { Wordlist } from "./scanning-source-files.js";
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Takes a list of scanned datasets and creates their output folders if missing.
|
|
||||||
*/
|
|
||||||
export function ensureOutputFolders(wordlists: Wordlist[]): void {
|
|
||||||
for (const wordlist of wordlists) {
|
|
||||||
if (!fs.existsSync(wordlist.outputDir)) {
|
|
||||||
fs.mkdirSync(wordlist.outputDir, { recursive: true });
|
|
||||||
console.log(
|
|
||||||
`📁 Created target folder: worddata/${wordlist.language}/${wordlist.pos}`,
|
|
||||||
);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
console.log(
|
|
||||||
"✅ All required output directories have been verified and created successfully.",
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
@ -1,5 +0,0 @@
|
||||||
import path from "path";
|
|
||||||
|
|
||||||
export function getWordFilePath(word: string, outputDir: string): string {
|
|
||||||
return path.join(outputDir, `${word}.json`);
|
|
||||||
}
|
|
||||||
|
|
@ -1,61 +0,0 @@
|
||||||
import fs from "fs";
|
|
||||||
import path from "path";
|
|
||||||
|
|
||||||
// Define a simple shape for what a discovered dataset looks like
|
|
||||||
export interface Wordlist {
|
|
||||||
language: string;
|
|
||||||
pos: string;
|
|
||||||
sourcePath: string;
|
|
||||||
outputDir: string;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Scans the source-data directory to find all available word lists.
|
|
||||||
*/
|
|
||||||
export function scanSourceData(baseDir: string): Wordlist[] {
|
|
||||||
const sourceBaseDir = path.join(baseDir, "source-data");
|
|
||||||
const discoveredWordlists: Wordlist[] = [];
|
|
||||||
|
|
||||||
// Safety check: if there's no source-data folder, return an empty array
|
|
||||||
if (!fs.existsSync(sourceBaseDir)) {
|
|
||||||
return discoveredWordlists;
|
|
||||||
}
|
|
||||||
|
|
||||||
// 1. Read the language directories (e.g., ['english'])
|
|
||||||
const languages = fs.readdirSync(sourceBaseDir);
|
|
||||||
|
|
||||||
for (const lang of languages) {
|
|
||||||
const langFolderPath = path.join(sourceBaseDir, lang);
|
|
||||||
|
|
||||||
// Make sure it's a directory, not a stray file
|
|
||||||
if (!fs.statSync(langFolderPath).isDirectory()) continue;
|
|
||||||
|
|
||||||
// 2. Read the files inside the language folder (e.g., ['nouns'])
|
|
||||||
const posFiles = fs.readdirSync(langFolderPath);
|
|
||||||
|
|
||||||
for (const pos of posFiles) {
|
|
||||||
const fullSourcePath = path.join(langFolderPath, pos);
|
|
||||||
|
|
||||||
// Make sure it's a file (like your extensionless "nouns" file)
|
|
||||||
if (!fs.statSync(fullSourcePath).isFile()) continue;
|
|
||||||
|
|
||||||
// 3. Package everything into a flat item and add it to our array
|
|
||||||
discoveredWordlists.push({
|
|
||||||
language: lang,
|
|
||||||
pos: pos,
|
|
||||||
sourcePath: fullSourcePath,
|
|
||||||
outputDir: path.join(baseDir, "worddata", lang, pos),
|
|
||||||
});
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// show summary
|
|
||||||
console.log(
|
|
||||||
`✅ Scan complete! Found ${discoveredWordlists.length} wordlist(s):`,
|
|
||||||
);
|
|
||||||
for (const list of discoveredWordlists) {
|
|
||||||
console.log(` • ${list.language.toUpperCase()} (${list.pos})`);
|
|
||||||
}
|
|
||||||
|
|
||||||
return discoveredWordlists;
|
|
||||||
}
|
|
||||||
|
|
@ -1,11 +0,0 @@
|
||||||
import fs from "fs";
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Writes data as formatted JSON to a file path.
|
|
||||||
* Safely catches and re-throws file system errors.
|
|
||||||
*/
|
|
||||||
export function writeJsonFile(filePath: string, data: unknown): void {
|
|
||||||
const tempPath = `${filePath}.tmp`;
|
|
||||||
fs.writeFileSync(tempPath, JSON.stringify(data, null, 2), "utf-8");
|
|
||||||
fs.renameSync(tempPath, filePath);
|
|
||||||
}
|
|
||||||
|
|
@ -1,342 +0,0 @@
|
||||||
# english nouns
|
|
||||||
|
|
||||||
## step 1a
|
|
||||||
|
|
||||||
get freqency source list + verify lemmatization + confirm language/POS tagging
|
|
||||||
|
|
||||||
example:
|
|
||||||
|
|
||||||
house
|
|
||||||
bank
|
|
||||||
asdf
|
|
||||||
|
|
||||||
## step 1b
|
|
||||||
|
|
||||||
transform list into JSON (headword, language, POS)
|
|
||||||
|
|
||||||
example:
|
|
||||||
|
|
||||||
```json
|
|
||||||
{
|
|
||||||
"headword": "house",
|
|
||||||
"language": "en",
|
|
||||||
"pos": "noun"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"headword": "bank",
|
|
||||||
"language": "en",
|
|
||||||
"pos": "noun"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"headword": "asdf",
|
|
||||||
"language": "en",
|
|
||||||
"pos": "noun"
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## step 2a
|
|
||||||
|
|
||||||
check existence in kaikki (eg. is there a english noun "asdf" in kaikki, if not put it separate list)
|
|
||||||
|
|
||||||
example output:
|
|
||||||
|
|
||||||
```json
|
|
||||||
{
|
|
||||||
"headword": "house",
|
|
||||||
"language": "en",
|
|
||||||
"pos": "noun"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"headword": "bank",
|
|
||||||
"language": "en",
|
|
||||||
"pos": "noun"
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
miss list (triage queue, not trash! contains junk, slipped inflections, and real Kaikki gaps, gets handled separately, never auto-drop!)
|
|
||||||
|
|
||||||
```json
|
|
||||||
{ "headword": "asdf", "language": "en", "pos": "noun" }
|
|
||||||
```
|
|
||||||
|
|
||||||
## step 2b
|
|
||||||
|
|
||||||
for each sense, extract: glosses, translations (with gender), examples
|
|
||||||
id will be generated via: `headword:lang:pos:sense_index`
|
|
||||||
|
|
||||||
words without glosses will be dropped! (put in special list)
|
|
||||||
|
|
||||||
example output:
|
|
||||||
|
|
||||||
```json
|
|
||||||
{
|
|
||||||
"id": "house:en:noun:0",
|
|
||||||
"headword": "house",
|
|
||||||
"language": "en",
|
|
||||||
"pos": "noun",
|
|
||||||
"sense": ["A building for human habitation."],
|
|
||||||
"examples": ["They bought a house in the city."],
|
|
||||||
"translations": {
|
|
||||||
"de": [{ "word": "Haus", "gender": "neuter" }],
|
|
||||||
"it": [{ "word": "casa", "gender": "feminine" }],
|
|
||||||
"es": [{ "word": "casa", "gender": "feminine" }],
|
|
||||||
"fr": [{ "word": "maison", "gender": "feminine" }]
|
|
||||||
}
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"id": "house:en:noun:1",
|
|
||||||
"headword": "house",
|
|
||||||
"language": "en",
|
|
||||||
"pos": "noun",
|
|
||||||
"sense": ["A noble family or lineage."],
|
|
||||||
"examples": ["The House of Tudor ruled England."],
|
|
||||||
"translations": {
|
|
||||||
"de": [
|
|
||||||
{ "word": "Adelsgeschlecht", "gender": "neuter" },
|
|
||||||
{ "word": "Haus", "gender": "neuter" }
|
|
||||||
]
|
|
||||||
}
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"id": "bank:en:noun:0",
|
|
||||||
"headword": "bank",
|
|
||||||
"language": "en",
|
|
||||||
"pos": "noun",
|
|
||||||
"sense": ["An institution where one can place and borrow money."],
|
|
||||||
"examples": ["She deposited her paycheck at the bank."],
|
|
||||||
"translations": {
|
|
||||||
"de": [{ "word": "Bank", "gender": "feminine" }],
|
|
||||||
"it": [{ "word": "banca", "gender": "feminine" }],
|
|
||||||
"es": [{ "word": "banco", "gender": "masculine" }],
|
|
||||||
"fr": [{ "word": "banque", "gender": "feminine" }]
|
|
||||||
}
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"id": "bank:en:noun:1",
|
|
||||||
"headword": "bank",
|
|
||||||
"language": "en",
|
|
||||||
"pos": "noun",
|
|
||||||
"sense": ["The land alongside a river or lake."],
|
|
||||||
"examples": ["They picnicked on the bank of the river."],
|
|
||||||
"translations": {
|
|
||||||
"de": [{ "word": "Ufer", "gender": "neuter" }],
|
|
||||||
"it": [{ "word": "riva", "gender": "feminine" }],
|
|
||||||
"es": [{ "word": "orilla", "gender": "feminine" }],
|
|
||||||
"fr": [{ "word": "rive", "gender": "feminine" }]
|
|
||||||
}
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"id": "bank:en:noun:2",
|
|
||||||
"headword": "bank",
|
|
||||||
"language": "en",
|
|
||||||
"pos": "noun",
|
|
||||||
"sense": ["A collection or store of something held in reserve."],
|
|
||||||
"examples": ["The hospital keeps a blood bank."],
|
|
||||||
"translations": {
|
|
||||||
"de": [{ "word": "Bank", "gender": "feminine" }]
|
|
||||||
}
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
## step 2c
|
|
||||||
|
|
||||||
fill gaps Kaikki left (LLM, 3 models, generate-then-vote)
|
|
||||||
|
|
||||||
takes 2b's partial cards. 3 different-family models (qwen, llama, gemma) generate, then vote.
|
|
||||||
|
|
||||||
separate focused sub-passes, one field at a time — never combined:
|
|
||||||
|
|
||||||
- **translations** → per missing language, generated with the gloss as context. generate → vote. no agreement → gap stays (cloud audit later)
|
|
||||||
- **examples** → for senses with no Kaikki example. generate → verify-vote ("is this a valid example of the gloss?"), since freeform sentences never exact-match. no agreement → no example (card still valid) no tiebreak: no agreement leaves the gap, never escalates to more models. vote records stored in pipeline.db for the cloud audit.
|
|
||||||
- **invariant**: the gloss is never LLM-generated — translations are filled, examples generated, difficulty graded, but the gloss must always be Kaikki's (no gloss → sense dropped in 2b)
|
|
||||||
|
|
||||||
example input:
|
|
||||||
|
|
||||||
```json
|
|
||||||
{
|
|
||||||
"id": "harbor:en:noun:0",
|
|
||||||
"headword": "harbor",
|
|
||||||
"language": "en",
|
|
||||||
"pos": "noun",
|
|
||||||
"glosses": ["A sheltered area of water where ships can dock safely."],
|
|
||||||
"examples": [],
|
|
||||||
"translations": { "de": [{ "word": "Hafen", "gender": "masculine" }] }
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
next substage is the translation generation:
|
|
||||||
|
|
||||||
```json
|
|
||||||
{
|
|
||||||
"id": "harbor:en:noun:0",
|
|
||||||
"headword": "harbor",
|
|
||||||
"language": "en",
|
|
||||||
"pos": "noun",
|
|
||||||
"glosses": ["A sheltered area of water where ships can dock safely."],
|
|
||||||
"examples": [],
|
|
||||||
"translations": {
|
|
||||||
"de": [{ "word": "Hafen", "gender": "masculine" }],
|
|
||||||
"it": [{ "word": "porto", "gender": null }],
|
|
||||||
"es": [{ "word": "puerto", "gender": null }],
|
|
||||||
"fr": [{ "word": "port", "gender": null }]
|
|
||||||
}
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
next substage is the gender modification (by looking up kaikki for it/es/fr or corresponding wiktionary, fill only nulls, not touching existing genders):
|
|
||||||
**runs only after all genders are present**
|
|
||||||
|
|
||||||
```json
|
|
||||||
{
|
|
||||||
"id": "harbor:en:noun:0",
|
|
||||||
"headword": "harbor",
|
|
||||||
"language": "en",
|
|
||||||
"pos": "noun",
|
|
||||||
"glosses": ["A sheltered area of water where ships can dock safely."],
|
|
||||||
"examples": [],
|
|
||||||
"translations": {
|
|
||||||
"de": [{ "word": "Hafen", "gender": "masculine" }],
|
|
||||||
"it": [{ "word": "porto", "gender": "masculine" }],
|
|
||||||
"es": [{ "word": "puerto", "gender": "masculine" }],
|
|
||||||
"fr": [{ "word": "port", "gender": "masculine" }]
|
|
||||||
}
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
next substage is the example generation:
|
|
||||||
|
|
||||||
```json
|
|
||||||
{
|
|
||||||
"id": "harbor:en:noun:0",
|
|
||||||
"headword": "harbor",
|
|
||||||
"language": "en",
|
|
||||||
"pos": "noun",
|
|
||||||
"glosses": ["A sheltered area of water where ships can dock safely."],
|
|
||||||
"examples": ["The fishing boats returned to the harbor at dusk."],
|
|
||||||
"translations": {
|
|
||||||
"de": [{ "word": "Hafen", "gender": "masculine" }],
|
|
||||||
"it": [{ "word": "porto", "gender": "masculine" }],
|
|
||||||
"es": [{ "word": "puerto", "gender": "masculine" }],
|
|
||||||
"fr": [{ "word": "port", "gender": "masculine" }]
|
|
||||||
}
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## step 3
|
|
||||||
|
|
||||||
adding difficulty level
|
|
||||||
|
|
||||||
CEFR is mapped to three buckets:
|
|
||||||
A1/A2 → easy
|
|
||||||
B1/B2 → intermediate
|
|
||||||
C1/C2 → hard
|
|
||||||
|
|
||||||
depending on number of senses per headword:
|
|
||||||
|
|
||||||
- One sense → derive difficulty from the CEFRLex distribution (first CEFR level crossing a frequency threshold, mapped to a bucket, not the peak), for the four covered languages (en/de/es/fr). Deterministic, no LLM
|
|
||||||
- Multiple senses → the three local LLMs grade each sense (easy/intermediate/hard) from the gloss. CEFRLex not involved.
|
|
||||||
- No CEFRLex entry at all (Italian, or single-sense word that's missing) → LLMs grade from gloss
|
|
||||||
|
|
||||||
to get coverage (not quality), 3 local llms are going to be used: gemma, qwen and llama
|
|
||||||
|
|
||||||
on the llm votes:
|
|
||||||
|
|
||||||
- **Majority agrees (3-0 or 2-1)** → ship the majority value.
|
|
||||||
- **3-way split (all three differ)** → no consensus → exclude the card to the review queue. Not shipped.
|
|
||||||
|
|
||||||
example output:
|
|
||||||
|
|
||||||
```json
|
|
||||||
{
|
|
||||||
"id": "house:en:noun:0",
|
|
||||||
"headword": "house",
|
|
||||||
"language": "en",
|
|
||||||
"pos": "noun",
|
|
||||||
"difficulty_level": "easy",
|
|
||||||
"glosses": ["A building for human habitation."],
|
|
||||||
"examples": ["They bought a house in the city."],
|
|
||||||
"translations": {
|
|
||||||
"de": [{ "word": "Haus", "gender": "neuter" }],
|
|
||||||
"it": [{ "word": "casa", "gender": "feminine" }],
|
|
||||||
"es": [{ "word": "casa", "gender": "feminine" }],
|
|
||||||
"fr": [{ "word": "maison", "gender": "feminine" }]
|
|
||||||
}
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"id": "house:en:noun:1",
|
|
||||||
"headword": "house",
|
|
||||||
"language": "en",
|
|
||||||
"pos": "noun",
|
|
||||||
"difficulty_level": "hard",
|
|
||||||
"glosses": ["A noble family or lineage."],
|
|
||||||
"examples": ["The House of Tudor ruled England."],
|
|
||||||
"translations": {
|
|
||||||
"de": [
|
|
||||||
{ "word": "Adelsgeschlecht", "gender": "neuter" },
|
|
||||||
{ "word": "Haus", "gender": "neuter" }
|
|
||||||
]
|
|
||||||
}
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
**important**: `difficulty_level` is mandatory on every shipped card (the user sets difficulty before a game)
|
|
||||||
|
|
||||||
This is coverage, not final quality: three local models make every shipped card's difficulty present and plausible, but not guaranteed correct
|
|
||||||
|
|
||||||
no cefr list: needs to be graded by online llms (see notes)
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## coverage report (trial deliverable)
|
|
||||||
|
|
||||||
- not a data stage => reads the finished cards and summarises them
|
|
||||||
- the trial (english + italian nouns, local models) runs steps 1a→3 then emits a report
|
|
||||||
|
|
||||||
### pipeline health — did it run?
|
|
||||||
|
|
||||||
- input lemmas → output cards (ratio; multi-sense makes cards > lemmas)
|
|
||||||
- dropped per stage + why: 2a misses (junk/inflection/gap), 2b no-gloss drops, 3 no-consensus exclusions
|
|
||||||
- parse failures / errors
|
|
||||||
|
|
||||||
### coverage — how complete?
|
|
||||||
|
|
||||||
- translation completeness: all 4 langs vs gaps, per language
|
|
||||||
- examples: from Kaikki vs LLM-generated vs none
|
|
||||||
- gender: translations still null after Wiktionary fill
|
|
||||||
- difficulty distribution per bucket (lopsided = broken CEFRLex threshold or bad grading)
|
|
||||||
|
|
||||||
### local good enough? — model agreement
|
|
||||||
|
|
||||||
- difficulty votes: 3-0 / 2-1 / 3-way split rates (high 3-way = locals can't do it → need cloud)
|
|
||||||
- translation gap-fill: agreement rate, gaps left unfilled
|
|
||||||
- → go/no-go on local-for-launch
|
|
||||||
|
|
||||||
### rent vs API? — workload volume
|
|
||||||
|
|
||||||
- total LLM calls across 2c + 3 (× per-token rate = API cost)
|
|
||||||
- cards needing LLM vs handled deterministically (english: high deterministic; italian: ~0, no CEFRLex)
|
|
||||||
- per-language call counts → scale english (low) and italian (high) to estimate the middle three
|
|
||||||
|
|
||||||
### review backlog — what's deferred
|
|
||||||
|
|
||||||
- miss-list size + composition, per language
|
|
||||||
- no-consensus exclusions (cloud-audit queue size)
|
|
||||||
- unfilled translation gaps (also cloud-audit work)
|
|
||||||
|
|
||||||
**framing**: english = optimistic floor (rich Kaikki, CEFRLex exists, models strongest).
|
|
||||||
italian = pessimistic ceiling (no CEFRLex, thinner Kaikki). all five languages sit between.
|
|
||||||
read to decide, not to archive.
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
Notes:
|
|
||||||
|
|
||||||
- the miss list needs to get verified/re-worked later on
|
|
||||||
- the kaikki data contains ipas and links to audio files, add them later if needed, they are not needed now
|
|
||||||
- use online llms to set the difficulty(cefr) of the not shipped words
|
|
||||||
- add plurals from kaikki/wiktionary
|
|
||||||
- postgres sync to prod db
|
|
||||||
|
|
@ -1,41 +0,0 @@
|
||||||
# trial run — implementation roadmap (pure vertical)
|
|
||||||
|
|
||||||
goal: english + italian nouns through the full pipeline on local models, emit
|
|
||||||
coverage report. build a thin end-to-end slice first, then widen each pass.
|
|
||||||
|
|
||||||
## phase 0 — foundation
|
|
||||||
|
|
||||||
- confirm @lila/shared types (lang codes, POS) exist and match
|
|
||||||
- package.json deps for what phase 1 needs
|
|
||||||
- one hardcoded test word to carry through the slice (e.g. "house")
|
|
||||||
|
|
||||||
## phase 1 — thin vertical slice (ONE word, 1a→3, crudest possible)
|
|
||||||
|
|
||||||
goal: prove a single card can travel the whole pipeline and come out the end.
|
|
||||||
allowed to be ugly — hardcode, skip voting, one model, fake CEFRLex.
|
|
||||||
|
|
||||||
- 1a: one lemma record, by hand or trivial read
|
|
||||||
- 2a: look it up in Kaikki, confirm it exists
|
|
||||||
- 2b: extract its senses → card(s) with gloss
|
|
||||||
- 2c: fill one missing translation with ONE local model, no voting
|
|
||||||
- 3: assign a difficulty crudely (even hardcoded "easy")
|
|
||||||
- OUT: one finished card. the pipeline has a shape.
|
|
||||||
|
|
||||||
## phase 2 — widen: real deterministic front (1a–2b, all words)
|
|
||||||
|
|
||||||
- 1a: real frequency list, verified
|
|
||||||
- 2a: real existence gate + miss-list triage
|
|
||||||
- 2b: real sense extraction, gender from tags, gloss-required drop
|
|
||||||
- still JSON output. inspect cards by hand.
|
|
||||||
|
|
||||||
## phase 3 — widen: real LLM back (2c + 3)
|
|
||||||
|
|
||||||
- storage seam: JSON → SQLite, schema, resumability
|
|
||||||
- 2c: real generate→vote, three families, all sub-passes
|
|
||||||
- 3: real CEFRLex single-sense + 3-model vote multi-sense + exclude-on-split
|
|
||||||
|
|
||||||
## phase 4 — coverage report + runs
|
|
||||||
|
|
||||||
- emit report
|
|
||||||
- run english, then italian
|
|
||||||
- decide: local good enough? rent vs API?
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue