removing not needed files
This commit is contained in:
lila
parent
597083e1fd
commit
88b16a1ed7
15 changed files with 80 additions and 555 deletions
|
|
@ -1,16 +0,0 @@
|
|||
export const LANG_MAP: Record<string, string> = {
|
||||
english: "en",
|
||||
italian: "it",
|
||||
german: "de",
|
||||
french: "fr",
|
||||
spanish: "es",
|
||||
};
|
||||
|
||||
export const POS_MAP: Record<string, string> = {
|
||||
nouns: "noun",
|
||||
verbs: "verb",
|
||||
adverbs: "adverb",
|
||||
adjectives: "adjective",
|
||||
};
|
||||
|
||||
export const ALL_LANGUAGES = ["en", "de", "it", "es", "fr"];
|
||||
|
|
@ -1,43 +0,0 @@
|
|||
export function buildSystemPrompt(
|
||||
sourceLanguage: string,
|
||||
pos: string,
|
||||
targetLanguages: string[],
|
||||
): string {
|
||||
return `You are a multilingual dictionary engine. Output ONLY a JSON object. No markdown, no explanations.
|
||||
|
||||
For each ${sourceLanguage} ${pos} provided, generate 1-2 distinct senses.
|
||||
|
||||
CEFR difficulty mapping:
|
||||
- A1/A2 → easy
|
||||
- B1/B2 → medium
|
||||
- C1/C2 → hard
|
||||
|
||||
Each sense must have:
|
||||
- sense: student-friendly definition, max 15 words
|
||||
- example: natural sentence using the word
|
||||
- difficulty_level: easy, medium, or hard
|
||||
- translations: object with keys ${targetLanguages.join(", ")}; each value is an array of {word, gender} where gender MUST be masculine, feminine, or neuter. Use null ONLY if the language has no grammatical gender for that word.
|
||||
|
||||
Output format: JSON object where keys are the input words, values are arrays of sense objects.
|
||||
|
||||
Example for ["house"]:
|
||||
{
|
||||
"house": [
|
||||
{
|
||||
"sense": "A building for human habitation.",
|
||||
"example": "They bought a house in the city.",
|
||||
"difficulty_level": "easy",
|
||||
"translations": {
|
||||
"de": [{"word": "Haus", "gender": "neuter"}],
|
||||
"it": [{"word": "casa", "gender": "feminine"}],
|
||||
"es": [{"word": "casa", "gender": "feminine"}],
|
||||
"fr": [{"word": "maison", "gender": "feminine"}]
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
/no-think
|
||||
|
||||
`;
|
||||
}
|
||||
|
|
@ -4,11 +4,6 @@
|
|||
"private": true,
|
||||
"type": "module",
|
||||
"scripts": {
|
||||
"db:reset": "tsx db/reset.ts",
|
||||
"extract": "tsx stage-1-extract/scripts/extract.ts",
|
||||
"reverse-link": "tsx stage-2-reverse-link/scripts/reverse-link.ts",
|
||||
"db:import": "tsx db/import.ts",
|
||||
"db:init": "tsx db/init.ts",
|
||||
"test": "vitest run",
|
||||
"test:watch": "vitest",
|
||||
"pipeline:run": "tsx --env-file .env pipeline.ts"
|
||||
|
|
|
|||
0
data-pipeline/pipeline.ts
Normal file
0
data-pipeline/pipeline.ts
Normal file
20
data-pipeline/source-data/french/adjectives
Normal file
20
data-pipeline/source-data/french/adjectives
Normal file
|
|
@ -0,0 +1,20 @@
|
|||
grand
|
||||
petit
|
||||
bon
|
||||
mauvais
|
||||
beau
|
||||
nouveau
|
||||
vieux
|
||||
jeune
|
||||
heureux
|
||||
triste
|
||||
fort
|
||||
faible
|
||||
rapide
|
||||
lent
|
||||
chaud
|
||||
froid
|
||||
facile
|
||||
difficile
|
||||
propre
|
||||
sale
|
||||
20
data-pipeline/source-data/german/verbs
Normal file
20
data-pipeline/source-data/german/verbs
Normal file
|
|
@ -0,0 +1,20 @@
|
|||
sein
|
||||
haben
|
||||
werden
|
||||
können
|
||||
müssen
|
||||
sagen
|
||||
machen
|
||||
geben
|
||||
kommen
|
||||
gehen
|
||||
wissen
|
||||
sehen
|
||||
lassen
|
||||
stehen
|
||||
finden
|
||||
bleiben
|
||||
liegen
|
||||
heißen
|
||||
denken
|
||||
nehmen
|
||||
20
data-pipeline/source-data/italian/adverbs
Normal file
20
data-pipeline/source-data/italian/adverbs
Normal file
|
|
@ -0,0 +1,20 @@
|
|||
bene
|
||||
male
|
||||
sempre
|
||||
mai
|
||||
spesso
|
||||
raramente
|
||||
oggi
|
||||
domani
|
||||
ieri
|
||||
qui
|
||||
lì
|
||||
molto
|
||||
poco
|
||||
troppo
|
||||
abbastanza
|
||||
velocemente
|
||||
lentamente
|
||||
insieme
|
||||
forse
|
||||
davvero
|
||||
20
data-pipeline/source-data/spanish/nouns
Normal file
20
data-pipeline/source-data/spanish/nouns
Normal file
|
|
@ -0,0 +1,20 @@
|
|||
mesa
|
||||
silla
|
||||
coche
|
||||
perro
|
||||
gato
|
||||
ventana
|
||||
puerta
|
||||
calle
|
||||
plaza
|
||||
mercado
|
||||
parque
|
||||
río
|
||||
montaña
|
||||
playa
|
||||
sol
|
||||
luna
|
||||
estrella
|
||||
cielo
|
||||
tierra
|
||||
árbol
|
||||
|
|
@ -1,11 +0,0 @@
|
|||
import fs from "fs";
|
||||
import readline from "readline";
|
||||
|
||||
/**
|
||||
* Creates a line-by-line reader stream for a given file path.
|
||||
*/
|
||||
export function createLineReader(sourcePath: string): readline.Interface {
|
||||
const fileStream = fs.createReadStream(sourcePath, "utf-8");
|
||||
|
||||
return readline.createInterface({ input: fileStream, crlfDelay: Infinity });
|
||||
}
|
||||
|
|
@ -1,20 +0,0 @@
|
|||
import fs from "fs";
|
||||
import type { Wordlist } from "./scanning-source-files.js";
|
||||
|
||||
/**
|
||||
* Takes a list of scanned datasets and creates their output folders if missing.
|
||||
*/
|
||||
export function ensureOutputFolders(wordlists: Wordlist[]): void {
|
||||
for (const wordlist of wordlists) {
|
||||
if (!fs.existsSync(wordlist.outputDir)) {
|
||||
fs.mkdirSync(wordlist.outputDir, { recursive: true });
|
||||
console.log(
|
||||
`📁 Created target folder: worddata/${wordlist.language}/${wordlist.pos}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
console.log(
|
||||
"✅ All required output directories have been verified and created successfully.",
|
||||
);
|
||||
}
|
||||
|
|
@ -1,5 +0,0 @@
|
|||
import path from "path";
|
||||
|
||||
export function getWordFilePath(word: string, outputDir: string): string {
|
||||
return path.join(outputDir, `${word}.json`);
|
||||
}
|
||||
|
|
@ -1,61 +0,0 @@
|
|||
import fs from "fs";
|
||||
import path from "path";
|
||||
|
||||
// Define a simple shape for what a discovered dataset looks like
|
||||
export interface Wordlist {
|
||||
language: string;
|
||||
pos: string;
|
||||
sourcePath: string;
|
||||
outputDir: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Scans the source-data directory to find all available word lists.
|
||||
*/
|
||||
export function scanSourceData(baseDir: string): Wordlist[] {
|
||||
const sourceBaseDir = path.join(baseDir, "source-data");
|
||||
const discoveredWordlists: Wordlist[] = [];
|
||||
|
||||
// Safety check: if there's no source-data folder, return an empty array
|
||||
if (!fs.existsSync(sourceBaseDir)) {
|
||||
return discoveredWordlists;
|
||||
}
|
||||
|
||||
// 1. Read the language directories (e.g., ['english'])
|
||||
const languages = fs.readdirSync(sourceBaseDir);
|
||||
|
||||
for (const lang of languages) {
|
||||
const langFolderPath = path.join(sourceBaseDir, lang);
|
||||
|
||||
// Make sure it's a directory, not a stray file
|
||||
if (!fs.statSync(langFolderPath).isDirectory()) continue;
|
||||
|
||||
// 2. Read the files inside the language folder (e.g., ['nouns'])
|
||||
const posFiles = fs.readdirSync(langFolderPath);
|
||||
|
||||
for (const pos of posFiles) {
|
||||
const fullSourcePath = path.join(langFolderPath, pos);
|
||||
|
||||
// Make sure it's a file (like your extensionless "nouns" file)
|
||||
if (!fs.statSync(fullSourcePath).isFile()) continue;
|
||||
|
||||
// 3. Package everything into a flat item and add it to our array
|
||||
discoveredWordlists.push({
|
||||
language: lang,
|
||||
pos: pos,
|
||||
sourcePath: fullSourcePath,
|
||||
outputDir: path.join(baseDir, "worddata", lang, pos),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// show summary
|
||||
console.log(
|
||||
`✅ Scan complete! Found ${discoveredWordlists.length} wordlist(s):`,
|
||||
);
|
||||
for (const list of discoveredWordlists) {
|
||||
console.log(` • ${list.language.toUpperCase()} (${list.pos})`);
|
||||
}
|
||||
|
||||
return discoveredWordlists;
|
||||
}
|
||||
|
|
@ -1,11 +0,0 @@
|
|||
import fs from "fs";
|
||||
|
||||
/**
|
||||
* Writes data as formatted JSON to a file path.
|
||||
* Safely catches and re-throws file system errors.
|
||||
*/
|
||||
export function writeJsonFile(filePath: string, data: unknown): void {
|
||||
const tempPath = `${filePath}.tmp`;
|
||||
fs.writeFileSync(tempPath, JSON.stringify(data, null, 2), "utf-8");
|
||||
fs.renameSync(tempPath, filePath);
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue