updating seeding pipeline
This commit is contained in:
lila
parent
c49c2fe2c3
commit
dfeb6a4cb0
2 changed files with 149 additions and 203 deletions
|
|
@ -1,203 +0,0 @@
|
|||
import fs from "node:fs/promises";
|
||||
import { eq } from "drizzle-orm";
|
||||
|
||||
import { SUPPORTED_LANGUAGE_CODES, SUPPORTED_POS } from "@glossa/shared";
|
||||
import { db } from "@glossa/db";
|
||||
import { terms, translations } from "@glossa/db/schema";
|
||||
|
||||
// the following generate unions of the imported const arrays
|
||||
type POS = (typeof SUPPORTED_POS)[number];
|
||||
type LANGUAGE_CODE = (typeof SUPPORTED_LANGUAGE_CODES)[number];
|
||||
|
||||
type Synset = {
|
||||
synset_id: string;
|
||||
pos: POS;
|
||||
translations: Partial<Record<LANGUAGE_CODE, string[]>>;
|
||||
};
|
||||
|
||||
type FileName = {
|
||||
sourceLang: LANGUAGE_CODE;
|
||||
targetLang: LANGUAGE_CODE;
|
||||
pos: POS;
|
||||
};
|
||||
|
||||
const dataDir = "./src/data/datafiles/";
|
||||
|
||||
const parseFilename = (filename: string): FileName => {
|
||||
const parts = filename.replace(".json", "").split("-");
|
||||
if (parts.length !== 3)
|
||||
throw new Error(
|
||||
`Invalid filename format: ${filename}. Expected: sourcelang-targetlang-pos.json`,
|
||||
);
|
||||
const [sourceLang, targetLang, pos] = parts;
|
||||
if (!SUPPORTED_LANGUAGE_CODES.includes(sourceLang as LANGUAGE_CODE))
|
||||
throw new Error(`Unsupported language code: ${sourceLang}`);
|
||||
if (!SUPPORTED_LANGUAGE_CODES.includes(targetLang as LANGUAGE_CODE))
|
||||
throw new Error(`Unsupported language code: ${targetLang}`);
|
||||
if (!SUPPORTED_POS.includes(pos as POS))
|
||||
throw new Error(`Unsupported POS: ${pos}`);
|
||||
return {
|
||||
sourceLang: sourceLang as LANGUAGE_CODE,
|
||||
targetLang: targetLang as LANGUAGE_CODE,
|
||||
pos: pos as POS,
|
||||
};
|
||||
};
|
||||
|
||||
const readFromJsonFile = async (filepath: string): Promise<Synset[]> => {
|
||||
const data = await fs.readFile(filepath, "utf8");
|
||||
const parsed = JSON.parse(data);
|
||||
if (!Array.isArray(parsed)) throw new Error("Expected a JSON array");
|
||||
return parsed as Synset[];
|
||||
};
|
||||
|
||||
const uploadSynsetToDB = async (
|
||||
synset: Synset,
|
||||
_fileInfo: FileName,
|
||||
): Promise<{ termInserted: boolean; translationsInserted: number }> => {
|
||||
// 1. Try to insert the term — skip if synset_id already exists
|
||||
const inserted = await db
|
||||
.insert(terms)
|
||||
.values({ synset_id: synset.synset_id, pos: synset.pos })
|
||||
.onConflictDoNothing()
|
||||
.returning({ id: terms.id });
|
||||
|
||||
let termId: string;
|
||||
let termInserted: boolean;
|
||||
|
||||
if (inserted.length > 0) {
|
||||
termId = inserted[0]!.id;
|
||||
termInserted = true;
|
||||
} else {
|
||||
// Term already exists — fetch its real DB id for the FK
|
||||
const [existing] = await db
|
||||
.select({ id: terms.id })
|
||||
.from(terms)
|
||||
.where(eq(terms.synset_id, synset.synset_id))
|
||||
.limit(1);
|
||||
if (!existing)
|
||||
throw new Error(`Term not found after conflict: ${synset.synset_id}`);
|
||||
termId = existing.id;
|
||||
termInserted = false;
|
||||
}
|
||||
|
||||
// 2. Build translation rows and upsert — skip duplicates silently
|
||||
const translationRows = Object.entries(synset.translations).flatMap(
|
||||
([lang, lemmas]) =>
|
||||
lemmas!.map((lemma) => ({
|
||||
id: crypto.randomUUID(),
|
||||
term_id: termId,
|
||||
language_code: lang as LANGUAGE_CODE,
|
||||
text: lemma,
|
||||
})),
|
||||
);
|
||||
|
||||
if (translationRows.length === 0) {
|
||||
return { termInserted, translationsInserted: 0 };
|
||||
}
|
||||
|
||||
const result = await db
|
||||
.insert(translations)
|
||||
.values(translationRows)
|
||||
.onConflictDoNothing()
|
||||
.returning({ id: translations.id });
|
||||
|
||||
return { termInserted, translationsInserted: result.length };
|
||||
};
|
||||
|
||||
const main = async () => {
|
||||
// step 1: discovering files
|
||||
console.log("\n");
|
||||
console.log("\n");
|
||||
console.log("##########################################");
|
||||
console.log("step 1: discovering files");
|
||||
console.log("##########################################");
|
||||
|
||||
console.log("🔍 Scanning datafiles directory...");
|
||||
const allFiles = await fs.readdir(dataDir);
|
||||
const jsonFiles = allFiles.filter((f) => f.endsWith(".json"));
|
||||
|
||||
if (jsonFiles.length === 0) {
|
||||
console.warn("⚠️ No JSON files found in", dataDir);
|
||||
return;
|
||||
}
|
||||
console.log(`📁 Found ${jsonFiles.length} file(s)\n`);
|
||||
|
||||
// step 2: validating filenames
|
||||
console.log("\n");
|
||||
console.log("\n");
|
||||
console.log("##########################################");
|
||||
console.log("step 2: validating filenames");
|
||||
console.log("##########################################");
|
||||
const validFiles: { filename: string; fileInfo: FileName }[] = [];
|
||||
for (const filename of jsonFiles) {
|
||||
try {
|
||||
const fileInfo = parseFilename(filename);
|
||||
validFiles.push({ filename, fileInfo });
|
||||
console.log(
|
||||
` ✅ ${filename} — ${fileInfo.sourceLang} → ${fileInfo.targetLang} (${fileInfo.pos})`,
|
||||
);
|
||||
} catch (e) {
|
||||
console.warn(` ⚠️ Skipping ${filename}: ${(e as Error).message}`);
|
||||
}
|
||||
}
|
||||
|
||||
if (validFiles.length === 0) {
|
||||
console.error("❌ No valid files to process. Exiting.");
|
||||
return;
|
||||
}
|
||||
|
||||
// step 3: processing each file
|
||||
console.log("\n");
|
||||
console.log("\n");
|
||||
console.log("##########################################");
|
||||
console.log("step 3: processing each file");
|
||||
console.log("##########################################");
|
||||
let totalTermsInserted = 0;
|
||||
let totalTranslationsInserted = 0;
|
||||
|
||||
for (const [i, { filename, fileInfo }] of validFiles.entries()) {
|
||||
const prefix = `[${i + 1}/${validFiles.length}]`;
|
||||
|
||||
console.log(`\n${prefix} 📄 ${filename}`);
|
||||
|
||||
const synsets = await readFromJsonFile(dataDir + filename);
|
||||
console.log(`${prefix} Loaded ${synsets.length} synsets`);
|
||||
|
||||
let fileTermsInserted = 0;
|
||||
let fileTranslationsInserted = 0;
|
||||
|
||||
for (const [j, synset] of synsets.entries()) {
|
||||
if (j > 0 && j % 500 === 0) {
|
||||
console.log(`${prefix} ⏳ ${j}/${synsets.length} synsets processed...`);
|
||||
}
|
||||
|
||||
const { termInserted, translationsInserted } = await uploadSynsetToDB(
|
||||
synset,
|
||||
fileInfo,
|
||||
);
|
||||
if (termInserted) fileTermsInserted++;
|
||||
fileTranslationsInserted += translationsInserted;
|
||||
}
|
||||
|
||||
console.log(
|
||||
`${prefix} ✅ Done — ${fileTermsInserted} new terms, ${fileTranslationsInserted} new translations`,
|
||||
);
|
||||
totalTermsInserted += fileTermsInserted;
|
||||
totalTranslationsInserted += fileTranslationsInserted;
|
||||
}
|
||||
|
||||
// step 4: Final summary
|
||||
console.log("\n");
|
||||
console.log("\n");
|
||||
console.log("##########################################");
|
||||
console.log("step 4: final summary");
|
||||
console.log("##########################################");
|
||||
console.log(`\n🎉 Seeding complete!`);
|
||||
console.log(` Terms inserted: ${totalTermsInserted}`);
|
||||
console.log(` Translations inserted: ${totalTranslationsInserted}`);
|
||||
};
|
||||
|
||||
main().catch((error) => {
|
||||
console.error(error);
|
||||
process.exit(1);
|
||||
});
|
||||
149
scripts/extract-own-save-to-json.py
Normal file
149
scripts/extract-own-save-to-json.py
Normal file
|
|
@ -0,0 +1,149 @@
|
|||
"""
|
||||
scripts/extract-omw-data.py
|
||||
|
||||
Extract ALL synsets from Open Multilingual Wordnet (OMW) for every supported
|
||||
language and POS. Replaces extract-en-it-nouns.py.
|
||||
|
||||
Output: one JSON file per POS, written to packages/db/src/data/datafiles/
|
||||
omw-noun.json
|
||||
omw-verb.json
|
||||
|
||||
Each file is a JSON array of objects matching SynsetRecord in seed.ts:
|
||||
{
|
||||
"source_id": "ili:i12345",
|
||||
"pos": "noun",
|
||||
"translations": { "en": ["dog", "canine"], "it": ["cane"] },
|
||||
"glosses": { "en": ["a domesticated animal..."] }
|
||||
}
|
||||
|
||||
Translations and glosses are absent for a language if that wordnet has no
|
||||
coverage for the synset — the seed script handles sparse data gracefully.
|
||||
|
||||
Usage:
|
||||
python scripts/extract-omw-data.py [output_dir]
|
||||
|
||||
output_dir defaults to packages/db/src/data/datafiles/
|
||||
|
||||
Prerequisites:
|
||||
pip install wn
|
||||
python -c "import wn; wn.download('oewn:2024'); wn.download('omw-it:1.4')"
|
||||
"""
|
||||
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import wn
|
||||
|
||||
# Mirror constants.ts — update both places if languages or POS change.
|
||||
SUPPORTED_LANGUAGE_CODES: list[str] = ["en", "it"]
|
||||
POS_MAP: dict[str, str] = {
|
||||
"n": "noun",
|
||||
"v": "verb",
|
||||
}
|
||||
|
||||
|
||||
def extract_all(output_dir: str = "packages/db/src/data/datafiles/") -> None:
|
||||
out = Path(output_dir)
|
||||
out.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# Load one Wordnet object per language up front.
|
||||
print("Loading wordnets...")
|
||||
wordnets: dict[str, wn.Wordnet] = {}
|
||||
for lang in SUPPORTED_LANGUAGE_CODES:
|
||||
try:
|
||||
wordnets[lang] = wn.Wordnet(lang=lang)
|
||||
synset_count = len(wordnets[lang].synsets())
|
||||
print(f" {lang}: {synset_count:,} total synsets")
|
||||
except wn.Error as e:
|
||||
print(f" ERROR loading {lang}: {e}")
|
||||
print(f" Run: python -c \"import wn; wn.download('omw-{lang}:1.4')\"")
|
||||
sys.exit(1)
|
||||
|
||||
for omw_pos, pos_label in POS_MAP.items():
|
||||
print(f"\n--- Extracting {pos_label}s (pos='{omw_pos}') ---")
|
||||
|
||||
# Collect per-ILI data across all languages.
|
||||
# Structure: { ili -> { lang -> { "lemmas": [...], "glosses": [...] } } }
|
||||
by_ili: dict[str, dict[str, dict[str, list[str]]]] = {}
|
||||
|
||||
for lang, wnet in wordnets.items():
|
||||
synsets = wnet.synsets(pos=omw_pos)
|
||||
covered = 0
|
||||
for synset in synsets:
|
||||
ili = synset.ili
|
||||
if not ili:
|
||||
continue # skip synsets without an ILI — can't cross-link
|
||||
covered += 1
|
||||
if ili not in by_ili:
|
||||
by_ili[ili] = {}
|
||||
|
||||
lemmas = [str(lemma) for lemma in synset.lemmas()]
|
||||
defns = [d for d in synset.definitions() if d]
|
||||
|
||||
by_ili[ili][lang] = {"lemmas": lemmas, "glosses": defns}
|
||||
|
||||
print(f" {lang}: {covered:,} {pos_label} synsets with ILI")
|
||||
|
||||
# Build output records — sort by ILI for a stable, diffable file.
|
||||
records: list[dict] = []
|
||||
for ili in sorted(by_ili.keys()):
|
||||
lang_data = by_ili[ili]
|
||||
translations: dict[str, list[str]] = {}
|
||||
glosses: dict[str, list[str]] = {}
|
||||
|
||||
for lang, data in lang_data.items():
|
||||
if data["lemmas"]:
|
||||
translations[lang] = data["lemmas"]
|
||||
if data["glosses"]:
|
||||
glosses[lang] = data["glosses"]
|
||||
|
||||
# Include the record even if only one language has coverage —
|
||||
# the seed script imports all terms regardless of cross-language overlap.
|
||||
records.append(
|
||||
{
|
||||
"source_id": f"ili:{ili}",
|
||||
"pos": pos_label,
|
||||
"translations": translations,
|
||||
"glosses": glosses,
|
||||
}
|
||||
)
|
||||
|
||||
output_file = out / f"omw-{pos_label}.json"
|
||||
with open(output_file, "w", encoding="utf-8") as f:
|
||||
json.dump(records, f, indent=2, ensure_ascii=False)
|
||||
|
||||
print(f"\nWrote {len(records):,} {pos_label} synsets → {output_file}")
|
||||
_print_coverage(records, pos_label)
|
||||
|
||||
|
||||
def _print_coverage(records: list[dict], pos_label: str) -> None:
|
||||
"""Print per-language translation and gloss counts."""
|
||||
lang_stats: dict[str, dict[str, int]] = {}
|
||||
for lang in SUPPORTED_LANGUAGE_CODES:
|
||||
lang_stats[lang] = {"translations": 0, "glosses": 0}
|
||||
|
||||
for r in records:
|
||||
for lang, lemmas in r["translations"].items():
|
||||
if lang in lang_stats:
|
||||
lang_stats[lang]["translations"] += len(lemmas)
|
||||
for lang, gloss_list in r["glosses"].items():
|
||||
if lang in lang_stats:
|
||||
lang_stats[lang]["glosses"] += len(gloss_list)
|
||||
|
||||
print(f"\nCoverage for {pos_label}s:")
|
||||
for lang, counts in lang_stats.items():
|
||||
t = counts["translations"]
|
||||
g = counts["glosses"]
|
||||
avg_t = t / len(records) if records else 0
|
||||
print(f" {lang}: {t:,} translations ({avg_t:.1f} avg/synset), {g:,} glosses")
|
||||
|
||||
# Sample output
|
||||
print(f"\nSample {pos_label}s (records 1000–1004):")
|
||||
for r in records[1000:1005]:
|
||||
print(f" {r['source_id']}: {r['translations']}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
output_dir = sys.argv[1] if len(sys.argv) > 1 else "packages/db/src/data/datafiles/"
|
||||
extract_all(output_dir)
|
||||
Loading…
Add table
Add a link
Reference in a new issue