forgejo-lila/lila
1
0
Fork 0
Code Issues 50 Pull requests Projects 1 Releases Packages Wiki Activity Actions

wip

Browse source
This commit is contained in:
lila 2026-06-03 00:02:19 +02:00
parent 0118798e36
commit cc89f0c75c
20 changed files with 384 additions and 5600 deletions
Download patch file Download diff file Expand all files Collapse all files

154
data-pipeline/db/import.ts
View file

@ -1,154 +0,0 @@
import fs from "node:fs/promises";
import path from "node:path";
import { fileURLToPath } from "node:url";
import { SUPPORTED_LANGUAGE_CODES } from "@lila/shared";
import { openDb } from "./index.js";
import type { ExtractedSense } from "../stage-1-extract/scripts/extract.js";
// ── Paths ─────────────────────────────────────────────────────────────────────
const __dirname = path.dirname(fileURLToPath(import.meta.url));
const OUTPUT_DIR = path.resolve(__dirname, "../stage-1-extract/output");
// ── Import ────────────────────────────────────────────────────────────────────
export async function importKaikki(): Promise<void> {
const db = openDb();
const insertEntry = db.prepare(`
INSERT INTO entries (headword, language, pos, sense_index, gloss, examples)
VALUES (?, ?, ?, ?, ?, ?)
ON CONFLICT (headword, language, pos, sense_index)
DO UPDATE SET
gloss = excluded.gloss,
examples = excluded.examples
RETURNING id
`);
const insertTranslation = db.prepare(`
INSERT INTO translations (entry_id, target_lang, word, sense_hint)
VALUES (?, ?, ?, ?)
ON CONFLICT (entry_id, target_lang, word) DO NOTHING
`);
let totalEntries = 0;
let totalTranslations = 0;
let totalSkipped = 0;
for (const lang of SUPPORTED_LANGUAGE_CODES) {
const filePath = path.join(OUTPUT_DIR, `${lang}.json`);
let senses: ExtractedSense[];
try {
const raw = await fs.readFile(filePath, "utf-8");
senses = JSON.parse(raw) as ExtractedSense[];
} catch {
console.warn(` Warning: no output file found for ${lang}, skipping`);
continue;
}
console.log(
` Importing ${lang}: ${senses.length.toLocaleString()} senses...`,
);
// Track next available sense_index per (headword, pos) to handle
// the same word appearing in multiple JSONL entries with the same POS.
const senseIndexMap = new Map<string, number>();
const importLang = db.transaction(() => {
let entries = 0;
let translations = 0;
let skipped = 0;
for (const sense of senses) {
const key = `${sense.headword}|${sense.pos}`;
const nextIndex = senseIndexMap.get(key) ?? 0;
senseIndexMap.set(key, nextIndex + 1);
const row = insertEntry.get(
sense.headword,
sense.language,
sense.pos,
nextIndex,
sense.gloss ?? null,
JSON.stringify(sense.examples),
) as { id: number } | undefined;
if (!row) {
skipped++;
continue;
}
entries++;
for (const t of sense.translations) {
insertTranslation.run(
row.id,
t.target_lang,
t.word,
t.sense_hint ?? null,
);
translations++;
}
}
return { entries, translations, skipped };
});
const counts = importLang();
totalEntries += counts.entries;
totalTranslations += counts.translations;
totalSkipped += counts.skipped;
console.log(
` entries: ${counts.entries.toLocaleString()}, translations: ${counts.translations.toLocaleString()}, skipped: ${counts.skipped.toLocaleString()}`,
);
}
db.close();
console.log(`\nImport complete:`);
console.log(` Total entries: ${totalEntries.toLocaleString()}`);
console.log(` Total translations: ${totalTranslations.toLocaleString()}`);
console.log(` Total skipped: ${totalSkipped.toLocaleString()}`);
}
// ── Check if already imported ─────────────────────────────────────────────────
export function isImported(): boolean {
const db = openDb();
const row = db.prepare("SELECT COUNT(*) as count FROM entries").get() as {
count: number;
};
db.close();
return row.count > 0;
}
// ── Main ─────────────────────────────────────────────────────────────────────
async function main(): Promise<void> {
const db = openDb();
const row = db.prepare("SELECT COUNT(*) as count FROM entries").get() as {
count: number;
};
db.close();
if (row.count > 0) {
console.log(
`pipeline.db already contains ${row.count.toLocaleString()} entries — skipping import.`,
);
console.log("Delete pipeline.db and re-run db:init to start fresh.");
process.exit(0);
}
console.log("Importing Kaikki data into pipeline.db...");
await importKaikki();
}
if (import.meta.url === `file://${process.argv[1]}`) {
main().catch((err) => {
console.error(err);
process.exit(1);
});
}

24
data-pipeline/db/index.ts
View file

@ -1,24 +0,0 @@
import path from "node:path";
import { fileURLToPath } from "node:url";
import Database from "better-sqlite3";
// ── Paths ─────────────────────────────────────────────────────────────────────
const __dirname = path.dirname(fileURLToPath(import.meta.url));
const DB_PATH = path.join(__dirname, "pipeline.db");
// ── Types ─────────────────────────────────────────────────────────────────────
export type Db = InstanceType<typeof Database>;
// ── Open ──────────────────────────────────────────────────────────────────────
export function openDb(): Db {
const db = new Database(DB_PATH);
db.pragma("journal_mode = WAL");
db.pragma("foreign_keys = ON");
return db;
}

42
data-pipeline/db/init.ts
View file

@ -1,42 +0,0 @@
import fs from "node:fs/promises";
import path from "node:path";
import { fileURLToPath } from "node:url";
import Database from "better-sqlite3";
// ── Paths ─────────────────────────────────────────────────────────────────────
const __dirname = path.dirname(fileURLToPath(import.meta.url));
const PATHS = {
schema: path.join(__dirname, "schema.sql"),
db: path.join(__dirname, "pipeline.db"),
};
// ── Init ──────────────────────────────────────────────────────────────────────
export async function initDb(): Promise<void> {
const schema = await fs.readFile(PATHS.schema, "utf-8");
const db = new Database(PATHS.db);
db.pragma("journal_mode = WAL");
db.pragma("foreign_keys = ON");
db.exec(schema);
db.close();
console.log(` pipeline.db initialised → ${PATHS.db}`);
}
// ── Main ─────────────────────────────────────────────────────────────────────
async function main(): Promise<void> {
console.log("Initialising pipeline.db...");
await initDb();
}
// after
if (import.meta.url === `file://${process.argv[1]}`) {
main().catch((err) => {
console.error(err);
process.exit(1);
});
}

BIN
data-pipeline/db/pipeline.db-shm
View file

Binary file not shown.

0
data-pipeline/db/pipeline.db-wal
View file

41
data-pipeline/db/reset.ts
View file

@ -1,41 +0,0 @@
import path from "node:path";
import { fileURLToPath } from "node:url";
import Database from "better-sqlite3";
// ── Paths ─────────────────────────────────────────────────────────────────────
const __dirname = path.dirname(fileURLToPath(import.meta.url));
const DB_PATH = path.join(__dirname, "pipeline.db");
// ── Main ──────────────────────────────────────────────────────────────────────
function main(): void {
const mode = process.argv[2];
if (!mode || (mode !== "round1" && mode !== "all")) {
console.error("Usage: pnpm db:reset round1 | all");
console.error(" round1 — delete all round1 sub-stage rows");
console.error(" all — delete all run_status rows except reverse_link");
process.exit(1);
}
const db = new Database(DB_PATH);
let result: { changes: number };
if (mode === "round1") {
result = db
.prepare("DELETE FROM run_status WHERE stage LIKE 'round1%'")
.run();
console.log(`Deleted ${result.changes} round1 rows from run_status`);
} else {
result = db
.prepare("DELETE FROM run_status WHERE stage NOT IN ('reverse_link')")
.run();
console.log(`Deleted ${result.changes} rows from run_status`);
}
db.close();
}
main();

164
data-pipeline/db/schema.sql
View file

@ -1,164 +0,0 @@
-- ── Base data ─────────────────────────────────────────────────────────────────
-- Imported from Kaikki on first run. Never mutated after import.
CREATE TABLE IF NOT EXISTS entries (
id INTEGER PRIMARY KEY,
headword TEXT NOT NULL,
language TEXT NOT NULL,
pos TEXT NOT NULL,
sense_index INTEGER NOT NULL DEFAULT 0,
gloss TEXT,
examples TEXT NOT NULL DEFAULT '[]', -- JSON array of strings
source TEXT NOT NULL DEFAULT 'kaikki',
UNIQUE (headword, language, pos, sense_index)
);
CREATE TABLE IF NOT EXISTS translations (
id INTEGER PRIMARY KEY,
entry_id INTEGER NOT NULL REFERENCES entries(id),
target_lang TEXT NOT NULL,
word TEXT NOT NULL,
sense_hint TEXT,
source TEXT NOT NULL DEFAULT 'kaikki',
UNIQUE (entry_id, target_lang, word)
);
-- ── Status tracking ───────────────────────────────────────────────────────────
-- One row per entry per model per stage. Drives resumability.
-- Sentinel rows use entry_id = 0 for one-time pipeline steps.
-- stage: round1 | round2 | tiebreak
-- status: pending | complete | needs_review | flagged
CREATE TABLE IF NOT EXISTS run_status (
id INTEGER PRIMARY KEY,
entry_id INTEGER NOT NULL,
model_name TEXT NOT NULL,
stage TEXT NOT NULL,
status TEXT NOT NULL,
created_at TEXT NOT NULL DEFAULT (datetime('now')),
updated_at TEXT NOT NULL DEFAULT (datetime('now')),
UNIQUE (entry_id, model_name, stage)
);
-- ── Round 1 output ────────────────────────────────────────────────────────────
-- Written atomically per entry per model.
-- Unique constraints enforce one model one vote.
CREATE TABLE IF NOT EXISTS model_entry_cefr_votes (
id INTEGER PRIMARY KEY,
entry_id INTEGER NOT NULL REFERENCES entries(id),
model_name TEXT NOT NULL,
cefr_level TEXT NOT NULL,
UNIQUE (entry_id, model_name)
);
CREATE TABLE IF NOT EXISTS model_translation_cefr_votes (
id INTEGER PRIMARY KEY,
translation_id INTEGER NOT NULL REFERENCES translations(id),
model_name TEXT NOT NULL,
cefr_level TEXT NOT NULL,
UNIQUE (translation_id, model_name)
);
CREATE TABLE IF NOT EXISTS model_translation_rejections (
id INTEGER PRIMARY KEY,
translation_id INTEGER NOT NULL REFERENCES translations(id),
model_name TEXT NOT NULL,
UNIQUE (translation_id, model_name)
);
CREATE TABLE IF NOT EXISTS generated_glosses (
id INTEGER PRIMARY KEY,
entry_id INTEGER NOT NULL REFERENCES entries(id),
model_name TEXT NOT NULL,
text TEXT NOT NULL,
UNIQUE (entry_id, model_name)
);
CREATE TABLE IF NOT EXISTS generated_examples (
id INTEGER PRIMARY KEY,
entry_id INTEGER NOT NULL REFERENCES entries(id),
model_name TEXT NOT NULL,
text TEXT NOT NULL,
UNIQUE (entry_id, model_name)
);
CREATE TABLE IF NOT EXISTS generated_translations (
id INTEGER PRIMARY KEY,
entry_id INTEGER NOT NULL REFERENCES entries(id),
model_name TEXT NOT NULL,
target_lang TEXT NOT NULL,
word TEXT NOT NULL,
UNIQUE (entry_id, model_name, target_lang)
);
-- ── Round 2 output ────────────────────────────────────────────────────────────
-- Each row represents one model voting for one candidate.
-- The candidate with the most votes wins in merge.
CREATE TABLE IF NOT EXISTS gloss_candidate_votes (
id INTEGER PRIMARY KEY,
gloss_id INTEGER NOT NULL REFERENCES generated_glosses(id),
model_name TEXT NOT NULL,
UNIQUE (gloss_id, model_name)
);
CREATE TABLE IF NOT EXISTS example_candidate_votes (
id INTEGER PRIMARY KEY,
example_id INTEGER NOT NULL REFERENCES generated_examples(id),
model_name TEXT NOT NULL,
UNIQUE (example_id, model_name)
);
CREATE TABLE IF NOT EXISTS translation_candidate_votes (
id INTEGER PRIMARY KEY,
translation_id INTEGER NOT NULL REFERENCES generated_translations(id),
model_name TEXT NOT NULL,
UNIQUE (translation_id, model_name)
);
-- ── Resolved output ───────────────────────────────────────────────────────────
-- Written by merge. Never updated after writing.
-- Only fully resolved records are written here — no nulls.
-- Absence of a row means unresolved. Flagged status tracked in run_status.
-- source: kaikki | model_name
CREATE TABLE IF NOT EXISTS resolved_entry_cefr (
id INTEGER PRIMARY KEY,
entry_id INTEGER NOT NULL REFERENCES entries(id),
cefr_level TEXT NOT NULL,
difficulty TEXT NOT NULL,
UNIQUE (entry_id)
);
CREATE TABLE IF NOT EXISTS resolved_translation_cefr (
id INTEGER PRIMARY KEY,
translation_id INTEGER NOT NULL REFERENCES translations(id),
cefr_level TEXT NOT NULL,
difficulty TEXT NOT NULL,
UNIQUE (translation_id)
);
CREATE TABLE IF NOT EXISTS resolved_glosses (
id INTEGER PRIMARY KEY,
entry_id INTEGER NOT NULL REFERENCES entries(id),
text TEXT NOT NULL,
source TEXT NOT NULL,
UNIQUE (entry_id)
);
CREATE TABLE IF NOT EXISTS resolved_examples (
id INTEGER PRIMARY KEY,
entry_id INTEGER NOT NULL REFERENCES entries(id),
text TEXT NOT NULL,
source TEXT NOT NULL
);
CREATE TABLE IF NOT EXISTS resolved_generated_translations (
id INTEGER PRIMARY KEY,
entry_id INTEGER NOT NULL REFERENCES entries(id),
target_lang TEXT NOT NULL,
word TEXT NOT NULL,
source TEXT NOT NULL,
UNIQUE (entry_id, target_lang)
);