lila/data-pipeline/sourceLists.ts

import { readdirSync, readFileSync } from "node:fs";
import path from "node:path";
import {
  SUPPORTED_LANGUAGE_CODES,
  SUPPORTED_POS,
  type SupportedLanguageCode,
  type SupportedPos,
} from "@lila/shared";

export type SourceList = {
  sourceLanguage: SupportedLanguageCode;
  pos: SupportedPos;
  words: string[];
  filePath: string;
};

const isLanguageCode = (value: string): value is SupportedLanguageCode =>
  (SUPPORTED_LANGUAGE_CODES as readonly string[]).includes(value);

const isPos = (value: string): value is SupportedPos =>
  (SUPPORTED_POS as readonly string[]).includes(value);

export const normalizeWords = (lines: readonly string[]): string[] => {
  const seen = new Set<string>();
  const words: string[] = [];
  for (const line of lines) {
    const word = line.trim();
    if (word === "" || seen.has(word)) continue;
    seen.add(word);
    words.push(word);
  }
  return words;
};

export const discoverSourceLists = (rootDir: string): SourceList[] => {
  const lists: SourceList[] = [];
  const languageDirs = readdirSync(rootDir, { withFileTypes: true })
    .filter((entry) => entry.isDirectory() && isLanguageCode(entry.name))
    .map((entry) => entry.name as SupportedLanguageCode)
    .sort();

  for (const language of languageDirs) {
    const languageDir = path.join(rootDir, language);
    const posFiles = readdirSync(languageDir, { withFileTypes: true })
      .filter((entry) => entry.isFile() && isPos(entry.name))
      .map((entry) => entry.name as SupportedPos)
      .sort();

    for (const pos of posFiles) {
      const filePath = path.join(languageDir, pos);
      const words = normalizeWords(readFileSync(filePath, "utf-8").split("\n"));
      lists.push({ sourceLanguage: language, pos, words, filePath });
    }
  }
  return lists;
};