wip

data-pipeline/utils/scanning-source-files.ts

@@ -0,0 +1,61 @@
+import fs from "fs";
+import path from "path";
+
+// Define a simple shape for what a discovered dataset looks like
+export interface Wordlist {
+  language: string;
+  pos: string;
+  sourcePath: string;
+  outputDir: string;
+}
+
+/**
+ * Scans the source-data directory to find all available word lists.
+ */
+export function scanSourceData(baseDir: string): Wordlist[] {
+  const sourceBaseDir = path.join(baseDir, "source-data");
+  const discoveredWordlists: Wordlist[] = [];
+
+  // Safety check: if there's no source-data folder, return an empty array
+  if (!fs.existsSync(sourceBaseDir)) {
+    return discoveredWordlists;
+  }
+
+  // 1. Read the language directories (e.g., ['english'])
+  const languages = fs.readdirSync(sourceBaseDir);
+
+  for (const lang of languages) {
+    const langFolderPath = path.join(sourceBaseDir, lang);
+
+    // Make sure it's a directory, not a stray file
+    if (!fs.statSync(langFolderPath).isDirectory()) continue;
+
+    // 2. Read the files inside the language folder (e.g., ['nouns'])
+    const posFiles = fs.readdirSync(langFolderPath);
+
+    for (const pos of posFiles) {
+      const fullSourcePath = path.join(langFolderPath, pos);
+
+      // Make sure it's a file (like your extensionless "nouns" file)
+      if (!fs.statSync(fullSourcePath).isFile()) continue;
+
+      // 3. Package everything into a flat item and add it to our array
+      discoveredWordlists.push({
+        language: lang,
+        pos: pos,
+        sourcePath: fullSourcePath,
+        outputDir: path.join(baseDir, "worddata", lang, pos),
+      });
+    }
+  }
+
+  // show summary
+  console.log(
+    `✅ Scan complete! Found ${discoveredWordlists.length} wordlist(s):`,
+  );
+  for (const list of discoveredWordlists) {
+    console.log(`   • ${list.language.toUpperCase()} (${list.pos})`);
+  }
+
+  return discoveredWordlists;
+}