Load all reversals in background

Rudimentary queue to handle all reversal loading
Disable letter button if reversal not loaded yet.

Author: FyreByrd

src/lib/data/stores/lexicon.svelte.ts

@@ -1,5 +1,6 @@
 import type { LoadEvent } from '@sveltejs/kit';
 import initSqlJs, { type Database } from 'sql.js';
+import { SvelteMap } from 'svelte/reactivity';
 
 // Store for vernacularLanguage
 export let vernacularLanguageId = $state({ value: '' });
@@ -31,10 +32,14 @@ export type ReversalWord = {
     letter: string;
     homonym_index?: never;
 };
-export let reversalWords: Record<string, ReversalWord[]> = $state({});
 
-// Store for the loaded reversalLetters, keyed by language
-export let reversalLetters: Record<string, string[]> = $state({});
+/**
+ * code -> letter -> file -> loaded
+ */
+export let reversals: SvelteMap<
+    string,
+    SvelteMap<string, SvelteMap<string, ReversalWord[]> | undefined> | undefined
+> = new SvelteMap();
 
 export type Word = VernacularWord | ReversalWord;
 
@@ -91,9 +96,21 @@ class CurrentReversal {
     // Store for selectedLanguageStore
     languageId: string | null = $state(null);
     // Derived store to get the current language's reversalWordsList
-    words = $derived(this.languageId ? reversalWords[this.languageId] || [] : []);
+    words = $derived(
+        this.languageId
+            ? reversals
+                  .get(this.languageId)
+                  ?.values() // = letters for language
+                  .filter((file) => !!file) // filter letters that don't have a file
+                  .flatMap((file) => file?.values().toArray()) // for each letter, flatten word list
+                  .toArray()
+                  .flat() || [] // return flattened array of all words
+            : []
+    );
     // Derived store to get the current language's reversalLetters
-    letters = $derived(new Set(this.languageId ? reversalLetters[this.languageId] || [] : []));
+    letters = $derived(
+        new Set(this.languageId ? reversals.get(this.languageId)?.keys() || [] : [])
+    );
 }
 
 export const currentReversal = new CurrentReversal();

src/lib/lexicon/index.ts

@@ -3,30 +3,41 @@ import config from '$lib/data/config';
 import {
     displayNames,
     initializeDatabase,
+    reversals,
     vernacularLanguageId,
     vernacularWords,
-    type VernacularWord
+    type ReversalWord,
+    type VernacularWord,
+    type VernacularWordReference
 } from '$lib/data/stores/lexicon.svelte';
-import type { ReversalIndex } from '$lib/lexicon';
+import { SvelteMap } from 'svelte/reactivity';
 import type { LayoutLoad } from './$types';
 
-const reversalIndexUrls = import.meta.glob('./**/index.json', {
+const reversalURLs = import.meta.glob('./**/*.json', {
     import: 'default',
     eager: true,
     base: '/src/gen-assets/reversal',
     query: '?url'
 }) as Record<string, string>;
 
+type FetchJob = {
+    code: string;
+    letter: string;
+    file: string;
+};
+
+const reversalQueue: FetchJob[] = [];
+
 export const load: LayoutLoad = async ({ fetch }) => {
     if (!(config as DictionaryConfig).writingSystems) {
         throw new Error('Writing systems configuration not found');
     }
     const dictionaryConfig = config as DictionaryConfig;
 
+    const writingSystems = Object.entries(dictionaryConfig.writingSystems);
+
     const [vernacularLanguage, vernacularWritingSystem] =
-        Object.entries(dictionaryConfig.writingSystems).find(([_, ws]) =>
-            ws.type.includes('main')
-        ) ?? [];
+        writingSystems.find(([_, ws]) => ws.type.includes('main')) ?? [];
 
     if (!(vernacularLanguage && vernacularWritingSystem)) {
         throw new Error('Vernacular language not found');
@@ -41,24 +52,35 @@ export const load: LayoutLoad = async ({ fetch }) => {
 
     const vernacularAlphabet = vernacularWritingSystem.alphabet;
 
-    const reversalWritingSystems = Object.entries(dictionaryConfig.writingSystems).filter(
-        ([_, ws]) => 'reversalFilename' in ws
-    );
-
-    const reversalAlphabets = reversalWritingSystems.map(([key, ws]) => ({ [key]: ws.alphabet }));
-    const reversalLanguages = reversalWritingSystems.map(([key, _]) => key);
+    for (const [code, ws] of writingSystems) {
+        if ('reversalFilename' in ws) {
+            reversals.set(
+                code,
+                ws.alphabet && new SvelteMap(ws.alphabet.map((letter) => [letter, undefined]))
+            );
+        }
+    }
 
-    const reversalIndexes: { [language: string]: ReversalIndex } = {}; // Updated type for reversalIndexes
+    reversalQueue.length = 0;
 
-    for (const [key] of reversalWritingSystems) {
-        const response = await fetch(reversalIndexUrls[`./${key}/index.json`]);
+    for (const [code, reversal] of reversals.entries()) {
+        const response = await fetch(reversalURLs[`./${code}/index.json`]);
         if (response.ok) {
-            reversalIndexes[key] = (await response.json()) as ReversalIndex; // Explicitly cast the JSON response
+            Object.entries((await response.json()) as Promise<Record<string, string[]>>).forEach(
+                ([letter, files]) => {
+                    reversal?.set(letter, new SvelteMap(files.map((f: string) => [f, []])));
+                    reversalQueue.push(...files.map((file: string) => ({ code, letter, file })));
+                }
+            );
         } else {
-            console.warn(`Failed to load reversal index for language: ${key}`);
+            console.warn(`Failed to load reversal index for language: ${code}`);
         }
     }
 
+    console.log(reversals);
+    reversalQueue.sort((a, b) => a.letter.localeCompare(b.letter, 'en-US'));
+    console.log(reversalQueue.slice(0));
+
     let db = await initializeDatabase({ fetch });
     let results = db.exec(`SELECT id, name, homonym_index, type, num_senses, summary FROM entries`);
 
@@ -74,38 +96,86 @@ export const load: LayoutLoad = async ({ fetch }) => {
             ) as VernacularWord;
 
             let firstLetter = entry.name.charAt(0).toLowerCase();
+            const startingPosition = firstLetter === '*' || firstLetter === '-' ? 1 : 0;
 
-            let firstTwoChars = '';
-            let startingPosition = 0;
-
-            if (firstLetter === '*' || firstLetter === '-') {
-                startingPosition = 1;
-            }
-            firstTwoChars = entry.name
+            const firstTwoChars = entry.name
                 .substring(startingPosition, 2 + startingPosition)
                 .toLowerCase();
 
-            if (vernacularAlphabet?.includes(firstTwoChars)) {
-                firstLetter = firstTwoChars;
-            } else {
-                firstLetter = entry.name.charAt(startingPosition).toLowerCase();
-            }
-
-            if (!vernacularAlphabet?.includes(firstLetter)) {
-                firstLetter = '*';
-            }
+            firstLetter = vernacularAlphabet?.includes(firstTwoChars)
+                ? firstTwoChars
+                : entry.name.charAt(startingPosition).toLowerCase();
 
-            entry.letter = firstLetter;
+            entry.letter = vernacularAlphabet?.includes(firstLetter) ? firstLetter : '*';
             return entry;
         });
         vernacularLanguageId.value = vernacularLanguage;
         vernacularWords.value = vernacularWordsList;
     }
 
+    // start loading reversals in background
+    loadReversals();
+
     return {
-        vernacularAlphabet,
-        reversalAlphabets,
-        reversalLanguages,
-        reversalIndexes
+        vernacularAlphabet
     };
 };
+
+async function loadReversals() {
+    const start = new Date().valueOf();
+    while (reversalQueue.length) {
+        const job = reversalQueue.shift();
+        if (job) {
+            await loadReversal(job);
+        }
+    }
+    console.log(`Loaded all reversals in ${(new Date().valueOf() - start) / 1000}s`);
+}
+
+async function loadReversal(job: FetchJob) {
+    const { code, letter, file } = job;
+
+    const key = `./${code}/${file}`;
+
+    const reversalFile = reversalURLs[key];
+    if (!reversalFile) {
+        console.error(`Reversal file not found in glob: ${key}`);
+        return;
+    }
+
+    const response = await fetch(reversalFile);
+    if (response.ok) {
+        const data: Record<string, { index: number; name: 'string' }[]> = await response.json();
+        reversals
+            .get(code)
+            ?.get(letter)
+            ?.set(
+                file,
+                Object.entries(data).map(
+                    ([name, entries]) =>
+                        ({
+                            name,
+                            indexes: entries.map((entry) => entry.index),
+                            vernacularWords: entries
+                                .map((entry) => {
+                                    const foundWord: VernacularWord | undefined =
+                                        vernacularWords.value.find((vw) => vw.id === entry.index);
+                                    if (foundWord) {
+                                        return {
+                                            name: foundWord.name,
+                                            homonym_index: foundWord.homonym_index || 0
+                                        } satisfies VernacularWordReference;
+                                    } else {
+                                        console.log(
+                                            `Index ${entry.index} not found in vernacularWords`
+                                        );
+                                        return null; // Return null for missing indexes
+                                    }
+                                })
+                                .filter((index) => index !== null), // Filter out null values
+                            letter: letter
+                        }) satisfies ReversalWord
+                )
+            );
+    }
+}