migration

.firebaserc

@@ -1,6 +1,6 @@
 {
   "projects": {
     "default": "talking-dictionaries-dev",
-    "production": "talking-dictionaries-alpha"
+    "production": "talking-dictionaries-alpha",
   }
 }

.github/workflows/lighthouse-audit.yml

@@ -21,7 +21,7 @@ jobs:
         with:
           urls: |
             ${{ github.event.deployment_status.target_url }}
-            ${{ github.event.deployment_status.target_url }}/achi/entries/list
+            ${{ github.event.deployment_status.target_url }}/achi/entries
             ${{ github.event.deployment_status.target_url }}/jaRhn6MAZim4Blvr1iEv/entry/yt9ja7ymh9xgba5i
           # configPath: './.github/lighthouserc.json' # https://github.com/GoogleChrome/lighthouse-ci/blob/main/docs/getting-started.md https://github.com/GoogleChrome/lighthouse-ci/blob/main/docs/configuration.md#assert
           uploadArtifacts: true

.github/workflows/lint.yml

@@ -11,11 +11,11 @@ jobs:
 
       - uses: pnpm/action-setup@v2
         with:
-          version: 8.6.0
+          version: 9.6.0
 
       - uses: actions/setup-node@v3
         with:
-          node-version: 18
+          node-version: 20
           cache: pnpm
 
       - run: pnpm install

.vscode/settings.json

@@ -33,9 +33,9 @@
   },
   // Silence stylistic rules in IDE, but still auto fix them
   "eslint.rules.customizations": [
-    { "rule": "@stylistic/*", "severity": "off" },
-    { "rule": "style*", "severity": "off" },
-    { "rule": "*indent", "severity": "off" },
+    { "rule": "style/*", "severity": "off" },
+    { "rule": "format/*", "severity": "off" },
+    { "rule": "*-indent", "severity": "off" },
     { "rule": "*-spacing", "severity": "off" },
     { "rule": "*-spaces", "severity": "off" },
     { "rule": "*-order", "severity": "off" },
@@ -47,6 +47,7 @@
     { "rule": "import/order", "severity": "off" },
     { "rule": "sort-imports", "severity": "off" },
     { "rule": "ts/no-empty-function", "severity": "off" },
+    { "rule": "svelte/indent", "severity": "off" },
   ],
   "eslint.validate": [
     "javascript",

README.md

@@ -7,22 +7,12 @@ A mobile-first community focused dictionary-building web app built by [Living To
 [<img src="https://img.shields.io/badge/Svelte-4-orange.svg"></a>](https://svelte.dev/)
 [<img src="https://img.shields.io/badge/SvelteKit-2-orange.svg"></a>](https://kit.svelte.dev/)
 [<img src="https://img.shields.io/badge/UnoCSS-Svelte_Scoped-blue.svg"></a>](https://unocss.dev/integrations/svelte-scoped)
-[<img src="https://img.shields.io/badge/Supabase-teal.svg"></a>](https://firebase.google.com/)
-[<img src="https://img.shields.io/badge/Firebase_(deprecated)-9-orange.svg"></a>](https://firebase.google.com/)
+[<img src="https://img.shields.io/badge/Supabase-Auth_Database-teal.svg"></a>](https://firebase.google.com/)
 [<img src="https://img.shields.io/badge/Vercel-SSR-black.svg"></a>](https://vercel.com/)
-[<img src="https://img.shields.io/badge/Algolia-Instantsearch.js-blue.svg"></a>](https://www.algolia.com/)
-
-Firebase is used for:
-
-- Authentication
-- Cloud Firestore
-- Storage
-
-These functions are being transitioned to Supabase with the exception of Storage, images will remain in GCP. Supabase may support this directly in the future.
+[<img src="https://img.shields.io/badge/Orama-Search-purple.svg"></a>](https://www.orama.com/)
+[<img src="https://img.shields.io/badge/GCP-Storage-blue.svg"></a>](https://cloud.google.com/storage)
 
 ## Contributing
 
-- Choose an already approved task from the [Development Roadmap](https://github.com/jwrunner/Living-Dictionaries/projects/1) or [create an issue](https://github.com/jwrunner/Living-Dictionaries/issues) to propose a new feature (please await discussion before creating a pull request). 
+- Choose an already approved task from the [Development Roadmap](https://github.com/jwrunner/Living-Dictionaries/projects/1) or [create an issue](https://github.com/jwrunner/Living-Dictionaries/issues) to propose a new feature (please await discussion before creating a pull request).
 - Read [CONTRIBUTING](https://livingdictionaries.app/kitbook/docs/CONTRIBUTING) and then move on to all the other pages under "Docs" in the Kitbook to understand how to set up a dev environment, repo conventions, commit conventions, and more.
-
-

package.json

@@ -3,6 +3,8 @@
   "type": "module",
   "scripts": {
     "dev": "pnpm --filter=site dev",
+    "prod": "pnpm --filter=site prod",
+    "mixed": "pnpm --filter=site mixed",
     "dev:open": "pnpm --filter=site dev --open",
     "build": "pnpm --filter=site build",
     "preview": "pnpm --filter=site preview",
@@ -19,22 +21,22 @@
     "lint:inspect": "npx @eslint/config-inspector",
     "lint:inspect-future": "eslint --inspect-config",
     "reset-db": "pnpm --filter=site reset-db",
-    "generate-types": "supabase gen types typescript --local --schema public > packages/site/src/lib/supabase/generated.types.ts",
+    "generate-types": "supabase gen types typescript --local --schema public > packages/types/supabase/generated.types.ts",
     "check-packages": "pnpm update --interactive --recursive --latest"
   },
   "devDependencies": {
-    "@antfu/eslint-config": "^2.12.1",
-    "@typescript-eslint/eslint-plugin": "6.21.0",
-    "eslint": "^8.56.0",
-    "eslint-plugin-svelte": "^2.33.2",
+    "@antfu/eslint-config": "^2.27.3",
+    "@typescript-eslint/eslint-plugin": "8.3.0",
+    "eslint": "^9.9.1",
+    "eslint-plugin-svelte": "^2.43.0",
     "eslint-plugin-svelte-stylistic": "^0.0.4",
-    "lint-staged": "^13.2.2",
-    "simple-git-hooks": "^2.11.0",
-    "start-server-and-test": "^2.0.3",
+    "lint-staged": "^15.2.9",
+    "simple-git-hooks": "^2.11.1",
+    "start-server-and-test": "^2.0.5",
     "supabase": "^1.145.4",
     "svelte": "^4.2.12",
     "typescript": "~5.1.6",
-    "vitest": "^1.4.0"
+    "vitest": "^2.1.3"
   },
   "simple-git-hooks": {
     "pre-commit": "pnpm lint-staged"

packages/functions/package.json

@@ -30,6 +30,6 @@
     "@types/node": "^18.11.18",
     "node-fetch": "^2.6.7",
     "typescript": "^5.1.6",
-    "vitest": "^1.4.0"
+    "vitest": "^2.0.5"
   }
 }

packages/scripts/.gitignore

@@ -2,4 +2,5 @@ logs
 service-account*
 .env
 sheets-viewer-SA.json
-.env.supabase
+.env.supabase
+.env.production.supabase

packages/scripts/config-firebase.ts

@@ -1,4 +1,3 @@
-import fs from 'node:fs'
 import { program } from 'commander'
 import { cert, initializeApp } from 'firebase-admin/app'
 import { FieldValue, getFirestore } from 'firebase-admin/firestore'
@@ -7,8 +6,7 @@ import { getAuth } from 'firebase-admin/auth'
 // import serviceAccountDev from './service-account-dev.json';
 // import serviceAccountProd from './service-account-prod.json';
 import { serviceAccountDev, serviceAccountProd } from './service-accounts'
-
-/// LOGGER///
+import './record-logs'
 
 program
   //   .version('0.0.1')
@@ -34,11 +32,5 @@ export const db = getFirestore()
 export const timestamp = FieldValue.serverTimestamp()
 export const storage = getStorage()
 export const auth = getAuth()
-const logFile = fs.createWriteStream(`./logs/${Date.now()}.txt`, { flags: 'w' }) // 'a' to append, 'w' to truncate the file every time the process starts.
-console.log = function (data: any) {
-  logFile.write(`${JSON.stringify(data)}\n`)
-  process.stdout.write(`${JSON.stringify(data)}\n`)
-}
-/// END-LOGGER///
 
-console.log(`Running on ${environment}`)
+console.log(`Firebase running on ${environment}`)

packages/scripts/config-supabase.ts

@@ -1,31 +1,63 @@
 import PG from 'pg'
 import { createClient } from '@supabase/supabase-js'
-import type { Database } from '@living-dictionaries/site/src/lib/supabase/database.types'
+import type { Database } from '@living-dictionaries/types'
 import * as dotenv from 'dotenv'
+import './record-logs'
 
-dotenv.config({ path: '.env.supabase' })
+// TODO: change to .env.development and .env.production
+dotenv.config({ path: '.env.supabase' }) // local project variables
+// dotenv.config({ path: '.env.production.supabase' }) // production project variables
 
-export const supabase = createClient<Database>(process.env.PUBLIC_SUPABASE_API_URL, process.env.SUPABASE_SERVICE_ROLE_KEY)
+export const admin_supabase = createClient<Database>(process.env.PUBLIC_SUPABASE_API_URL, process.env.SUPABASE_SERVICE_ROLE_KEY)
+export const anon_supabase = createClient<Database>(process.env.PUBLIC_SUPABASE_API_URL, process.env.PUBLIC_SUPABASE_ANON_KEY)
+export const jacob_ld_user_id = 'de2d3715-6337-45a3-a81a-d82c3210b2a7'
 
-export async function executeQuery(query: string) {
-  const client = new PG.Client({
+class DB {
+  private pool: PG.Pool
+
+  private config: PG.PoolConfig = {
     user: 'postgres',
     host: '127.0.0.1',
-    // host: 'db.actkqboqpzniojhgtqzw.supabase.co',
     database: 'postgres',
     password: 'postgres',
-    // password: '**',
     port: 54322,
-    // port: 5432,
-  })
-  try {
-    await client.connect()
-    await client.query(query)
-  } catch (error) {
-    console.error('Error in connection/executing query:', error)
-  } finally {
-    await client.end().catch((error) => {
-      console.error('Error ending client connection:', error)
-    })
+
+    // user: 'postgres.actkqboqpzniojhgtqzw',
+    // host: 'aws-0-us-west-1.pooler.supabase.com',
+    // database: 'postgres',
+    // password: '**',
+    // port: 6543,
+
+    max: 10,
+    idleTimeoutMillis: 30000,
+    connectionTimeoutMillis: 5000,
+    allowExitOnIdle: false,
+  }
+
+  async get_db_connection(): Promise<PG.PoolClient> {
+    if (!this.pool) {
+      this.pool = new PG.Pool(this.config)
+      const client = await this.pool.connect()
+      console.info(`----> √ Postgres DB connection established! <----`)
+      return client
+    }
+    return this.pool.connect()
+  }
+
+  async execute_query(query: string): Promise<void> {
+    const client = await this.get_db_connection()
+    try {
+      await client.query(query)
+    } catch (error) {
+      console.error('Error executing query:', error)
+      throw new Error(error)
+    } finally {
+      client.release()
+    }
   }
 }
+
+export const postgres = new DB()
+
+const environment = 'dev'
+console.log(`Supabase running on ${environment}`)

packages/scripts/import/post-request.ts

@@ -16,6 +16,10 @@ export async function post_request<T extends Record<string, any>, ExpectedRespon
   fetch?: typeof fetch
   headers?: RequestInit['headers']
 }): Promise<Return<ExpectedResponse>> {
+  console.info(data)
+  // for running through data without db involved
+  // return { data: { speaker_id: data?.speaker_id, dialect_id: data?.dialect_id }, error: null }
+
   const fetch_to_use = options?.fetch || fetch
 
   const response = await fetch_to_use(route, {

packages/scripts/migrate-to-supabase/all-current-senses-have-entry-placeholders.ts

@@ -0,0 +1,72 @@
+import fs from 'node:fs'
+import Chain from 'stream-chain'
+import Parser from 'stream-json'
+import StreamArray from 'stream-json/streamers/StreamArray'
+import { admin_supabase, jacob_ld_user_id } from '../config-supabase'
+import { remove_seconds_underscore } from './utils/remove-seconds-underscore'
+import { convert_entry } from './convert-entries'
+
+export async function ensure_all_current_senses_have_entry_placeholders() {
+  // load
+  const { data: senses } = await admin_supabase.from('senses')
+    .select('id, entry_id')
+  const { data: entries } = await admin_supabase.from('entries')
+    .select('id')
+
+  const entries_needing_added = new Set<string>()
+  for (const sense of senses) {
+    if (!entries.find(entry => entry.id === sense.entry_id)) {
+      console.log(`need placeholder entry for sense ${sense.id}, entry ${sense.entry_id}`)
+      entries_needing_added.add(sense.entry_id)
+    }
+  }
+  console.log({ entries_to_add: entries_needing_added.size })
+
+  const pipeline = Chain.chain([
+    fs.createReadStream('./migrate-to-supabase/firestore-data/firestore-entries.json'),
+    Parser.parser(),
+    StreamArray.streamArray(),
+  ])
+
+  try {
+    for await (const { value: fb_entry } of pipeline) {
+      if (entries_needing_added.has(fb_entry.id)) {
+        const corrected_fb_entry = remove_seconds_underscore(fb_entry)
+        const [, supa_data] = convert_entry(JSON.parse(JSON.stringify(corrected_fb_entry)))
+        const { entry } = supa_data
+
+        const { data: dictionary } = await admin_supabase.from('dictionaries').select().eq('id', entry.dictionary_id).single()
+        if (!dictionary) {
+          console.log({ creating_dict: entry.dictionary_id })
+          const { error } = await admin_supabase.from('dictionaries').insert({
+            created_by: jacob_ld_user_id,
+            updated_by: jacob_ld_user_id,
+            id: entry.dictionary_id,
+            name: 'CHANGE',
+          })
+          if (error) {
+            console.info({ entry })
+            throw new Error(error.message)
+          }
+        }
+
+        const insert = {
+          id: entry.id,
+          dictionary_id: entry.dictionary_id,
+          lexeme: {},
+          created_by: entry.created_by,
+          created_at: entry.created_at,
+          updated_by: entry.updated_by,
+          updated_at: entry.updated_at,
+        }
+        console.log({ insert })
+        await admin_supabase.from('entries').insert(insert)
+      }
+    }
+    console.log('finished')
+  } catch (err) {
+    console.error(err)
+    pipeline.destroy()
+    pipeline.input.destroy()
+  }
+}

packages/scripts/migrate-to-supabase/auth.ts

@@ -1,11 +1,9 @@
 import type { UserRecord } from 'firebase-admin/auth'
 import { auth } from '../config-firebase'
-import { executeQuery } from '../config-supabase'
+import { postgres } from '../config-supabase'
 import { write_users_insert } from './write-users-insert'
 
-migrate_users()
-
-async function migrate_users() {
+export async function migrate_users() {
   const users = await get_users()
   console.log({ total_users: users.length })
   for (const user of users)
@@ -14,12 +12,12 @@ async function migrate_users() {
     console.log(user.toJSON())
   const sql = write_users_insert(users)
   console.log(sql)
-  await executeQuery(sql)
+  await postgres.execute_query(sql)
 }
 
 const BATCH_SIZE = 1000
 
-async function get_users(): Promise<UserRecord[]> {
+export async function get_users(): Promise<UserRecord[]> {
   try {
     const listUsersResult = await auth.listUsers()
     const { users, pageToken } = listUsersResult
@@ -29,7 +27,7 @@ async function get_users(): Promise<UserRecord[]> {
     if (pageToken) {
       const listUsersResult = await auth.listUsers(BATCH_SIZE, pageToken)
       const { users: nextUsers } = listUsersResult
-      // const nextUsers = await get_users(pageToken)
+      // const nextUsers = await get_users(pageToken) // had issue
       return [...users, ...nextUsers]
     }