-
-
Notifications
You must be signed in to change notification settings - Fork 1
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Loading branch information
Showing
17 changed files
with
433 additions
and
439 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,32 +1,31 @@ | ||
| import algoliasearch from 'algoliasearch'; | ||
| import { projectId } from '../config-firebase'; | ||
| import { adminKey } from './algolia-admin-key.json'; | ||
| import { AlgoliaEntry } from '@living-dictionaries/types'; | ||
| import algoliasearch from 'algoliasearch' | ||
| import type { AlgoliaEntry } from '@living-dictionaries/types' | ||
| import { projectId } from '../config-firebase' | ||
| import { adminKey } from './algolia-admin-key.json' | ||
|
|
||
| const ALGOLIA_APP_ID = 'XCVBAYSYXD'; | ||
| const ALGOLIA_APP_ID = 'XCVBAYSYXD' | ||
|
|
||
| export const client = algoliasearch(ALGOLIA_APP_ID, adminKey); | ||
| export const client = algoliasearch(ALGOLIA_APP_ID, adminKey) | ||
|
|
||
| const index = client.initIndex( | ||
| projectId === 'talking-dictionaries-dev' ? 'entries_dev' : 'entries_prod' | ||
| ); | ||
| projectId === 'talking-dictionaries-dev' ? 'entries_dev' : 'entries_prod', | ||
| ) | ||
|
|
||
| const MAX_CHUNK_SIZE = 3000; | ||
| const MAX_CHUNK_SIZE = 3000 | ||
| // https://www.algolia.com/doc/api-reference/api-methods/add-objects/#examples | ||
| // if forced to iterate instead of save all at once, take note of the rate limiting at 5000 backlogged requests https://www.algolia.com/doc/faq/indexing/is-there-a-rate-limit/ | ||
|
|
||
| export async function updateIndex(entries: AlgoliaEntry[]) { | ||
| try { | ||
| for (let startOfChunkIndex = 0; startOfChunkIndex < entries.length; startOfChunkIndex += MAX_CHUNK_SIZE) { | ||
| const endOfChunk = startOfChunkIndex + MAX_CHUNK_SIZE; | ||
| const chunk = entries.slice(startOfChunkIndex, endOfChunk); | ||
| console.log({ startOfChunkIndex, endOfChunk, CHUNK_SIZE: MAX_CHUNK_SIZE, chunkLength: chunk.length }); | ||
| const endOfChunk = startOfChunkIndex + MAX_CHUNK_SIZE | ||
| const chunk = entries.slice(startOfChunkIndex, endOfChunk) | ||
| console.log({ startOfChunkIndex, endOfChunk, CHUNK_SIZE: MAX_CHUNK_SIZE, chunkLength: chunk.length }) | ||
|
|
||
| const { objectIDs } = await index.saveObjects(chunk); | ||
| console.log(`Entries indexed: ${objectIDs.length}`); | ||
| const { objectIDs } = await index.saveObjects(chunk) | ||
| console.log(`Entries indexed: ${objectIDs.length}`) | ||
| } | ||
| } catch (err) { | ||
| console.log(err); | ||
| console.log(err) | ||
| } | ||
|
|
||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,45 +1,43 @@ | ||
| import { db } from '../config-firebase'; | ||
| import { updateIndex } from './algolia'; | ||
| import { ActualDatabaseEntry } from '@living-dictionaries/types'; | ||
| import type { ActualDatabaseEntry } from '@living-dictionaries/types' | ||
| import * as prepare from '@living-dictionaries/functions/src/algolia/prepareDataForIndex' | ||
| import { db } from '../config-firebase' | ||
| import { updateIndex } from './algolia' | ||
|
|
||
| // import { prepareDataForIndex } from '@living-dictionaries/functions/src/algolia/prepareDataForIndex'; | ||
| import * as prepare from '@living-dictionaries/functions/src/algolia/prepareDataForIndex'; | ||
| // @ts-ignore | ||
| // @ts-expect-error | ||
| const prepareDataForIndex = prepare.default | ||
| .prepareDataForIndex as typeof import('@living-dictionaries/functions/src/algolia/prepareDataForIndex').prepareDataForIndex; // b/c file is declared to be commonjs by its package.json | ||
| .prepareDataForIndex as typeof import('@living-dictionaries/functions/src/algolia/prepareDataForIndex').prepareDataForIndex // b/c file is declared to be commonjs by its package.json | ||
|
|
||
| async function updateMostRecentEntries(count: number, { dry = true }) { | ||
| const entriesSnapshot = await db.collectionGroup('words').orderBy('ua', 'desc').limit(count).get(); | ||
| const entries = await prepareEntriesFromSnapshot(entriesSnapshot); | ||
| const entriesSnapshot = await db.collectionGroup('words').orderBy('ua', 'desc').limit(count).get() | ||
| const entries = await prepareEntriesFromSnapshot(entriesSnapshot) | ||
|
|
||
| if (!dry) | ||
| await updateIndex(entries); | ||
| await updateIndex(entries) | ||
| } | ||
|
|
||
|
|
||
| async function updateIndexByField(fieldToIndex: string, { dry = true }) { | ||
| // The field must be indexed first in Firebase | ||
| const entriesSnapshot = await db.collectionGroup('words').where(fieldToIndex, '!=', null).get(); | ||
| const entries = await prepareEntriesFromSnapshot(entriesSnapshot); | ||
| const entriesSnapshot = await db.collectionGroup('words').where(fieldToIndex, '!=', null).get() | ||
| const entries = await prepareEntriesFromSnapshot(entriesSnapshot) | ||
|
|
||
| if (!dry) | ||
| await updateIndex(entries); | ||
| await updateIndex(entries) | ||
| } | ||
|
|
||
| // eslint-disable-next-line no-undef | ||
| async function prepareEntriesFromSnapshot(entriesSnapshot: FirebaseFirestore.QuerySnapshot<FirebaseFirestore.DocumentData>) { | ||
| const entryPromises = entriesSnapshot.docs.map(async (doc) => { | ||
| const dbEntry = doc.data() as ActualDatabaseEntry; | ||
| const dictionaryId = doc.ref.parent.parent.id; // dictionary/words/entry-123 -> doc.ref: entry-123, doc.ref.parent: words, doc.ref.parent.parent: dictionary | ||
| const algoliaEntry = await prepareDataForIndex(dbEntry, dictionaryId, db); | ||
| const time = dbEntry.ua.toDate(); | ||
| console.log({ dbEntry, algoliaEntry, time }); | ||
| return { ...algoliaEntry, objectID: doc.id }; | ||
| }); | ||
|
|
||
| const entries = await Promise.all(entryPromises); | ||
| return entries; | ||
| const dbEntry = doc.data() as ActualDatabaseEntry | ||
| const dictionaryId = doc.ref.parent.parent.id // dictionary/words/entry-123 -> doc.ref: entry-123, doc.ref.parent: words, doc.ref.parent.parent: dictionary | ||
| const algoliaEntry = await prepareDataForIndex(dbEntry, dictionaryId, db) | ||
| const time = dbEntry.ua.toDate() | ||
| console.log({ dbEntry, algoliaEntry, time }) | ||
| return { ...algoliaEntry, objectID: doc.id } | ||
| }) | ||
|
|
||
| const entries = await Promise.all(entryPromises) | ||
| return entries | ||
| } | ||
|
|
||
| // updateIndexByField('nc', { dry: true }); | ||
| updateMostRecentEntries(300, { dry: false }); | ||
| updateMostRecentEntries(300, { dry: false }) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,44 +1,44 @@ | ||
| import { program } from 'commander'; | ||
| import { initializeApp, cert } from 'firebase-admin/app'; | ||
| import { FieldValue, getFirestore } from 'firebase-admin/firestore'; | ||
| import { getStorage } from 'firebase-admin/storage'; | ||
| import { getAuth } from 'firebase-admin/auth'; | ||
| import fs from 'node:fs' | ||
| import { program } from 'commander' | ||
| import { cert, initializeApp } from 'firebase-admin/app' | ||
| import { FieldValue, getFirestore } from 'firebase-admin/firestore' | ||
| import { getStorage } from 'firebase-admin/storage' | ||
| import { getAuth } from 'firebase-admin/auth' | ||
| // import serviceAccountDev from './service-account-dev.json'; | ||
| // import serviceAccountProd from './service-account-prod.json'; | ||
| import { serviceAccountDev, serviceAccountProd } from './service-accounts'; | ||
| import { serviceAccountDev, serviceAccountProd } from './service-accounts' | ||
|
|
||
| /// LOGGER/// | ||
|
|
||
| program | ||
| // .version('0.0.1') | ||
| .option('-e, --environment [dev/prod]', 'Firebase Project', 'dev') | ||
| .allowUnknownOption() // because config is shared by multiple scripts | ||
| .parse(process.argv); | ||
| .parse(process.argv) | ||
|
|
||
| export const environment = program.opts().environment === 'prod' ? 'prod' : 'dev'; | ||
| export const projectId = | ||
| environment === 'prod' ? 'talking-dictionaries-alpha' : 'talking-dictionaries-dev'; | ||
| export const environment = program.opts().environment === 'prod' ? 'prod' : 'dev' | ||
| export const projectId | ||
| = environment === 'prod' ? 'talking-dictionaries-alpha' : 'talking-dictionaries-dev' | ||
|
|
||
| const serviceAccount = environment === 'dev' ? serviceAccountDev : serviceAccountProd; | ||
| const serviceAccount = environment === 'dev' ? serviceAccountDev : serviceAccountProd | ||
|
|
||
| initializeApp({ | ||
| // @ts-expect-error | ||
| credential: cert(serviceAccount), | ||
| databaseURL: `https://${projectId}.firebaseio.com`, | ||
| storageBucket: `${projectId}.appspot.com`, | ||
| }); | ||
| export const db = getFirestore(); | ||
| }) | ||
| export const db = getFirestore() | ||
| // const settings = { timestampsInSnapshots: true }; | ||
| // db.settings(settings); | ||
| export const timestamp = FieldValue.serverTimestamp(); | ||
| export const storage = getStorage(); | ||
| export const auth = getAuth(); | ||
|
|
||
| ///LOGGER/// | ||
| import fs from 'fs'; | ||
| const logFile = fs.createWriteStream(`./logs/${Date.now()}.txt`, { flags: 'w' }); // 'a' to append, 'w' to truncate the file every time the process starts. | ||
| export const timestamp = FieldValue.serverTimestamp() | ||
| export const storage = getStorage() | ||
| export const auth = getAuth() | ||
| const logFile = fs.createWriteStream(`./logs/${Date.now()}.txt`, { flags: 'w' }) // 'a' to append, 'w' to truncate the file every time the process starts. | ||
| console.log = function (data: any) { | ||
| logFile.write(JSON.stringify(data) + '\n'); | ||
| process.stdout.write(JSON.stringify(data) + '\n'); | ||
| }; | ||
| ///END-LOGGER/// | ||
| logFile.write(`${JSON.stringify(data)}\n`) | ||
| process.stdout.write(`${JSON.stringify(data)}\n`) | ||
| } | ||
| /// END-LOGGER/// | ||
|
|
||
| console.log(`Running on ${environment}`); | ||
| console.log(`Running on ${environment}`) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,25 +1,25 @@ | ||
| import { db } from './config-firebase'; | ||
| import { db } from './config-firebase' | ||
|
|
||
| export async function countAllEntries() { | ||
| let overallEntryCount = 0; | ||
| let overallEntryCount = 0 | ||
|
|
||
| const dictionarySnaps = await db.collection('dictionaries').get(); | ||
| const dictionaryIds = dictionarySnaps.docs.map(doc => doc.id); | ||
| const dictionarySnaps = await db.collection('dictionaries').get() | ||
| const dictionaryIds = dictionarySnaps.docs.map(doc => doc.id) | ||
|
|
||
| for (const dictionaryId of dictionaryIds) { | ||
| if (dictionaryId.startsWith('tdv1-')) continue; | ||
| if (dictionaryId.startsWith('tdv1-')) continue | ||
|
|
||
| const countData = await db.collection(`dictionaries/${dictionaryId}/words`).count().get(); | ||
| const { count: entryCount } = countData.data(); | ||
| console.log({ dictionaryId, entryCount, overallEntryCount }); | ||
| overallEntryCount += entryCount; | ||
| console.log({ dictionaryId, entryCount, overallEntryCount }); | ||
| await db.doc(`dictionaries/${dictionaryId}`).update({ entryCount }); | ||
| const countData = await db.collection(`dictionaries/${dictionaryId}/words`).count().get() | ||
| const { count: entryCount } = countData.data() | ||
| console.log({ dictionaryId, entryCount, overallEntryCount }) | ||
| overallEntryCount += entryCount | ||
| console.log({ dictionaryId, entryCount, overallEntryCount }) | ||
| await db.doc(`dictionaries/${dictionaryId}`).update({ entryCount }) | ||
| } | ||
|
|
||
| await db.doc('stats/data').update({ overallEntryCount }); | ||
| await db.doc('stats/data').update({ overallEntryCount }) | ||
|
|
||
| return true; | ||
| return true | ||
| } | ||
|
|
||
| countAllEntries().then(() => console.log('done')).catch(console.error); | ||
| countAllEntries().then(() => console.log('done')).catch(console.error) |
Oops, something went wrong.