Skip to content

DO NOT MERGE Firestore CSI tree-shaking alternate implementation #7920

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 32 commits into from
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
32 commits
Select commit Hold shift + click to select a range
76668ba
query_engine.ts: QueryEngineFieldIndexPlugin added
dconeybe Jan 3, 2024
fe3f569
IndexManagerFieldIndexPlugin added
dconeybe Jan 3, 2024
5536c49
component_provider.ts: make index backfiller pluggable
dconeybe Jan 3, 2024
386b274
Merge remote-tracking branch 'origin/master' into CsiTreeShake2
dconeybe Jan 4, 2024
01f61be
wire up IndexBackfillerSchedulerFactory to FirestoreClient
dconeybe Jan 4, 2024
2663159
Add size reports for CSI
dconeybe Jan 4, 2024
5f5ff88
Make deleteAllPersistentCacheIndexes() more tree-shakeable
dconeybe Jan 4, 2024
78b5104
Fix build of test_index_manager.ts and spec_test_runner.ts
dconeybe Jan 4, 2024
d8177f0
Merge remote-tracking branch 'origin/master' into CsiTreeShake2
dconeybe Jan 4, 2024
8e66542
query_engine.test.ts fixed
dconeybe Jan 4, 2024
36bf52c
local_store_indexeddb.test.ts fixed
dconeybe Jan 4, 2024
f00209a
query_engine.test.ts: fix typos in test names
dconeybe Jan 4, 2024
17271c8
index_manager.test.ts: add tests for installFieldIndexPlugin()
dconeybe Jan 4, 2024
dd92512
local_store.test.ts: fix build
dconeybe Jan 4, 2024
b7e87ba
Fix unit tests with memory persistence
dconeybe Jan 4, 2024
3e676f3
index_backfiller.test.ts: fix the tests
dconeybe Jan 4, 2024
e5b15e6
index_manager.test.ts: fix tests
dconeybe Jan 4, 2024
834443c
local_store_indexeddb.test.ts: fix tests
dconeybe Jan 4, 2024
775d68c
spec_test_runner.ts: fix it
dconeybe Jan 4, 2024
66a87fb
local_store_indexeddb.test.ts: fix spurious errors when calling persi…
dconeybe Jan 4, 2024
350151a
Make it easy to run the unit tests with in-memory mock persistence.
dconeybe Jan 5, 2024
f723d93
Rework how the plugin gets installed
dconeybe Jan 5, 2024
cc89d78
cleanup using constructor interfaces rather than factories
dconeybe Jan 5, 2024
78eabe3
query_engine.test.ts fix
dconeybe Jan 5, 2024
622a067
local_store.test.ts: fix build
dconeybe Jan 5, 2024
04ab201
local_store_indexeddb.test.ts: fix build
dconeybe Jan 5, 2024
6c18143
index_backfiller.test.ts: fix build
dconeybe Jan 5, 2024
fc90b42
spec_test_runner.ts: fix build
dconeybe Jan 5, 2024
55cdb25
index_manager.test.ts: fix build
dconeybe Jan 5, 2024
5fc5fc8
query_engine.test.ts: fix failing CSI tests
dconeybe Jan 5, 2024
fe68fac
spec_test_runner.ts: fix indexing spec tests
dconeybe Jan 5, 2024
57b5bcb
yarn lint:fix
dconeybe Jan 5, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
57 changes: 18 additions & 39 deletions packages/firestore/src/api/persistent_cache_index_manager.ts
Original file line number Diff line number Diff line change
Expand Up @@ -16,9 +16,10 @@
*/

import {
FirestoreClient,
firestoreClientDeleteAllFieldIndexes,
firestoreClientSetPersistentCacheIndexAutoCreationEnabled,
FirestoreClient
firestoreClientDisablePersistentCacheIndexAutoCreation,
firestoreClientEnablePersistentCacheIndexAutoCreation
} from '../core/firestore_client';
import { cast } from '../util/input_validation';
import { logDebug, logWarn } from '../util/log';
Expand Down Expand Up @@ -76,7 +77,12 @@ export function getPersistentCacheIndexManager(
export function enablePersistentCacheIndexAutoCreation(
indexManager: PersistentCacheIndexManager
): void {
setPersistentCacheIndexAutoCreationEnabled(indexManager, true);
indexManager._client.verifyNotTerminated();
firestoreClientEnablePersistentCacheIndexAutoCreation(indexManager._client)
.then(_ => logDebug('enablePersistentCacheIndexAutoCreation() succeeded.'))
.catch(error =>
logWarn('enablePersistentCacheIndexAutoCreation() failed', error)
);
}

/**
Expand All @@ -87,7 +93,12 @@ export function enablePersistentCacheIndexAutoCreation(
export function disablePersistentCacheIndexAutoCreation(
indexManager: PersistentCacheIndexManager
): void {
setPersistentCacheIndexAutoCreationEnabled(indexManager, false);
indexManager._client.verifyNotTerminated();
firestoreClientDisablePersistentCacheIndexAutoCreation(indexManager._client)
.then(_ => logDebug('disablePersistentCacheIndexAutoCreation() succeeded.'))
.catch(error =>
logWarn('disablePersistentCacheIndexAutoCreation() failed', error)
);
}

/**
Expand All @@ -100,41 +111,9 @@ export function deleteAllPersistentCacheIndexes(
indexManager: PersistentCacheIndexManager
): void {
indexManager._client.verifyNotTerminated();

const promise = firestoreClientDeleteAllFieldIndexes(indexManager._client);

promise
.then(_ => logDebug('deleting all persistent cache indexes succeeded'))
.catch(error =>
logWarn('deleting all persistent cache indexes failed', error)
);
}

function setPersistentCacheIndexAutoCreationEnabled(
indexManager: PersistentCacheIndexManager,
isEnabled: boolean
): void {
indexManager._client.verifyNotTerminated();

const promise = firestoreClientSetPersistentCacheIndexAutoCreationEnabled(
indexManager._client,
isEnabled
);

promise
.then(_ =>
logDebug(
`setting persistent cache index auto creation ` +
`isEnabled=${isEnabled} succeeded`
)
)
.catch(error =>
logWarn(
`setting persistent cache index auto creation ` +
`isEnabled=${isEnabled} failed`,
error
)
);
firestoreClientDeleteAllFieldIndexes(indexManager._client)
.then(_ => logDebug('deleteAllPersistentCacheIndexes() succeeded.'))
.catch(error => logWarn('deleteAllPersistentCacheIndexes() failed', error));
}

/**
Expand Down
132 changes: 84 additions & 48 deletions packages/firestore/src/core/component_provider.ts
Original file line number Diff line number Diff line change
Expand Up @@ -22,8 +22,8 @@ import {
IndexBackfillerScheduler
} from '../local/index_backfiller';
import {
indexedDbStoragePrefix,
IndexedDbPersistence
IndexedDbPersistence,
indexedDbStoragePrefix
} from '../local/indexeddb_persistence';
import { LocalStore } from '../local/local_store';
import { newLocalStore } from '../local/local_store_impl';
Expand All @@ -34,7 +34,7 @@ import {
MemoryLruDelegate,
MemoryPersistence
} from '../local/memory_persistence';
import { Scheduler, Persistence } from '../local/persistence';
import { Persistence, Scheduler } from '../local/persistence';
import { QueryEngine } from '../local/query_engine';
import {
ClientId,
Expand All @@ -57,6 +57,7 @@ import { JsonProtoSerializer } from '../remote/serializer';
import { hardAssert } from '../util/assert';
import { AsyncQueue } from '../util/async_queue';
import { Code, FirestoreError } from '../util/error';
import { logDebug } from '../util/log';

import { DatabaseInfo } from './database_info';
import { EventManager, newEventManager } from './event_manager';
Expand Down Expand Up @@ -90,6 +91,7 @@ export interface ComponentConfiguration {
* cache. Implementations override `initialize()` to provide all components.
*/
export interface OfflineComponentProvider {
asyncQueue: AsyncQueue;
persistence: Persistence;
sharedClientState: SharedClientState;
localStore: LocalStore;
Expand All @@ -109,16 +111,23 @@ export interface OfflineComponentProvider {
export class MemoryOfflineComponentProvider
implements OfflineComponentProvider
{
asyncQueue!: AsyncQueue;
persistence!: Persistence;
sharedClientState!: SharedClientState;
localStore!: LocalStore;
gcScheduler!: Scheduler | null;
indexBackfillerScheduler!: Scheduler | null;
indexBackfillerScheduler: Scheduler | null = null;
synchronizeTabs = false;

serializer!: JsonProtoSerializer;

get schedulers(): Scheduler[] {
const schedulers = [this.gcScheduler, this.indexBackfillerScheduler];
return schedulers.filter(scheduler => !!scheduler) as Scheduler[];
}

async initialize(cfg: ComponentConfiguration): Promise<void> {
this.asyncQueue = cfg.asyncQueue;
this.serializer = newSerializer(cfg.databaseInfo.databaseId);
this.sharedClientState = this.createSharedClientState(cfg);
this.persistence = this.createPersistence(cfg);
Expand All @@ -128,10 +137,6 @@ export class MemoryOfflineComponentProvider
cfg,
this.localStore
);
this.indexBackfillerScheduler = this.createIndexBackfillerScheduler(
cfg,
this.localStore
);
}

createGarbageCollectionScheduler(
Expand All @@ -141,13 +146,6 @@ export class MemoryOfflineComponentProvider
return null;
}

createIndexBackfillerScheduler(
cfg: ComponentConfiguration,
localStore: LocalStore
): Scheduler | null {
return null;
}

createLocalStore(cfg: ComponentConfiguration): LocalStore {
return newLocalStore(
this.persistence,
Expand All @@ -166,8 +164,9 @@ export class MemoryOfflineComponentProvider
}

async terminate(): Promise<void> {
this.gcScheduler?.stop();
this.indexBackfillerScheduler?.stop();
for (const scheduler of this.schedulers) {
scheduler.stop();
}
this.sharedClientState.shutdown();
await this.persistence.shutdown();
}
Expand Down Expand Up @@ -215,6 +214,8 @@ export class IndexedDbOfflineComponentProvider extends MemoryOfflineComponentPro
indexBackfillerScheduler!: Scheduler | null;
synchronizeTabs = false;

private primaryStateListenerNotified = false;

constructor(
protected readonly onlineComponentProvider: OnlineComponentProvider,
protected readonly cacheSizeBytes: number | undefined,
Expand All @@ -237,19 +238,30 @@ export class IndexedDbOfflineComponentProvider extends MemoryOfflineComponentPro
// NOTE: This will immediately call the listener, so we make sure to
// set it after localStore / remoteStore are started.
await this.persistence.setPrimaryStateListener(() => {
if (this.gcScheduler && !this.gcScheduler.started) {
this.gcScheduler.start();
}
if (
this.indexBackfillerScheduler &&
!this.indexBackfillerScheduler.started
) {
this.indexBackfillerScheduler.start();
}
this.primaryStateListenerNotified = true;
this.startSchedulers();
return Promise.resolve();
});
}

private startSchedulers(): void {
if (!this.primaryStateListenerNotified) {
return;
}

for (const scheduler of this.schedulers) {
if (!scheduler.started) {
scheduler.start();
}
}
}

installIndexBackfillerScheduler(scheduler: IndexBackfillerScheduler): void {
hardAssert(!this.indexBackfillerScheduler);
this.indexBackfillerScheduler = scheduler;
this.startSchedulers();
}

createLocalStore(cfg: ComponentConfiguration): LocalStore {
return newLocalStore(
this.persistence,
Expand All @@ -268,14 +280,6 @@ export class IndexedDbOfflineComponentProvider extends MemoryOfflineComponentPro
return new LruScheduler(garbageCollector, cfg.asyncQueue, localStore);
}

createIndexBackfillerScheduler(
cfg: ComponentConfiguration,
localStore: LocalStore
): Scheduler | null {
const indexBackfiller = new IndexBackfiller(localStore, this.persistence);
return new IndexBackfillerScheduler(cfg.asyncQueue, indexBackfiller);
}

createPersistence(cfg: ComponentConfiguration): IndexedDbPersistence {
const persistenceKey = indexedDbStoragePrefix(
cfg.databaseInfo.databaseId,
Expand Down Expand Up @@ -305,6 +309,30 @@ export class IndexedDbOfflineComponentProvider extends MemoryOfflineComponentPro
}
}

export function indexedDbOfflineComponentProviderInstallFieldIndexPlugin(
componentProvider: IndexedDbOfflineComponentProvider
): void {
if (componentProvider.indexBackfillerScheduler) {
return;
}

logDebug(
'Installing IndexBackfillerScheduler into OfflineComponentProvider ' +
'to support persistent cache indexing.'
);

const indexBackfiller = new IndexBackfiller(
componentProvider.localStore,
componentProvider.persistence
);
const scheduler = new IndexBackfillerScheduler(
componentProvider.asyncQueue,
indexBackfiller
);

componentProvider.installIndexBackfillerScheduler(scheduler);
}

/**
* Provides all components needed for Firestore with multi-tab IndexedDB
* persistence.
Expand All @@ -316,6 +344,8 @@ export class IndexedDbOfflineComponentProvider extends MemoryOfflineComponentPro
export class MultiTabOfflineComponentProvider extends IndexedDbOfflineComponentProvider {
synchronizeTabs = true;

private isPrimary: boolean | null = null;

constructor(
protected readonly onlineComponentProvider: OnlineComponentProvider,
protected readonly cacheSizeBytes: number | undefined
Expand Down Expand Up @@ -346,27 +376,33 @@ export class MultiTabOfflineComponentProvider extends IndexedDbOfflineComponentP
// NOTE: This will immediately call the listener, so we make sure to
// set it after localStore / remoteStore are started.
await this.persistence.setPrimaryStateListener(async isPrimary => {
this.isPrimary = isPrimary;

await syncEngineApplyPrimaryState(
this.onlineComponentProvider.syncEngine,
isPrimary
);
if (this.gcScheduler) {
if (isPrimary && !this.gcScheduler.started) {
this.gcScheduler.start();
} else if (!isPrimary) {
this.gcScheduler.stop();
}
}
if (this.indexBackfillerScheduler) {
if (isPrimary && !this.indexBackfillerScheduler.started) {
this.indexBackfillerScheduler.start();
} else if (!isPrimary) {
this.indexBackfillerScheduler.stop();
}
}

this.startOrStopSchedulers();
});
}

private startOrStopSchedulers(): void {
for (const scheduler of this.schedulers) {
if (this.isPrimary === true && !scheduler.started) {
scheduler.start();
} else if (this.isPrimary === false) {
scheduler.stop();
}
}
}

installIndexBackfillerScheduler(scheduler: IndexBackfillerScheduler): void {
hardAssert(!this.indexBackfillerScheduler);
this.indexBackfillerScheduler = scheduler;
this.startOrStopSchedulers();
}

createSharedClientState(cfg: ComponentConfiguration): SharedClientState {
const window = getWindow();
if (!WebStorageSharedClientState.isAvailable(window)) {
Expand Down
Loading