From 5b9b2848b0b1eba26649d35ebf2bf3d0c42a6f24 Mon Sep 17 00:00:00 2001 From: oliver-oloughlin Date: Thu, 7 Dec 2023 18:44:38 +0100 Subject: [PATCH 1/3] feat: added pagination/filtering for findHistory() --- src/collection.ts | 73 +++++++++------ src/types.ts | 12 +-- src/utils.ts | 5 +- tests/collection/history.test.ts | 82 ++++++++++++----- tests/db/atomic.test.ts | 4 +- tests/db/deleteAll.test.ts | 6 +- tests/db/wipe.test.ts | 6 +- tests/indexable_collection/history.test.ts | 88 +++++++++++++----- tests/serialized_collection/history.test.ts | 85 +++++++++++++----- .../history.test.ts | 89 ++++++++++++++----- 10 files changed, 316 insertions(+), 134 deletions(-) diff --git a/src/collection.ts b/src/collection.ts index 54fe54e..058b402 100644 --- a/src/collection.ts +++ b/src/collection.ts @@ -353,7 +353,11 @@ export class Collection< */ async findBySecondaryIndex< const K extends SecondaryIndexKeys, - >(index: K, value: CheckKeyOf, options?: ListOptions) { + >( + index: K, + value: CheckKeyOf, + options?: ListOptions>, + ) { // Serialize and compress index value const serialized = this._serializer.serialize(value) const compressed = this._serializer.compress(serialized) @@ -426,18 +430,19 @@ export class Collection< * @param id - Document id. * @returns A promise resolving to a list of history entries. */ - async findHistory(id: KvId) { + async findHistory(id: KvId, options?: ListOptions>) { // Initialize result list and create history key prefix const result: HistoryEntry[] = [] - const historyKeyPrefix = extendKey(this._keys.history, id) + const keyPrefix = extendKey(this._keys.history, id) + const selector = createListSelector(keyPrefix, options) // Create hsitory entries iterator - const iter = this.kv.list>({ - prefix: historyKeyPrefix, - }) + const iter = this.kv.list>(selector, options) // Collect history entries for await (const { value, key } of iter) { + let historyEntry: HistoryEntry = value + // Handle serialized entries if (value.type === "write" && this._isSerialized) { const { ids } = value.value as SerializedEntry @@ -456,29 +461,35 @@ export class Collection< const serialized = this._serializer.decompress(data) const deserialized = this._serializer.deserialize(serialized) - // Add history entry - result.push({ + // Set history entry + historyEntry = { type: value.type, timestamp: value.timestamp, value: this._model.__validate?.(deserialized) ?? this._model.parse(deserialized as any), - }) + } } else if (value.type === "write") { - // Add history entry with parsed value - result.push({ + // Set history entry + historyEntry = { type: value.type, timestamp: value.timestamp, value: this._model.__validate?.(value.value) ?? this._model.parse(value.value as any), - }) - } else { - // Add "delete" history entry - result.push(value) + } + } + + // Filter and add history entry to result list + const filter = options?.filter + if (!filter || filter(historyEntry)) { + result.push(historyEntry) } } - // Return result list - return result + // Return result list and iterator cursor + return { + result, + cursor: iter.cursor || undefined, + } } /** @@ -648,7 +659,11 @@ export class Collection< */ async deleteBySecondaryIndex< const K extends SecondaryIndexKeys, - >(index: K, value: CheckKeyOf, options?: ListOptions) { + >( + index: K, + value: CheckKeyOf, + options?: ListOptions>, + ) { // Serialize and compress index value const serialized = this._serializer.serialize(value) const compressed = this._serializer.compress(serialized) @@ -794,7 +809,7 @@ export class Collection< index: K, value: CheckKeyOf, data: UpdateData, - options?: UpdateManyOptions, + options?: UpdateManyOptions>, ) { // Serialize and compress index value const serialized = this._serializer.serialize(value) @@ -839,7 +854,7 @@ export class Collection< */ async updateMany( value: UpdateData, - options?: UpdateManyOptions, + options?: UpdateManyOptions>, ) { // Update each document, add commit result to result list return await this.handleMany( @@ -929,7 +944,7 @@ export class Collection< * @param options - List options, optional. * @returns A promise that resovles to an object containing the iterator cursor */ - async deleteMany(options?: AtomicListOptions) { + async deleteMany(options?: AtomicListOptions>) { // Perform quick delete if all documents are to be deleted if (selectsAll(options)) { // Create list iterator and empty keys list, init atomic operation @@ -997,7 +1012,7 @@ export class Collection< * @param options - List options, optional. * @returns A promise that resovles to an object containing a list of the retrieved documents and the iterator cursor */ - async getMany(options?: ListOptions) { + async getMany(options?: ListOptions>) { // Get each document, return result list and current iterator cursor return await this.handleMany( this._keys.id, @@ -1028,7 +1043,7 @@ export class Collection< */ async forEach( fn: (doc: Document) => unknown, - options?: ListOptions, + options?: ListOptions>, ) { // Execute callback function for each document entry const { cursor } = await this.handleMany( @@ -1068,7 +1083,7 @@ export class Collection< index: K, value: CheckKeyOf, fn: (doc: Document) => unknown, - options?: UpdateManyOptions, + options?: UpdateManyOptions>, ) { // Serialize and compress index value const serialized = this._serializer.serialize(value) @@ -1116,7 +1131,7 @@ export class Collection< */ async map( fn: (doc: Document) => T, - options?: ListOptions, + options?: ListOptions>, ) { // Execute callback function for each document entry, return result and cursor return await this.handleMany( @@ -1156,7 +1171,7 @@ export class Collection< index: K, value: CheckKeyOf, fn: (doc: Document) => T, - options?: UpdateManyOptions, + options?: UpdateManyOptions>, ) { // Serialize and compress index value const serialized = this._serializer.serialize(value) @@ -1194,7 +1209,7 @@ export class Collection< * @param options - Count options, optional. * @returns A promise that resolves to a number representing the count. */ - async count(options?: CountOptions) { + async count(options?: CountOptions>) { // Initiate count result let result = 0 @@ -1232,7 +1247,7 @@ export class Collection< >( index: K, value: CheckKeyOf, - options?: CountOptions, + options?: CountOptions>, ) { // Serialize and compress index value const serialized = this._serializer.serialize(value) @@ -1804,7 +1819,7 @@ export class Collection< protected async handleMany( prefixKey: KvKey, fn: (doc: Document) => T, - options: ListOptions | undefined, + options: ListOptions> | undefined, ) { // Create list iterator with given options const selector = createListSelector(prefixKey, options) diff --git a/src/types.ts b/src/types.ts index bdc0dfb..e955344 100644 --- a/src/types.ts +++ b/src/types.ts @@ -349,14 +349,14 @@ export type SetOptions = NonNullable["2"]> & { overwrite?: boolean } -export type ListOptions = Deno.KvListOptions & { +export type ListOptions = Deno.KvListOptions & { /** * Filter documents based on predicate. * * @param doc - Document. * @returns true or false. */ - filter?: (doc: Document) => boolean + filter?: (value: T) => boolean /** Id of document to start from. */ startId?: KvId @@ -370,11 +370,11 @@ export type AtomicBatchOptions = { atomicBatchSize?: number } -export type AtomicListOptions = +export type AtomicListOptions = & ListOptions & AtomicBatchOptions -export type CountOptions = +export type CountOptions = & CountAllOptions & Pick, "filter"> @@ -393,11 +393,11 @@ export type UpdateOptions = Omit & { export type MergeType = "shallow" | "deep" -export type UpdateManyOptions = +export type UpdateManyOptions = & ListOptions & UpdateOptions -export type CountAllOptions = Pick, "consistency"> +export type CountAllOptions = Pick, "consistency"> export type EnqueueOptions = & Omit< diff --git a/src/utils.ts b/src/utils.ts index b45901a..aa5c103 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -9,7 +9,6 @@ import type { KvId, KvKey, KvObject, - KvValue, ListOptions, ParsedQueueMessage, PreparedEnqueue, @@ -438,7 +437,7 @@ export function parseQueueMessage( * @param options - List options. * @returns A list selector. */ -export function createListSelector( +export function createListSelector( prefixKey: KvKey, options: ListOptions | undefined, ): Deno.KvListSelector { @@ -471,7 +470,7 @@ export function createListSelector( * @param options - List options. * @returns true if list options selects all entries, false if potentially not. */ -export function selectsAll( +export function selectsAll( options: ListOptions | undefined, ) { return ( diff --git a/tests/collection/history.test.ts b/tests/collection/history.test.ts index 4acf0ae..f4783de 100644 --- a/tests/collection/history.test.ts +++ b/tests/collection/history.test.ts @@ -14,13 +14,13 @@ Deno.test("collection - history", async (t) => { }) const id = "id" - await db.users.write(id, mockUser1) + await db.users.set(id, mockUser1, { overwrite: true }) await sleep(10) - await db.users.write(id, mockUser2) + await db.users.set(id, mockUser2, { overwrite: true }) await sleep(10) - await db.users.write(id, mockUser3) + await db.users.set(id, mockUser3, { overwrite: true }) - const [h1, h2, h3] = await db.users.findHistory(id) + const { result: [h1, h2, h3] } = await db.users.findHistory(id) assert(h1.type === "write") assert(h1.value.username === mockUser1.username) assert(h1.timestamp.valueOf() <= h2.timestamp.valueOf()) @@ -42,17 +42,17 @@ Deno.test("collection - history", async (t) => { }) const id = "id" - await db.users.write(id, mockUser1) + await db.users.set(id, mockUser1, { overwrite: true }) await sleep(10) await db.users.delete(id) await sleep(10) - await db.users.write(id, mockUser2) + await db.users.set(id, mockUser2, { overwrite: true }) await sleep(10) - await db.users.write(id, mockUser3) + await db.users.set(id, mockUser3, { overwrite: true }) await sleep(10) await db.users.delete(id) - const [h1, h2, h3, h4, h5] = await db.users.findHistory(id) + const { result: [h1, h2, h3, h4, h5] } = await db.users.findHistory(id) assert(h1.type === "write") assert(h1.value.username === mockUser1.username) assert(h1.timestamp.valueOf() <= h2.timestamp.valueOf()) @@ -78,13 +78,13 @@ Deno.test("collection - history", async (t) => { }) const id = "id" - await db.users.write(id, mockUser1) + await db.users.set(id, mockUser1, { overwrite: true }) await sleep(10) await db.users.update(id, mockUser2) await sleep(10) await db.users.update(id, mockUser3) - const [h1, h2, h3] = await db.users.findHistory(id) + const { result: [h1, h2, h3] } = await db.users.findHistory(id) assert(h1.type === "write") assert(h1.value.username === mockUser1.username) assert(h1.timestamp.valueOf() <= h2.timestamp.valueOf()) @@ -106,15 +106,15 @@ Deno.test("collection - history", async (t) => { }) const id = "id" - await db.users.write(id, mockUser1) + await db.users.set(id, mockUser1, { overwrite: true }) await sleep(10) await db.users.deleteMany() await sleep(10) - await db.users.write(id, mockUser2) + await db.users.set(id, mockUser2, { overwrite: true }) await sleep(10) await db.users.deleteMany({ filter: () => true }) - const [h1, h2, h3, h4] = await db.users.findHistory(id) + const { result: [h1, h2, h3, h4] } = await db.users.findHistory(id) assert(h1.type === "write") assert(h1.value.username === mockUser1.username) assert(h1.timestamp.valueOf() <= h2.timestamp.valueOf()) @@ -137,17 +137,55 @@ Deno.test("collection - history", async (t) => { }) const id = "id" - await db.users.write(id, mockUser1) + await db.users.set(id, mockUser1, { overwrite: true }) await db.users.update(id, mockUser2) await db.users.delete(id) await db.users.deleteMany() - const history = await db.users.findHistory(id) + const { result: history } = await db.users.findHistory(id) assert(history.length === 0) }) }, ) + await t.step("Should find filtered history", async () => { + await useKv(async (kv) => { + const db = kvdex(kv, { + users: collection(model(), { history: true }), + }) + + const id = "id" + await db.users.set(id, mockUser1, { overwrite: true }) + await db.users.delete(id) + await db.users.set(id, mockUser2, { overwrite: true }) + await db.users.update(id, mockUser3) + + const { result: history1 } = await db.users.findHistory(id, { + filter: (entry) => entry.type === "delete", + }) + + const { result: history2 } = await db.users.findHistory(id, { + filter: (entry) => + entry.type === "write" && entry.value.age === mockUser1.age, + }) + + assert(history1.length === 1) + assert(history2.length === 2) + + assert( + history2.some((h) => + h.type === "write" && h.value.username === mockUser1.username + ), + ) + + assert( + history2.some((h) => + h.type === "write" && h.value.username === mockUser2.username + ), + ) + }) + }) + await t.step("Should delete all document history", async () => { await useKv(async (kv) => { const db = kvdex(kv, { @@ -155,22 +193,22 @@ Deno.test("collection - history", async (t) => { }) const id = "id" - await db.users.write(id, mockUser1) - await db.users.write(id, mockUser2) - await db.users.write(id, mockUser3) + await db.users.set(id, mockUser1, { overwrite: true }) + await db.users.set(id, mockUser2, { overwrite: true }) + await db.users.set(id, mockUser3, { overwrite: true }) const cr = await db.users.add(mockUser1) assert(cr.ok) - const history1_1 = await db.users.findHistory(id) - const history1_2 = await db.users.findHistory(cr.id) + const { result: history1_1 } = await db.users.findHistory(id) + const { result: history1_2 } = await db.users.findHistory(cr.id) assert(history1_1.length === 3) assert(history1_2.length === 1) await db.users.deleteHistory(id) - const history2_1 = await db.users.findHistory(id) - const history2_2 = await db.users.findHistory(cr.id) + const { result: history2_1 } = await db.users.findHistory(id) + const { result: history2_2 } = await db.users.findHistory(cr.id) assert(history2_1.length === 0) assert(history2_2.length === 1) }) diff --git a/tests/db/atomic.test.ts b/tests/db/atomic.test.ts index a1a6857..2c88de8 100644 --- a/tests/db/atomic.test.ts +++ b/tests/db/atomic.test.ts @@ -393,11 +393,11 @@ Deno.test("db - atomic", async (t) => { filter: (d) => d.value === 100, }) - const [h] = await db.numbers.findHistory(doc.id) + const { result: [h] } = await db.numbers.findHistory(doc.id) assert(h.type === "write") assert(h.value === 100) - const [h1, h2] = await db.numbers.findHistory(id) + const { result: [h1, h2] } = await db.numbers.findHistory(id) assert(h1.type === "write") assert(h1.value === 200) diff --git a/tests/db/deleteAll.test.ts b/tests/db/deleteAll.test.ts index 93f95e0..4cb1cc5 100644 --- a/tests/db/deleteAll.test.ts +++ b/tests/db/deleteAll.test.ts @@ -50,9 +50,9 @@ Deno.test("db - deleteAll", async (t) => { await db.deleteAll() const count2 = await db.countAll() - const h1 = await db.i_users.findHistory(docs1[0].id) - const h2 = await db.s_users.findHistory(docs2[0].id) - const h3 = await db.u64s.findHistory(docs3[0].id) + const { result: h1 } = await db.i_users.findHistory(docs1[0].id) + const { result: h2 } = await db.s_users.findHistory(docs2[0].id) + const { result: h3 } = await db.u64s.findHistory(docs3[0].id) assert(count2 === 0) assert(h1.length > 0) diff --git a/tests/db/wipe.test.ts b/tests/db/wipe.test.ts index 4c300ba..72f1a85 100644 --- a/tests/db/wipe.test.ts +++ b/tests/db/wipe.test.ts @@ -50,9 +50,9 @@ Deno.test("db - wipe", async (t) => { await db.wipe() const count2 = await db.countAll() - const h1 = await db.i_users.findHistory(docs1[0].id) - const h2 = await db.s_users.findHistory(docs2[0].id) - const h3 = await db.u64s.findHistory(docs3[0].id) + const { result: h1 } = await db.i_users.findHistory(docs1[0].id) + const { result: h2 } = await db.s_users.findHistory(docs2[0].id) + const { result: h3 } = await db.u64s.findHistory(docs3[0].id) assert(count2 === 0) assert(h1.length === 0) diff --git a/tests/indexable_collection/history.test.ts b/tests/indexable_collection/history.test.ts index 5dd0bac..0a38137 100644 --- a/tests/indexable_collection/history.test.ts +++ b/tests/indexable_collection/history.test.ts @@ -20,13 +20,13 @@ Deno.test("indexable_collection - history", async (t) => { }) const id = "id" - await db.users.write(id, mockUser1) + await db.users.set(id, mockUser1, { overwrite: true }) await sleep(10) - await db.users.write(id, mockUser2) + await db.users.set(id, mockUser2, { overwrite: true }) await sleep(10) - await db.users.write(id, mockUser3) + await db.users.set(id, mockUser3, { overwrite: true }) - const [h1, h2, h3] = await db.users.findHistory(id) + const { result: [h1, h2, h3] } = await db.users.findHistory(id) assert(h1.type === "write") assert(h1.value.username === mockUser1.username) assert(h1.timestamp.valueOf() <= h2.timestamp.valueOf()) @@ -54,17 +54,17 @@ Deno.test("indexable_collection - history", async (t) => { }) const id = "id" - await db.users.write(id, mockUser1) + await db.users.set(id, mockUser1, { overwrite: true }) await sleep(10) await db.users.delete(id) await sleep(10) - await db.users.write(id, mockUser2) + await db.users.set(id, mockUser2, { overwrite: true }) await sleep(10) - await db.users.write(id, mockUser3) + await db.users.set(id, mockUser3, { overwrite: true }) await sleep(10) await db.users.delete(id) - const [h1, h2, h3, h4, h5] = await db.users.findHistory(id) + const { result: [h1, h2, h3, h4, h5] } = await db.users.findHistory(id) assert(h1.type === "write") assert(h1.value.username === mockUser1.username) assert(h1.timestamp.valueOf() <= h2.timestamp.valueOf()) @@ -96,13 +96,13 @@ Deno.test("indexable_collection - history", async (t) => { }) const id = "id" - await db.users.write(id, mockUser1) + await db.users.set(id, mockUser1, { overwrite: true }) await sleep(10) await db.users.update(id, mockUser2) await sleep(10) await db.users.update(id, mockUser3) - const [h1, h2, h3] = await db.users.findHistory(id) + const { result: [h1, h2, h3] } = await db.users.findHistory(id) assert(h1.type === "write") assert(h1.value.username === mockUser1.username) assert(h1.timestamp.valueOf() <= h2.timestamp.valueOf()) @@ -130,15 +130,15 @@ Deno.test("indexable_collection - history", async (t) => { }) const id = "id" - await db.users.write(id, mockUser1) + await db.users.set(id, mockUser1, { overwrite: true }) await sleep(10) await db.users.deleteMany() await sleep(10) - await db.users.write(id, mockUser2) + await db.users.set(id, mockUser2, { overwrite: true }) await sleep(10) await db.users.deleteMany({ filter: () => true }) - const [h1, h2, h3, h4] = await db.users.findHistory(id) + const { result: [h1, h2, h3, h4] } = await db.users.findHistory(id) assert(h1.type === "write") assert(h1.value.username === mockUser1.username) assert(h1.timestamp.valueOf() <= h2.timestamp.valueOf()) @@ -166,17 +166,61 @@ Deno.test("indexable_collection - history", async (t) => { }) const id = "id" - await db.users.write(id, mockUser1) + await db.users.set(id, mockUser1, { overwrite: true }) await db.users.update(id, mockUser2) await db.users.delete(id) await db.users.deleteMany() - const history = await db.users.findHistory(id) + const { result: history } = await db.users.findHistory(id) assert(history.length === 0) }) }, ) + await t.step("Should find filtered history", async () => { + await useKv(async (kv) => { + const db = kvdex(kv, { + users: collection(model(), { + history: true, + indices: { + username: "primary", + age: "secondary", + }, + }), + }) + + const id = "id" + await db.users.set(id, mockUser1, { overwrite: true }) + await db.users.delete(id) + await db.users.set(id, mockUser2, { overwrite: true }) + await db.users.update(id, mockUser3) + + const { result: history1 } = await db.users.findHistory(id, { + filter: (entry) => entry.type === "delete", + }) + + const { result: history2 } = await db.users.findHistory(id, { + filter: (entry) => + entry.type === "write" && entry.value.age === mockUser1.age, + }) + + assert(history1.length === 1) + assert(history2.length === 2) + + assert( + history2.some((h) => + h.type === "write" && h.value.username === mockUser1.username + ), + ) + + assert( + history2.some((h) => + h.type === "write" && h.value.username === mockUser2.username + ), + ) + }) + }) + await t.step("Should delete all document history", async () => { await useKv(async (kv) => { const db = kvdex(kv, { @@ -190,22 +234,22 @@ Deno.test("indexable_collection - history", async (t) => { }) const id = "id" - await db.users.write(id, mockUser1) - await db.users.write(id, mockUser2) - await db.users.write(id, mockUser3) + await db.users.set(id, mockUser1, { overwrite: true }) + await db.users.set(id, mockUser2, { overwrite: true }) + await db.users.set(id, mockUser3, { overwrite: true }) const cr = await db.users.add(generateUsers(1)[0]) assert(cr.ok) - const history1_1 = await db.users.findHistory(id) - const history1_2 = await db.users.findHistory(cr.id) + const { result: history1_1 } = await db.users.findHistory(id) + const { result: history1_2 } = await db.users.findHistory(cr.id) assert(history1_1.length === 3) assert(history1_2.length === 1) await db.users.deleteHistory(id) - const history2_1 = await db.users.findHistory(id) - const history2_2 = await db.users.findHistory(cr.id) + const { result: history2_1 } = await db.users.findHistory(id) + const { result: history2_2 } = await db.users.findHistory(cr.id) assert(history2_1.length === 0) assert(history2_2.length === 1) }) diff --git a/tests/serialized_collection/history.test.ts b/tests/serialized_collection/history.test.ts index 4b348c6..01d97d5 100644 --- a/tests/serialized_collection/history.test.ts +++ b/tests/serialized_collection/history.test.ts @@ -17,13 +17,13 @@ Deno.test("serialized_collection - history", async (t) => { }) const id = "id" - await db.users.write(id, mockUser1) + await db.users.set(id, mockUser1, { overwrite: true }) await sleep(10) - await db.users.write(id, mockUser2) + await db.users.set(id, mockUser2, { overwrite: true }) await sleep(10) - await db.users.write(id, mockUser3) + await db.users.set(id, mockUser3, { overwrite: true }) - const [h1, h2, h3] = await db.users.findHistory(id) + const { result: [h1, h2, h3] } = await db.users.findHistory(id) assert(h1.type === "write") assert(h1.value.username === mockUser1.username) assert(h1.timestamp.valueOf() <= h2.timestamp.valueOf()) @@ -48,17 +48,17 @@ Deno.test("serialized_collection - history", async (t) => { }) const id = "id" - await db.users.write(id, mockUser1) + await db.users.set(id, mockUser1, { overwrite: true }) await sleep(10) await db.users.delete(id) await sleep(10) - await db.users.write(id, mockUser2) + await db.users.set(id, mockUser2, { overwrite: true }) await sleep(10) - await db.users.write(id, mockUser3) + await db.users.set(id, mockUser3, { overwrite: true }) await sleep(10) await db.users.delete(id) - const [h1, h2, h3, h4, h5] = await db.users.findHistory(id) + const { result: [h1, h2, h3, h4, h5] } = await db.users.findHistory(id) assert(h1.type === "write") assert(h1.value.username === mockUser1.username) assert(h1.timestamp.valueOf() <= h2.timestamp.valueOf()) @@ -87,13 +87,13 @@ Deno.test("serialized_collection - history", async (t) => { }) const id = "id" - await db.users.write(id, mockUser1) + await db.users.set(id, mockUser1, { overwrite: true }) await sleep(10) await db.users.update(id, mockUser2) await sleep(10) await db.users.update(id, mockUser3) - const [h1, h2, h3] = await db.users.findHistory(id) + const { result: [h1, h2, h3] } = await db.users.findHistory(id) assert(h1.type === "write") assert(h1.value.username === mockUser1.username) assert(h1.timestamp.valueOf() <= h2.timestamp.valueOf()) @@ -118,15 +118,15 @@ Deno.test("serialized_collection - history", async (t) => { }) const id = "id" - await db.users.write(id, mockUser1) + await db.users.set(id, mockUser1, { overwrite: true }) await sleep(10) await db.users.deleteMany() await sleep(10) - await db.users.write(id, mockUser2) + await db.users.set(id, mockUser2, { overwrite: true }) await sleep(10) await db.users.deleteMany({ filter: () => true }) - const [h1, h2, h3, h4] = await db.users.findHistory(id) + const { result: [h1, h2, h3, h4] } = await db.users.findHistory(id) assert(h1.type === "write") assert(h1.value.username === mockUser1.username) assert(h1.timestamp.valueOf() <= h2.timestamp.valueOf()) @@ -149,17 +149,58 @@ Deno.test("serialized_collection - history", async (t) => { }) const id = "id" - await db.users.write(id, mockUser1) + await db.users.set(id, mockUser1, { overwrite: true }) await db.users.update(id, mockUser2) await db.users.delete(id) await db.users.deleteMany() - const history = await db.users.findHistory(id) + const { result: history } = await db.users.findHistory(id) assert(history.length === 0) }) }, ) + await t.step("Should find filtered history", async () => { + await useKv(async (kv) => { + const db = kvdex(kv, { + users: collection(model(), { + history: true, + serialize: "auto", + }), + }) + + const id = "id" + await db.users.set(id, mockUser1, { overwrite: true }) + await db.users.delete(id) + await db.users.set(id, mockUser2, { overwrite: true }) + await db.users.update(id, mockUser3) + + const { result: history1 } = await db.users.findHistory(id, { + filter: (entry) => entry.type === "delete", + }) + + const { result: history2 } = await db.users.findHistory(id, { + filter: (entry) => + entry.type === "write" && entry.value.age === mockUser1.age, + }) + + assert(history1.length === 1) + assert(history2.length === 2) + + assert( + history2.some((h) => + h.type === "write" && h.value.username === mockUser1.username + ), + ) + + assert( + history2.some((h) => + h.type === "write" && h.value.username === mockUser2.username + ), + ) + }) + }) + await t.step("Should delete all document history", async () => { await useKv(async (kv) => { const db = kvdex(kv, { @@ -170,22 +211,22 @@ Deno.test("serialized_collection - history", async (t) => { }) const id = "id" - await db.users.write(id, mockUser1) - await db.users.write(id, mockUser2) - await db.users.write(id, mockUser3) + await db.users.set(id, mockUser1, { overwrite: true }) + await db.users.set(id, mockUser2, { overwrite: true }) + await db.users.set(id, mockUser3, { overwrite: true }) const cr = await db.users.add(mockUser1) assert(cr.ok) - const history1_1 = await db.users.findHistory(id) - const history1_2 = await db.users.findHistory(cr.id) + const { result: history1_1 } = await db.users.findHistory(id) + const { result: history1_2 } = await db.users.findHistory(cr.id) assert(history1_1.length === 3) assert(history1_2.length === 1) await db.users.deleteHistory(id) - const history2_1 = await db.users.findHistory(id) - const history2_2 = await db.users.findHistory(cr.id) + const { result: history2_1 } = await db.users.findHistory(id) + const { result: history2_2 } = await db.users.findHistory(cr.id) assert(history2_1.length === 0) assert(history2_2.length === 1) }) diff --git a/tests/serialized_indexable_collection/history.test.ts b/tests/serialized_indexable_collection/history.test.ts index 52ccc3a..e9323b9 100644 --- a/tests/serialized_indexable_collection/history.test.ts +++ b/tests/serialized_indexable_collection/history.test.ts @@ -21,13 +21,13 @@ Deno.test("serialized_indexable_collection - history", async (t) => { }) const id = "id" - await db.users.write(id, mockUser1) + await db.users.set(id, mockUser1, { overwrite: true }) await sleep(10) - await db.users.write(id, mockUser2) + await db.users.set(id, mockUser2, { overwrite: true }) await sleep(10) - await db.users.write(id, mockUser3) + await db.users.set(id, mockUser3, { overwrite: true }) - const [h1, h2, h3] = await db.users.findHistory(id) + const { result: [h1, h2, h3] } = await db.users.findHistory(id) assert(h1.type === "write") assert(h1.value.username === mockUser1.username) assert(h1.timestamp.valueOf() <= h2.timestamp.valueOf()) @@ -56,17 +56,17 @@ Deno.test("serialized_indexable_collection - history", async (t) => { }) const id = "id" - await db.users.write(id, mockUser1) + await db.users.set(id, mockUser1, { overwrite: true }) await sleep(10) await db.users.delete(id) await sleep(10) - await db.users.write(id, mockUser2) + await db.users.set(id, mockUser2, { overwrite: true }) await sleep(10) - await db.users.write(id, mockUser3) + await db.users.set(id, mockUser3, { overwrite: true }) await sleep(10) await db.users.delete(id) - const [h1, h2, h3, h4, h5] = await db.users.findHistory(id) + const { result: [h1, h2, h3, h4, h5] } = await db.users.findHistory(id) assert(h1.type === "write") assert(h1.value.username === mockUser1.username) assert(h1.timestamp.valueOf() <= h2.timestamp.valueOf()) @@ -99,13 +99,13 @@ Deno.test("serialized_indexable_collection - history", async (t) => { }) const id = "id" - await db.users.write(id, mockUser1) + await db.users.set(id, mockUser1, { overwrite: true }) await sleep(10) await db.users.update(id, mockUser2) await sleep(10) await db.users.update(id, mockUser3) - const [h1, h2, h3] = await db.users.findHistory(id) + const { result: [h1, h2, h3] } = await db.users.findHistory(id) assert(h1.type === "write") assert(h1.value.username === mockUser1.username) assert(h1.timestamp.valueOf() <= h2.timestamp.valueOf()) @@ -134,15 +134,15 @@ Deno.test("serialized_indexable_collection - history", async (t) => { }) const id = "id" - await db.users.write(id, mockUser1) + await db.users.set(id, mockUser1, { overwrite: true }) await sleep(10) await db.users.deleteMany() await sleep(10) - await db.users.write(id, mockUser2) + await db.users.set(id, mockUser2, { overwrite: true }) await sleep(10) await db.users.deleteMany({ filter: () => true }) - const [h1, h2, h3, h4] = await db.users.findHistory(id) + const { result: [h1, h2, h3, h4] } = await db.users.findHistory(id) assert(h1.type === "write") assert(h1.value.username === mockUser1.username) assert(h1.timestamp.valueOf() <= h2.timestamp.valueOf()) @@ -171,17 +171,62 @@ Deno.test("serialized_indexable_collection - history", async (t) => { }) const id = "id" - await db.users.write(id, mockUser1) + await db.users.set(id, mockUser1, { overwrite: true }) await db.users.update(id, mockUser2) await db.users.delete(id) await db.users.deleteMany() - const history = await db.users.findHistory(id) + const { result: history } = await db.users.findHistory(id) assert(history.length === 0) }) }, ) + await t.step("Should find filtered history", async () => { + await useKv(async (kv) => { + const db = kvdex(kv, { + users: collection(model(), { + history: true, + serialize: "auto", + indices: { + username: "primary", + age: "secondary", + }, + }), + }) + + const id = "id" + await db.users.set(id, mockUser1, { overwrite: true }) + await db.users.delete(id) + await db.users.set(id, mockUser2, { overwrite: true }) + await db.users.update(id, mockUser3) + + const { result: history1 } = await db.users.findHistory(id, { + filter: (entry) => entry.type === "delete", + }) + + const { result: history2 } = await db.users.findHistory(id, { + filter: (entry) => + entry.type === "write" && entry.value.age === mockUser1.age, + }) + + assert(history1.length === 1) + assert(history2.length === 2) + + assert( + history2.some((h) => + h.type === "write" && h.value.username === mockUser1.username + ), + ) + + assert( + history2.some((h) => + h.type === "write" && h.value.username === mockUser2.username + ), + ) + }) + }) + await t.step("Should delete all document history", async () => { await useKv(async (kv) => { const db = kvdex(kv, { @@ -196,22 +241,22 @@ Deno.test("serialized_indexable_collection - history", async (t) => { }) const id = "id" - await db.users.write(id, mockUser1) - await db.users.write(id, mockUser2) - await db.users.write(id, mockUser3) + await db.users.set(id, mockUser1, { overwrite: true }) + await db.users.set(id, mockUser2, { overwrite: true }) + await db.users.set(id, mockUser3, { overwrite: true }) const cr = await db.users.add(generateLargeUsers(1)[0]) assert(cr.ok) - const history1_1 = await db.users.findHistory(id) - const history1_2 = await db.users.findHistory(cr.id) + const { result: history1_1 } = await db.users.findHistory(id) + const { result: history1_2 } = await db.users.findHistory(cr.id) assert(history1_1.length === 3) assert(history1_2.length === 1) await db.users.deleteHistory(id) - const history2_1 = await db.users.findHistory(id) - const history2_2 = await db.users.findHistory(cr.id) + const { result: history2_1 } = await db.users.findHistory(id) + const { result: history2_2 } = await db.users.findHistory(cr.id) assert(history2_1.length === 0) assert(history2_2.length === 1) }) From 41ed3f923e24c2e7031c1234829f53f6da29fafe Mon Sep 17 00:00:00 2001 From: oliver-oloughlin Date: Fri, 8 Dec 2023 15:56:31 +0100 Subject: [PATCH 2/3] feat: added upsert by id/index + passing tests --- src/collection.ts | 55 +++++++++++++ src/types.ts | 31 ++++++- tests/collection/upsert.test.ts | 44 ++++++++++ tests/indexable_collection/upsert.test.ts | 81 +++++++++++++++++++ tests/serialized_collection/upsert.test.ts | 44 ++++++++++ .../upsert.test.ts | 81 +++++++++++++++++++ 6 files changed, 334 insertions(+), 2 deletions(-) create mode 100644 tests/collection/upsert.test.ts create mode 100644 tests/indexable_collection/upsert.test.ts create mode 100644 tests/serialized_collection/upsert.test.ts create mode 100644 tests/serialized_indexable_collection/upsert.test.ts diff --git a/src/collection.ts b/src/collection.ts index 058b402..5da5aa9 100644 --- a/src/collection.ts +++ b/src/collection.ts @@ -11,6 +11,7 @@ import type { HistoryEntry, IdempotentListener, IdGenerator, + IdUpsertInput, IndexDataEntry, KvId, KvKey, @@ -22,6 +23,7 @@ import type { ParseInputType, PossibleCollectionOptions, PrimaryIndexKeys, + PrimaryIndexUpsertInput, QueueHandlers, QueueListenerOptions, QueueMessageHandler, @@ -33,6 +35,8 @@ import type { UpdateData, UpdateManyOptions, UpdateOptions, + UpsertInput, + UpsertOptions, WatchOptions, } from "./types.ts" import { @@ -830,6 +834,57 @@ export class Collection< ) } + async upsert< + const TIndex extends PrimaryIndexKeys, + >( + input: UpsertInput, + options?: UpsertOptions, + ) { + // Check if is id or primary index upsert + if ((input as any).index !== undefined) { + const inp = input as PrimaryIndexUpsertInput + + // First attempt update + const updateCr = await this.updateByPrimaryIndex( + ...inp.index, + inp.update, + options, + ) + + if (updateCr.ok) { + return updateCr + } + + // If id is present, set new entry with given id + if (inp.id) { + return await this.set(inp.id, inp.set, { + ...options, + overwrite: false, + }) + } + + // If no id, set new entry with generated id + return await this.add(inp.set, { + ...options, + overwrite: false, + }) + } else { + // First attempt update + const id = (input as IdUpsertInput).id + const updateCr = await this.update(id, input.update, options) + + if (updateCr.ok) { + return updateCr + } + + // Set new entry with given id + return await this.set(id, input.set, { + ...options, + overwrite: false, + }) + } + } + /** * Update the value of multiple existing documents in the collection. * diff --git a/src/types.ts b/src/types.ts index e955344..c30bd18 100644 --- a/src/types.ts +++ b/src/types.ts @@ -351,9 +351,9 @@ export type SetOptions = NonNullable["2"]> & { export type ListOptions = Deno.KvListOptions & { /** - * Filter documents based on predicate. + * Filter based on predicate. * - * @param doc - Document. + * @param value - Input value. * @returns true or false. */ filter?: (value: T) => boolean @@ -415,6 +415,33 @@ export type QueueListenerOptions = { export type WatchOptions = NonNullable[1]> +export type IdUpsertInput = { + id: KvId + set: ParseInputType + update: UpdateData +} + +export type PrimaryIndexUpsertInput< + TInput, + TOutput extends KvValue, + TIndex, +> = { + id?: KvId + index: [TIndex, CheckKeyOf] + set: ParseInputType + update: UpdateData +} + +export type UpsertInput< + TInput, + TOutput extends KvValue, + TIndex, +> = + | IdUpsertInput + | PrimaryIndexUpsertInput + +export type UpsertOptions = UpdateOptions + /********************/ /* */ /* SCHEMA TYPES */ diff --git a/tests/collection/upsert.test.ts b/tests/collection/upsert.test.ts new file mode 100644 index 0000000..ebeae8d --- /dev/null +++ b/tests/collection/upsert.test.ts @@ -0,0 +1,44 @@ +import { assert } from "../deps.ts" +import { mockUser1, mockUser2, mockUser3 } from "../mocks.ts" +import { useDb } from "../utils.ts" + +Deno.test("collection - upsert", async (t) => { + await t.step("Should set new doucment entry by id", async () => { + await useDb(async (db) => { + const id = "id" + + const cr = await db.users.upsert({ + id: id, + set: mockUser2, + update: mockUser3, + }) + + assert(cr.ok) + + const doc = await db.users.find(id) + assert(doc !== null) + assert(doc.value.username === mockUser2.username) + }) + }) + + await t.step("Should update existing document entry by id", async () => { + await useDb(async (db) => { + const id = "id" + + const cr1 = await db.users.set(id, mockUser1) + assert(cr1.ok) + + const cr2 = await db.users.upsert({ + id: id, + set: mockUser2, + update: mockUser3, + }) + + assert(cr2.ok) + + const doc = await db.users.find(id) + assert(doc !== null) + assert(doc.value.username === mockUser3.username) + }) + }) +}) diff --git a/tests/indexable_collection/upsert.test.ts b/tests/indexable_collection/upsert.test.ts new file mode 100644 index 0000000..a712c3a --- /dev/null +++ b/tests/indexable_collection/upsert.test.ts @@ -0,0 +1,81 @@ +import { assert } from "../deps.ts" +import { mockUser1, mockUser2, mockUser3 } from "../mocks.ts" +import { useDb } from "../utils.ts" + +Deno.test("indexable_collection - upsert", async (t) => { + await t.step("Should set new doucment entry by id", async () => { + await useDb(async (db) => { + const id = "id" + + const cr = await db.i_users.upsert({ + id: id, + set: mockUser2, + update: mockUser3, + }) + + assert(cr.ok) + + const doc = await db.i_users.find(id) + assert(doc !== null) + assert(doc.value.username === mockUser2.username) + }) + }) + + await t.step("Should set new doucment entry by index", async () => { + await useDb(async (db) => { + const cr = await db.i_users.upsert({ + index: ["username", mockUser1.username], + set: mockUser2, + update: mockUser3, + }) + + assert(cr.ok) + + const doc = await db.i_users.find(cr.id) + assert(doc !== null) + assert(doc.value.username === mockUser2.username) + }) + }) + + await t.step("Should update existing document entry by id", async () => { + await useDb(async (db) => { + const id = "id" + + const cr1 = await db.i_users.set(id, mockUser1) + assert(cr1.ok) + + const cr2 = await db.i_users.upsert({ + id: id, + set: mockUser2, + update: mockUser3, + }) + + assert(cr2.ok) + + const doc = await db.i_users.find(id) + assert(doc !== null) + assert(doc.value.username === mockUser3.username) + }) + }) + + await t.step("Should update existing document entry by index", async () => { + await useDb(async (db) => { + const id = "id" + + const cr1 = await db.i_users.set(id, mockUser1) + assert(cr1.ok) + + const cr2 = await db.i_users.upsert({ + index: ["username", mockUser1.username], + set: mockUser2, + update: mockUser3, + }) + + assert(cr2.ok) + + const doc = await db.i_users.find(id) + assert(doc !== null) + assert(doc.value.username === mockUser3.username) + }) + }) +}) diff --git a/tests/serialized_collection/upsert.test.ts b/tests/serialized_collection/upsert.test.ts new file mode 100644 index 0000000..9a48761 --- /dev/null +++ b/tests/serialized_collection/upsert.test.ts @@ -0,0 +1,44 @@ +import { assert } from "../deps.ts" +import { mockUser1, mockUser2, mockUser3 } from "../mocks.ts" +import { useDb } from "../utils.ts" + +Deno.test("serialized_collection - upsert", async (t) => { + await t.step("Should set new doucment entry by id", async () => { + await useDb(async (db) => { + const id = "id" + + const cr = await db.s_users.upsert({ + id: id, + set: mockUser2, + update: mockUser3, + }) + + assert(cr.ok) + + const doc = await db.s_users.find(id) + assert(doc !== null) + assert(doc.value.username === mockUser2.username) + }) + }) + + await t.step("Should update existing document entry by id", async () => { + await useDb(async (db) => { + const id = "id" + + const cr1 = await db.s_users.set(id, mockUser1) + assert(cr1.ok) + + const cr2 = await db.s_users.upsert({ + id: id, + set: mockUser2, + update: mockUser3, + }) + + assert(cr2.ok) + + const doc = await db.s_users.find(id) + assert(doc !== null) + assert(doc.value.username === mockUser3.username) + }) + }) +}) diff --git a/tests/serialized_indexable_collection/upsert.test.ts b/tests/serialized_indexable_collection/upsert.test.ts new file mode 100644 index 0000000..0fa344f --- /dev/null +++ b/tests/serialized_indexable_collection/upsert.test.ts @@ -0,0 +1,81 @@ +import { assert } from "../deps.ts" +import { mockUser1, mockUser2, mockUser3 } from "../mocks.ts" +import { useDb } from "../utils.ts" + +Deno.test("serialized_indexable_collection - upsert", async (t) => { + await t.step("Should set new doucment entry by id", async () => { + await useDb(async (db) => { + const id = "id" + + const cr = await db.is_users.upsert({ + id: id, + set: mockUser2, + update: mockUser3, + }) + + assert(cr.ok) + + const doc = await db.is_users.find(id) + assert(doc !== null) + assert(doc.value.username === mockUser2.username) + }) + }) + + await t.step("Should set new doucment entry by index", async () => { + await useDb(async (db) => { + const cr = await db.is_users.upsert({ + index: ["username", mockUser1.username], + set: mockUser2, + update: mockUser3, + }) + + assert(cr.ok) + + const doc = await db.is_users.find(cr.id) + assert(doc !== null) + assert(doc.value.username === mockUser2.username) + }) + }) + + await t.step("Should update existing document entry by id", async () => { + await useDb(async (db) => { + const id = "id" + + const cr1 = await db.is_users.set(id, mockUser1) + assert(cr1.ok) + + const cr2 = await db.is_users.upsert({ + id: id, + set: mockUser2, + update: mockUser3, + }) + + assert(cr2.ok) + + const doc = await db.is_users.find(id) + assert(doc !== null) + assert(doc.value.username === mockUser3.username) + }) + }) + + await t.step("Should update existing document entry by index", async () => { + await useDb(async (db) => { + const id = "id" + + const cr1 = await db.is_users.set(id, mockUser1) + assert(cr1.ok) + + const cr2 = await db.is_users.upsert({ + index: ["username", mockUser1.username], + set: mockUser2, + update: mockUser3, + }) + + assert(cr2.ok) + + const doc = await db.is_users.find(id) + assert(doc !== null) + assert(doc.value.username === mockUser3.username) + }) + }) +}) From babf494018b8f92015638aa6f11214a3e3d38cec Mon Sep 17 00:00:00 2001 From: oliver-oloughlin Date: Fri, 8 Dec 2023 16:13:02 +0100 Subject: [PATCH 3/3] chore: updated readme and jsdoc for upsert --- README.md | 46 ++++++++++++++++++++++++++++++++++++++++++- src/collection.ts | 50 ++++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 94 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 35f0445..6086228 100644 --- a/README.md +++ b/README.md @@ -41,6 +41,7 @@ _Supported Deno verisons:_ **^1.38.5** - [updateByPrimaryIndex()](#updatebyprimaryindex) - [updateBySecondaryIndex()](#updatebysecondaryindex) - [updateMany()](#updatemany) + - [upsert()](#upsert) - [delete()](#delete) - [deleteByPrimaryIndex()](#deletebyprimaryindex) - [deleteBySecondaryIndex()](#deletebysecondaryindex) @@ -245,7 +246,7 @@ timestamp, type of either "write" or "delete", and a copy of the document value if the type is "write". ```ts -const history = await db.users.findHistory("user_id") +const { result } = await db.users.findHistory("user_id") ``` ### findUndelivered() @@ -406,6 +407,49 @@ const { result } = await db.users.updateMany({ age: 67 }, { const { result } = await db.users.updateMany({ username: "XuserX" }) ``` +### upsert() + +Update an existing document by either id or primary index, or set a new document +entry if no document with matching id/index exists. When upserting by primary +index, an id can be optionally specified which will be used when setting a new +document entry, otherwise an id will be generated. + +```ts +// Upsert by id +const result1 = await db.users.upsert({ + id: "user_id", + update: { username: "Chris" }, + set: { + username: "Chris", + age: 54, + activities: ["bowling"], + address: { + country: "USA", + city: "Las Vegas" + street: "St. Boulevard" + houseNumber: 23 + } + } +}) + +// Upsert by index +const result2 = await db.users.upsert({ + index: ["username", "Jack"], + update: { username: "Chris" }, + set: { + username: "Chris", + age: 54, + activities: ["bowling"], + address: { + country: "USA", + city: "Las Vegas" + street: "St. Boulevard" + houseNumber: 23 + } + } +}) +``` + ### delete() Delete one or more documents with the given ids from the KV store. diff --git a/src/collection.ts b/src/collection.ts index 5da5aa9..8d85080 100644 --- a/src/collection.ts +++ b/src/collection.ts @@ -428,7 +428,7 @@ export class Collection< * * @example * ```ts - * const history = await db.users.findHistory("user_id") + * const { result } = await db.users.findHistory("user_id") * ``` * * @param id - Document id. @@ -834,6 +834,54 @@ export class Collection< ) } + /** + * Update an existing document by either id or primary index, or set a new document + * entry if no document with matching id/index exists. + * + * When upserting by primary index, an id can be optionally specified which + * will be used when setting a new document entry, otherwise an id will be generated. + * + * @example + * ```ts + * // Upsert by id + * const result1 = await db.users.upsert({ + * id: "user_id", + * update: { username: "Chris" }, + * set: { + * username: "Chris", + * age: 54, + * activities: ["bowling"], + * address: { + * country: "USA", + * city: "Las Vegas" + * street: "St. Boulevard" + * houseNumber: 23 + * } + * } + * }) + * + * // Upsert by index + * const result2 = await db.users.upsert({ + * index: ["username", "Jack"], + * update: { username: "Chris" }, + * set: { + * username: "Chris", + * age: 54, + * activities: ["bowling"], + * address: { + * country: "USA", + * city: "Las Vegas" + * street: "St. Boulevard" + * houseNumber: 23 + * } + * } + * }) + * ``` + * + * @param input - Upsert input, including id or index, update data and set data. + * @param options - Upsert options. + * @returns A promise resolving to either CommitResult or CommitError. + */ async upsert< const TIndex extends PrimaryIndexKeys, >(