Skip to content
This repository has been archived by the owner on Jan 2, 2025. It is now read-only.

Commit

Permalink
Remove sqlite, refactor to use level (#69)
Browse files Browse the repository at this point in the history
* init

* checkpoint, db dump method, refactor with-db

* fix zod level db fns, migrate "create.ts" from dev_package_examples

* wip refactoring dev server to level from sqlite

* Added a new test for the `dev_package_examples/list.test.ts` file.

* Replaced Kysely query with ZodLevelDatabase dump and mapping to required format.

* list endpoint support

* wip progress on refactor to level

* db reset

* implement export file refactor

* implement download

* fix tests

* remove log line

* small fixes to prevent level open/close issues

* remove sqlite

* remove sqlite types
  • Loading branch information
seveibar authored Jul 7, 2024
1 parent 4d52124 commit 4deac86
Show file tree
Hide file tree
Showing 40 changed files with 634 additions and 387 deletions.
Binary file modified bun.lockb
Binary file not shown.
Binary file modified dev-server-api/bun.lockb
Binary file not shown.
4 changes: 0 additions & 4 deletions dev-server-api/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,6 @@
"build": "winterspec bundle-routes -i ./routes -o static-routes.ts"
},
"devDependencies": {
"@types/better-sqlite3": "^7.6.9",
"@types/bun": "latest",
"@types/sql.js": "^1.4.9",
"make-vfs": "^1.0.10"
Expand All @@ -16,9 +15,6 @@
"typescript": "^5.0.0"
},
"dependencies": {
"better-sqlite3": "^11.0.0",
"kysely": "^0.27.3",
"kysely-bun-sqlite": "^0.3.2",
"level": "^8.0.1",
"redaxios": "^0.5.1",
"winterspec": "0.0.81",
Expand Down
25 changes: 25 additions & 0 deletions dev-server-api/routes/api/db/download.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
import { withWinterSpec } from "src/with-winter-spec"
import { z } from "zod"

export default withWinterSpec({
methods: ["GET"],
jsonResponse: z.object({
dev_server_database_dump: z.any(),
}),
auth: "none",
})(async (req, ctx) => {
return new Response(
JSON.stringify(
{
dev_server_database_dump: await ctx.db.dump(),
},
null,
" "
),
{
headers: {
"content-type": "application/json",
},
}
)
})
40 changes: 15 additions & 25 deletions dev-server-api/routes/api/dev_package_examples/create.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,4 @@
import { withWinterSpec } from "src/with-winter-spec"
import { NotFoundError } from "edgespec/middleware"
import { z } from "zod"

export default withWinterSpec({
Expand All @@ -17,35 +16,26 @@ export default withWinterSpec({
file_path: z.string(),
tscircuit_soup: z.any(),
error: z.string().nullable().optional(),
last_updated_at: z.string().datetime(),
last_updated_at: z.string().datetime().nullable(), // TODO remove nullable
}),
}),
auth: "none",
})(async (req, ctx) => {
const tscircuit_soup = req.jsonBody.tscircuit_soup
? JSON.stringify(req.jsonBody.tscircuit_soup)
: undefined
const dev_package_example = await ctx.db
.insertInto("dev_package_example")
.values({
file_path: req.jsonBody.file_path,
export_name: req.jsonBody.export_name,
error: req.jsonBody.error,
tscircuit_soup,
is_loading: req.jsonBody.is_loading ? 1 : 0,
last_updated_at: new Date().toISOString(),
})
.onConflict((oc) =>
oc.columns(["file_path"]).doUpdateSet({
export_name: req.jsonBody.export_name,
error: req.jsonBody.error,
tscircuit_soup,
is_loading: req.jsonBody.is_loading ? 1 : 0,
last_updated_at: new Date().toISOString(),
})
)
.returningAll()
.executeTakeFirstOrThrow()

const existingDevPackageExample = await ctx.db.find("dev_package_example", {
file_path: req.jsonBody.file_path,
})

const dev_package_example = await ctx.db.put("dev_package_example", {
dev_package_example_id: existingDevPackageExample?.dev_package_example_id,
file_path: req.jsonBody.file_path,
export_name: req.jsonBody.export_name,
error: req.jsonBody.error,
tscircuit_soup,
is_loading: Boolean(req.jsonBody.is_loading),
last_updated_at: new Date().toISOString(),
})

return ctx.json({
dev_package_example,
Expand Down
33 changes: 12 additions & 21 deletions dev-server-api/routes/api/dev_package_examples/get.ts
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ export default withWinterSpec({
dev_package_example_id: z.coerce.number(),
file_path: z.string(),
tscircuit_soup: z.any(),
completed_edit_events: z.array(z.any()).nullable().default(null),
completed_edit_events: z.array(z.any()).default([]),
is_loading: z.boolean(),
error: z.string().nullable().optional().default(null),
soup_last_updated_at: z.string().datetime().nullable().default(null),
Expand All @@ -26,30 +26,21 @@ export default withWinterSpec({
.datetime()
.nullable()
.default(null),
last_updated_at: z.string().datetime(),
last_updated_at: z.string().datetime().nullable(),
}),
}),
auth: "none",
})(async (req, ctx) => {
const dev_package_example = await ctx.db.get(
"dev_package_example",
req.commonParams.dev_package_example_id
)

if (!dev_package_example) {
throw new NotFoundError("Package not found")
}

return ctx.json({
dev_package_example: await ctx.db
.selectFrom("dev_package_example")
.selectAll()
.where(
"dev_package_example_id",
"=",
req.commonParams.dev_package_example_id
)
.executeTakeFirstOrThrow((e) => {
throw new NotFoundError("Package not found")
})
.then((r) => ({
...r,
is_loading: r.is_loading === 1,
tscircuit_soup: JSON.parse(r.tscircuit_soup),
completed_edit_events: r.completed_edit_events
? JSON.parse(r.completed_edit_events)
: null,
})),
dev_package_example,
})
})
39 changes: 17 additions & 22 deletions dev-server-api/routes/api/dev_package_examples/list.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,3 @@
import { sql } from "kysely"
import { withWinterSpec } from "src/with-winter-spec"
import { z } from "zod"

Expand All @@ -9,32 +8,28 @@ export default withWinterSpec({
z.object({
dev_package_example_id: z.coerce.number(),
file_path: z.string(),
export_name: z.string(),
is_loading: z.coerce.boolean(),
edit_events_last_updated_at: z
.string()
.datetime()
.nullable()
.default(null),
soup_last_updated_at: z.string().datetime().nullable().default(null),
last_updated_at: z.string().datetime(),
export_name: z.string().nullable(),
is_loading: z.boolean(),
edit_events_last_updated_at: z.string().datetime().nullable(),
soup_last_updated_at: z.string().datetime().nullable(),
last_updated_at: z.string().datetime().nullable(),
})
),
}),
auth: "none",
})(async (req, ctx) => {
const dev_package_examples = await ctx.db
.selectFrom("dev_package_example")
.select([
"dev_package_example_id",
"file_path",
"export_name",
"last_updated_at",
"edit_events_last_updated_at",
"soup_last_updated_at",
sql`(is_loading = 1)`.$castTo<boolean>().as("is_loading"),
])
.execute()
const dev_package_examples = (await ctx.db.list("dev_package_example")).map(
(dpe) => ({
dev_package_example_id: dpe.dev_package_example_id,
file_path: dpe.file_path,
export_name: dpe.export_name,
is_loading: dpe.is_loading,
edit_events_last_updated_at: dpe.edit_events_last_updated_at,
soup_last_updated_at: dpe.soup_last_updated_at,
last_updated_at: dpe.last_updated_at,
})
)

return ctx.json({
dev_package_examples,
})
Expand Down
76 changes: 39 additions & 37 deletions dev-server-api/routes/api/dev_package_examples/update.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
import { withWinterSpec } from "src/with-winter-spec"
import { NotFoundError } from "edgespec/middleware"
import { z } from "zod"
import { DevPackageExampleSchema } from "src/db/schema"

export default withWinterSpec({
methods: ["POST"],
Expand All @@ -12,46 +13,47 @@ export default withWinterSpec({
error: z.string().nullable().optional().default(null),
}),
jsonResponse: z.object({
dev_package_example: z.object({
dev_package_example_id: z.coerce.number(),
tscircuit_soup: z.any(),
error: z.string().nullable().optional(),
last_updated_at: z.string().datetime(),
}),
dev_package_example: DevPackageExampleSchema,
}),
auth: "none",
})(async (req, ctx) => {
const dev_package_example = await ctx.db.get(
"dev_package_example",
req.jsonBody.dev_package_example_id
)

if (!dev_package_example) {
throw new NotFoundError("Package not found")
}

const new_dev_package_example = {
...dev_package_example,
}

if (req.jsonBody.completed_edit_events !== undefined) {
new_dev_package_example.completed_edit_events =
req.jsonBody.completed_edit_events
new_dev_package_example.edit_events_last_updated_at =
new Date().toISOString()
}
if (req.jsonBody.edit_events_last_applied_at !== undefined) {
new_dev_package_example.edit_events_last_applied_at =
req.jsonBody.edit_events_last_applied_at
}
if (req.jsonBody.tscircuit_soup !== undefined) {
new_dev_package_example.tscircuit_soup = req.jsonBody.tscircuit_soup
new_dev_package_example.error = null
new_dev_package_example.soup_last_updated_at = new Date().toISOString()
}
if (req.jsonBody.error !== undefined) {
new_dev_package_example.error = req.jsonBody.error
}

new_dev_package_example.last_updated_at = new Date().toISOString()

await ctx.db.put("dev_package_example", new_dev_package_example)

return ctx.json({
dev_package_example: await ctx.db
.updateTable("dev_package_example")
.set({
last_updated_at: new Date().toISOString(),
})
.$if(req.jsonBody.tscircuit_soup !== undefined, (q) =>
q
.set("tscircuit_soup", req.jsonBody.tscircuit_soup)
.set("error", null)
.set("soup_last_updated_at", new Date().toISOString())
)
.$if(req.jsonBody.error !== undefined, (q) =>
q.set("error", req.jsonBody.error)
)
.$if(req.jsonBody.completed_edit_events !== undefined, (q) =>
q
.set(
"completed_edit_events",
JSON.stringify(req.jsonBody.completed_edit_events)
)
.set("edit_events_last_updated_at", new Date().toISOString())
)
.$if(req.jsonBody.edit_events_last_applied_at !== undefined, (q) =>
q.set(
"edit_events_last_applied_at",
req.jsonBody.edit_events_last_applied_at!
)
)
.returningAll()
.where("dev_package_example_id", "=", req.jsonBody.dev_package_example_id)
.executeTakeFirstOrThrow(),
dev_package_example: new_dev_package_example,
})
})
14 changes: 6 additions & 8 deletions dev-server-api/routes/api/dev_server/reset.ts
Original file line number Diff line number Diff line change
@@ -1,15 +1,13 @@
import { sql } from "kysely"
import { withWinterSpec } from "src/with-winter-spec"
import { z } from "zod"
import { unlinkSync } from "fs"
import { getDbFilePath } from "src/db/get-db"

export default (req: Request) => {
unlinkSync(getDbFilePath())

export default withWinterSpec({
methods: ["GET", "POST"],
auth: "none",
})(async (req, ctx) => {
await ctx.db.clear()
return new Response(JSON.stringify({}), {
headers: {
"content-type": "application/json",
},
})
}
})
23 changes: 10 additions & 13 deletions dev-server-api/routes/api/export_files/create.ts
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
import { ExportFileSchema } from "src/db/schema"
import { publicMapExportFile } from "src/lib/public-mapping/public-map-export-file"
import { export_file } from "src/lib/zod/export_file"
import { withWinterSpec } from "src/with-winter-spec"
Expand All @@ -8,25 +9,21 @@ export default withWinterSpec({
jsonBody: z.object({
export_request_id: z.number().int(),
file_name: z.string(),
file_content_base64: z.string().transform((a) => Buffer.from(a, "base64")),
file_content_base64: z.string(),
}),
jsonResponse: z.object({
export_file,
export_file: ExportFileSchema.omit({ file_content_base64: true }),
}),
auth: "none",
})(async (req, ctx) => {
const db_export_file = await ctx.db
.insertInto("export_file")
.values({
export_request_id: req.jsonBody.export_request_id,
file_name: req.jsonBody.file_name,
file_content: req.jsonBody.file_content_base64,
created_at: new Date().toISOString(),
})
.returningAll()
.executeTakeFirstOrThrow()
const export_file = await ctx.db.put("export_file", {
export_request_id: req.jsonBody.export_request_id,
file_name: req.jsonBody.file_name,
file_content_base64: req.jsonBody.file_content_base64,
created_at: new Date().toISOString(),
})

return ctx.json({
export_file: publicMapExportFile(db_export_file),
export_file,
})
})
17 changes: 10 additions & 7 deletions dev-server-api/routes/api/export_files/download.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
import { publicMapExportFile } from "src/lib/public-mapping/public-map-export-file"
import { export_file } from "src/lib/zod/export_file"
import { withWinterSpec } from "src/with-winter-spec"
import { NotFoundError } from "winterspec/middleware"
import { z } from "zod"

export default withWinterSpec({
Expand All @@ -10,15 +11,17 @@ export default withWinterSpec({
}),
auth: "none",
})(async (req, ctx) => {
const db_export_file = await ctx.db
.selectFrom("export_file")
.selectAll()
.where("export_file_id", "=", req.query.export_file_id)
.executeTakeFirstOrThrow()
const export_file = await ctx.db.get("export_file", req.query.export_file_id)

return new Response(db_export_file.file_content, {
if (!export_file) {
throw new NotFoundError("Export file not found")
}

const file_content = Buffer.from(export_file.file_content_base64!, "base64")

return new Response(file_content, {
headers: {
"Content-Disposition": `attachment; filename="${db_export_file.file_name}"`,
"Content-Disposition": `attachment; filename="${export_file.file_name}"`,
},
})
})
Loading

0 comments on commit 4deac86

Please sign in to comment.