Remove sqlite, refactor to use level (#69)

* init

* checkpoint, db dump method, refactor with-db

* fix zod level db fns, migrate "create.ts" from dev_package_examples

* wip refactoring dev server to level from sqlite

* Added a new test for the `dev_package_examples/list.test.ts` file.

* Replaced Kysely query with ZodLevelDatabase dump and mapping to required format.

* list endpoint support

* wip progress on refactor to level

* db reset

* implement export file refactor

* implement download

* fix tests

* remove log line

* small fixes to prevent level open/close issues

* remove sqlite

* remove sqlite types

Author: seveibar

bun.lockb

@@ -7,7 +7,6 @@
     "build": "winterspec bundle-routes -i ./routes -o static-routes.ts"
   },
   "devDependencies": {
-    "@types/better-sqlite3": "^7.6.9",
     "@types/bun": "latest",
     "@types/sql.js": "^1.4.9",
     "make-vfs": "^1.0.10"
@@ -16,9 +15,6 @@
     "typescript": "^5.0.0"
   },
   "dependencies": {
-    "better-sqlite3": "^11.0.0",
-    "kysely": "^0.27.3",
-    "kysely-bun-sqlite": "^0.3.2",
     "level": "^8.0.1",
     "redaxios": "^0.5.1",
     "winterspec": "0.0.81",

dev-server-api/bun.lockb

@@ -0,0 +1,25 @@
+import { withWinterSpec } from "src/with-winter-spec"
+import { z } from "zod"
+
+export default withWinterSpec({
+  methods: ["GET"],
+  jsonResponse: z.object({
+    dev_server_database_dump: z.any(),
+  }),
+  auth: "none",
+})(async (req, ctx) => {
+  return new Response(
+    JSON.stringify(
+      {
+        dev_server_database_dump: await ctx.db.dump(),
+      },
+      null,
+      "  "
+    ),
+    {
+      headers: {
+        "content-type": "application/json",
+      },
+    }
+  )
+})

dev-server-api/package.json

@@ -1,5 +1,4 @@
 import { withWinterSpec } from "src/with-winter-spec"
-import { NotFoundError } from "edgespec/middleware"
 import { z } from "zod"
 
 export default withWinterSpec({
@@ -17,35 +16,26 @@ export default withWinterSpec({
       file_path: z.string(),
       tscircuit_soup: z.any(),
       error: z.string().nullable().optional(),
-      last_updated_at: z.string().datetime(),
+      last_updated_at: z.string().datetime().nullable(), // TODO remove nullable
     }),
   }),
   auth: "none",
 })(async (req, ctx) => {
   const tscircuit_soup = req.jsonBody.tscircuit_soup
-    ? JSON.stringify(req.jsonBody.tscircuit_soup)
-    : undefined
-  const dev_package_example = await ctx.db
-    .insertInto("dev_package_example")
-    .values({
-      file_path: req.jsonBody.file_path,
-      export_name: req.jsonBody.export_name,
-      error: req.jsonBody.error,
-      tscircuit_soup,
-      is_loading: req.jsonBody.is_loading ? 1 : 0,
-      last_updated_at: new Date().toISOString(),
-    })
-    .onConflict((oc) =>
-      oc.columns(["file_path"]).doUpdateSet({
-        export_name: req.jsonBody.export_name,
-        error: req.jsonBody.error,
-        tscircuit_soup,
-        is_loading: req.jsonBody.is_loading ? 1 : 0,
-        last_updated_at: new Date().toISOString(),
-      })
-    )
-    .returningAll()
-    .executeTakeFirstOrThrow()
+
+  const existingDevPackageExample = await ctx.db.find("dev_package_example", {
+    file_path: req.jsonBody.file_path,
+  })
+
+  const dev_package_example = await ctx.db.put("dev_package_example", {
+    dev_package_example_id: existingDevPackageExample?.dev_package_example_id,
+    file_path: req.jsonBody.file_path,
+    export_name: req.jsonBody.export_name,
+    error: req.jsonBody.error,
+    tscircuit_soup,
+    is_loading: Boolean(req.jsonBody.is_loading),
+    last_updated_at: new Date().toISOString(),
+  })
 
   return ctx.json({
     dev_package_example,

dev-server-api/routes/api/db/download.ts

@@ -12,7 +12,7 @@ export default withWinterSpec({
       dev_package_example_id: z.coerce.number(),
       file_path: z.string(),
       tscircuit_soup: z.any(),
-      completed_edit_events: z.array(z.any()).nullable().default(null),
+      completed_edit_events: z.array(z.any()).default([]),
       is_loading: z.boolean(),
       error: z.string().nullable().optional().default(null),
       soup_last_updated_at: z.string().datetime().nullable().default(null),
@@ -26,30 +26,21 @@ export default withWinterSpec({
         .datetime()
         .nullable()
         .default(null),
-      last_updated_at: z.string().datetime(),
+      last_updated_at: z.string().datetime().nullable(),
     }),
   }),
   auth: "none",
 })(async (req, ctx) => {
+  const dev_package_example = await ctx.db.get(
+    "dev_package_example",
+    req.commonParams.dev_package_example_id
+  )
+
+  if (!dev_package_example) {
+    throw new NotFoundError("Package not found")
+  }
+
   return ctx.json({
-    dev_package_example: await ctx.db
-      .selectFrom("dev_package_example")
-      .selectAll()
-      .where(
-        "dev_package_example_id",
-        "=",
-        req.commonParams.dev_package_example_id
-      )
-      .executeTakeFirstOrThrow((e) => {
-        throw new NotFoundError("Package not found")
-      })
-      .then((r) => ({
-        ...r,
-        is_loading: r.is_loading === 1,
-        tscircuit_soup: JSON.parse(r.tscircuit_soup),
-        completed_edit_events: r.completed_edit_events
-          ? JSON.parse(r.completed_edit_events)
-          : null,
-      })),
+    dev_package_example,
   })
 })

dev-server-api/routes/api/dev_package_examples/create.ts

@@ -1,4 +1,3 @@
-import { sql } from "kysely"
 import { withWinterSpec } from "src/with-winter-spec"
 import { z } from "zod"
 
@@ -9,32 +8,28 @@ export default withWinterSpec({
       z.object({
         dev_package_example_id: z.coerce.number(),
         file_path: z.string(),
-        export_name: z.string(),
-        is_loading: z.coerce.boolean(),
-        edit_events_last_updated_at: z
-          .string()
-          .datetime()
-          .nullable()
-          .default(null),
-        soup_last_updated_at: z.string().datetime().nullable().default(null),
-        last_updated_at: z.string().datetime(),
+        export_name: z.string().nullable(),
+        is_loading: z.boolean(),
+        edit_events_last_updated_at: z.string().datetime().nullable(),
+        soup_last_updated_at: z.string().datetime().nullable(),
+        last_updated_at: z.string().datetime().nullable(),
       })
     ),
   }),
   auth: "none",
 })(async (req, ctx) => {
-  const dev_package_examples = await ctx.db
-    .selectFrom("dev_package_example")
-    .select([
-      "dev_package_example_id",
-      "file_path",
-      "export_name",
-      "last_updated_at",
-      "edit_events_last_updated_at",
-      "soup_last_updated_at",
-      sql`(is_loading = 1)`.$castTo<boolean>().as("is_loading"),
-    ])
-    .execute()
+  const dev_package_examples = (await ctx.db.list("dev_package_example")).map(
+    (dpe) => ({
+      dev_package_example_id: dpe.dev_package_example_id,
+      file_path: dpe.file_path,
+      export_name: dpe.export_name,
+      is_loading: dpe.is_loading,
+      edit_events_last_updated_at: dpe.edit_events_last_updated_at,
+      soup_last_updated_at: dpe.soup_last_updated_at,
+      last_updated_at: dpe.last_updated_at,
+    })
+  )
+
   return ctx.json({
     dev_package_examples,
   })

dev-server-api/routes/api/dev_package_examples/get.ts

@@ -1,6 +1,7 @@
 import { withWinterSpec } from "src/with-winter-spec"
 import { NotFoundError } from "edgespec/middleware"
 import { z } from "zod"
+import { DevPackageExampleSchema } from "src/db/schema"
 
 export default withWinterSpec({
   methods: ["POST"],
@@ -12,46 +13,47 @@ export default withWinterSpec({
     error: z.string().nullable().optional().default(null),
   }),
   jsonResponse: z.object({
-    dev_package_example: z.object({
-      dev_package_example_id: z.coerce.number(),
-      tscircuit_soup: z.any(),
-      error: z.string().nullable().optional(),
-      last_updated_at: z.string().datetime(),
-    }),
+    dev_package_example: DevPackageExampleSchema,
   }),
   auth: "none",
 })(async (req, ctx) => {
+  const dev_package_example = await ctx.db.get(
+    "dev_package_example",
+    req.jsonBody.dev_package_example_id
+  )
+
+  if (!dev_package_example) {
+    throw new NotFoundError("Package not found")
+  }
+
+  const new_dev_package_example = {
+    ...dev_package_example,
+  }
+
+  if (req.jsonBody.completed_edit_events !== undefined) {
+    new_dev_package_example.completed_edit_events =
+      req.jsonBody.completed_edit_events
+    new_dev_package_example.edit_events_last_updated_at =
+      new Date().toISOString()
+  }
+  if (req.jsonBody.edit_events_last_applied_at !== undefined) {
+    new_dev_package_example.edit_events_last_applied_at =
+      req.jsonBody.edit_events_last_applied_at
+  }
+  if (req.jsonBody.tscircuit_soup !== undefined) {
+    new_dev_package_example.tscircuit_soup = req.jsonBody.tscircuit_soup
+    new_dev_package_example.error = null
+    new_dev_package_example.soup_last_updated_at = new Date().toISOString()
+  }
+  if (req.jsonBody.error !== undefined) {
+    new_dev_package_example.error = req.jsonBody.error
+  }
+
+  new_dev_package_example.last_updated_at = new Date().toISOString()
+
+  await ctx.db.put("dev_package_example", new_dev_package_example)
+
   return ctx.json({
-    dev_package_example: await ctx.db
-      .updateTable("dev_package_example")
-      .set({
-        last_updated_at: new Date().toISOString(),
-      })
-      .$if(req.jsonBody.tscircuit_soup !== undefined, (q) =>
-        q
-          .set("tscircuit_soup", req.jsonBody.tscircuit_soup)
-          .set("error", null)
-          .set("soup_last_updated_at", new Date().toISOString())
-      )
-      .$if(req.jsonBody.error !== undefined, (q) =>
-        q.set("error", req.jsonBody.error)
-      )
-      .$if(req.jsonBody.completed_edit_events !== undefined, (q) =>
-        q
-          .set(
-            "completed_edit_events",
-            JSON.stringify(req.jsonBody.completed_edit_events)
-          )
-          .set("edit_events_last_updated_at", new Date().toISOString())
-      )
-      .$if(req.jsonBody.edit_events_last_applied_at !== undefined, (q) =>
-        q.set(
-          "edit_events_last_applied_at",
-          req.jsonBody.edit_events_last_applied_at!
-        )
-      )
-      .returningAll()
-      .where("dev_package_example_id", "=", req.jsonBody.dev_package_example_id)
-      .executeTakeFirstOrThrow(),
+    dev_package_example: new_dev_package_example,
   })
 })

dev-server-api/routes/api/dev_package_examples/list.ts

@@ -1,15 +1,13 @@
-import { sql } from "kysely"
 import { withWinterSpec } from "src/with-winter-spec"
-import { z } from "zod"
-import { unlinkSync } from "fs"
-import { getDbFilePath } from "src/db/get-db"
-
-export default (req: Request) => {
-  unlinkSync(getDbFilePath())
 
+export default withWinterSpec({
+  methods: ["GET", "POST"],
+  auth: "none",
+})(async (req, ctx) => {
+  await ctx.db.clear()
   return new Response(JSON.stringify({}), {
     headers: {
       "content-type": "application/json",
     },
   })
-}
+})

dev-server-api/routes/api/dev_package_examples/update.ts

@@ -1,3 +1,4 @@
+import { ExportFileSchema } from "src/db/schema"
 import { publicMapExportFile } from "src/lib/public-mapping/public-map-export-file"
 import { export_file } from "src/lib/zod/export_file"
 import { withWinterSpec } from "src/with-winter-spec"
@@ -8,25 +9,21 @@ export default withWinterSpec({
   jsonBody: z.object({
     export_request_id: z.number().int(),
     file_name: z.string(),
-    file_content_base64: z.string().transform((a) => Buffer.from(a, "base64")),
+    file_content_base64: z.string(),
   }),
   jsonResponse: z.object({
-    export_file,
+    export_file: ExportFileSchema.omit({ file_content_base64: true }),
   }),
   auth: "none",
 })(async (req, ctx) => {
-  const db_export_file = await ctx.db
-    .insertInto("export_file")
-    .values({
-      export_request_id: req.jsonBody.export_request_id,
-      file_name: req.jsonBody.file_name,
-      file_content: req.jsonBody.file_content_base64,
-      created_at: new Date().toISOString(),
-    })
-    .returningAll()
-    .executeTakeFirstOrThrow()
+  const export_file = await ctx.db.put("export_file", {
+    export_request_id: req.jsonBody.export_request_id,
+    file_name: req.jsonBody.file_name,
+    file_content_base64: req.jsonBody.file_content_base64,
+    created_at: new Date().toISOString(),
+  })
 
   return ctx.json({
-    export_file: publicMapExportFile(db_export_file),
+    export_file,
   })
 })

dev-server-api/routes/api/dev_server/reset.ts

@@ -1,6 +1,7 @@
 import { publicMapExportFile } from "src/lib/public-mapping/public-map-export-file"
 import { export_file } from "src/lib/zod/export_file"
 import { withWinterSpec } from "src/with-winter-spec"
+import { NotFoundError } from "winterspec/middleware"
 import { z } from "zod"
 
 export default withWinterSpec({
@@ -10,15 +11,17 @@ export default withWinterSpec({
   }),
   auth: "none",
 })(async (req, ctx) => {
-  const db_export_file = await ctx.db
-    .selectFrom("export_file")
-    .selectAll()
-    .where("export_file_id", "=", req.query.export_file_id)
-    .executeTakeFirstOrThrow()
+  const export_file = await ctx.db.get("export_file", req.query.export_file_id)
 
-  return new Response(db_export_file.file_content, {
+  if (!export_file) {
+    throw new NotFoundError("Export file not found")
+  }
+
+  const file_content = Buffer.from(export_file.file_content_base64!, "base64")
+
+  return new Response(file_content, {
     headers: {
-      "Content-Disposition": `attachment; filename="${db_export_file.file_name}"`,
+      "Content-Disposition": `attachment; filename="${export_file.file_name}"`,
     },
   })
 })