Merge pull request #119 from powersync-ja/mongodb-types

[MongoDB] Support Decimal type

Author: rkistner

.github/workflows/test.yml

@@ -152,7 +152,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        mysql-version: [ 8.0, 8.4 ]
+        mysql-version: [8.0, 8.4]
 
     steps:
       - uses: actions/checkout@v4
@@ -208,3 +208,56 @@ jobs:
 
       - name: Test
         run: pnpm test --filter='./modules/module-mysql'
+
+  run-mongodb-tests:
+    name: MongoDB Test
+    runs-on: ubuntu-latest
+    needs: run-core-tests
+
+    strategy:
+      fail-fast: false
+      matrix:
+        mongodb-version: ['6.0', '7.0', '8.0']
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Start MongoDB
+        uses: supercharge/[email protected]
+        with:
+          mongodb-version: ${{ matrix.mongodb-version }}
+          mongodb-replica-set: test-rs
+
+      - name: Setup Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version-file: '.nvmrc'
+
+      - uses: pnpm/action-setup@v4
+        name: Install pnpm
+        with:
+          version: 9
+          run_install: false
+
+      - name: Get pnpm store directory
+        shell: bash
+        run: |
+          echo "STORE_PATH=$(pnpm store path --silent)" >> $GITHUB_ENV
+
+      - uses: actions/cache@v3
+        name: Setup pnpm cache
+        with:
+          path: ${{ env.STORE_PATH }}
+          key: ${{ runner.os }}-pnpm-store-${{ hashFiles('**/pnpm-lock.yaml') }}
+          restore-keys: |
+            ${{ runner.os }}-pnpm-store-
+
+      - name: Install dependencies
+        run: pnpm install
+
+      - name: Build
+        shell: bash
+        run: pnpm build
+
+      - name: Test
+        run: pnpm test --filter='./modules/module-mongodb'

modules/module-mongodb/package.json

@@ -13,7 +13,7 @@
     "build": "tsc -b",
     "build:tests": "tsc -b test/tsconfig.json",
     "clean": "rm -rf ./dist && tsc -b --clean",
-    "test": "vitest --no-threads"
+    "test": "vitest"
   },
   "exports": {
     ".": {

modules/module-mongodb/src/api/MongoRouteAPIAdapter.ts

@@ -313,6 +313,14 @@ export class MongoRouteAPIAdapter implements api.RouteAPI {
       return 'Binary';
     } else if (data instanceof mongo.Long) {
       return 'Long';
+    } else if (data instanceof RegExp) {
+      return 'RegExp';
+    } else if (data instanceof mongo.MinKey) {
+      return 'MinKey';
+    } else if (data instanceof mongo.MaxKey) {
+      return 'MaxKey';
+    } else if (data instanceof mongo.Decimal128) {
+      return 'Decimal';
     } else if (Array.isArray(data)) {
       return 'Array';
     } else if (data instanceof Uint8Array) {

modules/module-mongodb/src/replication/MongoRelation.ts

@@ -62,6 +62,12 @@ export function toMongoSyncRulesValue(data: any): SqliteValue {
     return new Uint8Array(data.buffer);
   } else if (data instanceof mongo.Long) {
     return data.toBigInt();
+  } else if (data instanceof mongo.Decimal128) {
+    return data.toString();
+  } else if (data instanceof mongo.MinKey || data instanceof mongo.MaxKey) {
+    return null;
+  } else if (data instanceof RegExp) {
+    return JSON.stringify({ pattern: data.source, options: data.flags });
   } else if (Array.isArray(data)) {
     // We may be able to avoid some parse + stringify cycles here for JsonSqliteContainer.
     return JSONBig.stringify(data.map((element) => filterJsonData(element)));
@@ -112,6 +118,12 @@ function filterJsonData(data: any, depth = 0): any {
     return undefined;
   } else if (data instanceof mongo.Long) {
     return data.toBigInt();
+  } else if (data instanceof mongo.Decimal128) {
+    return data.toString();
+  } else if (data instanceof mongo.MinKey || data instanceof mongo.MaxKey) {
+    return data._bsontype;
+  } else if (data instanceof mongo.BSONRegExp) {
+    return JSON.stringify({ pattern: data.pattern, options: data.options });
   } else if (Array.isArray(data)) {
     return data.map((element) => filterJsonData(element, depth + 1));
   } else if (ArrayBuffer.isView(data)) {

modules/module-mongodb/test/src/mongo_test.test.ts

@@ -23,14 +23,18 @@ describe('mongo data types', () => {
         int2: 1000,
         int4: 1000000,
         int8: 9007199254740993n,
-        float: 3.14
+        float: 3.14,
+        decimal: new mongo.Decimal128('3.14')
       },
       { _id: 2 as any, nested: { test: 'thing' } },
       { _id: 3 as any, date: new Date('2023-03-06 15:47+02') },
       {
         _id: 4 as any,
         timestamp: mongo.Timestamp.fromBits(123, 456),
-        objectId: mongo.ObjectId.createFromHexString('66e834cc91d805df11fa0ecb')
+        objectId: mongo.ObjectId.createFromHexString('66e834cc91d805df11fa0ecb'),
+        regexp: new mongo.BSONRegExp('test', 'i'),
+        minKey: new mongo.MinKey(),
+        maxKey: new mongo.MaxKey()
       }
     ]);
   }
@@ -47,14 +51,18 @@ describe('mongo data types', () => {
         int2: [1000],
         int4: [1000000],
         int8: [9007199254740993n],
-        float: [3.14]
+        float: [3.14],
+        decimal: [new mongo.Decimal128('3.14')]
       },
       { _id: 2 as any, nested: [{ test: 'thing' }] },
       { _id: 3 as any, date: [new Date('2023-03-06 15:47+02')] },
       {
         _id: 10 as any,
         timestamp: [mongo.Timestamp.fromBits(123, 456)],
-        objectId: [mongo.ObjectId.createFromHexString('66e834cc91d805df11fa0ecb')]
+        objectId: [mongo.ObjectId.createFromHexString('66e834cc91d805df11fa0ecb')],
+        regexp: [new mongo.BSONRegExp('test', 'i')],
+        minKey: [new mongo.MinKey()],
+        maxKey: [new mongo.MaxKey()]
       }
     ]);
   }
@@ -70,7 +78,8 @@ describe('mongo data types', () => {
       int4: 1000000n,
       int8: 9007199254740993n,
       float: 3.14,
-      null: null
+      null: null,
+      decimal: '3.14'
     });
     expect(transformed[1]).toMatchObject({
       _id: 2n,
@@ -85,7 +94,10 @@ describe('mongo data types', () => {
     expect(transformed[3]).toMatchObject({
       _id: 4n,
       objectId: '66e834cc91d805df11fa0ecb',
-      timestamp: 1958505087099n
+      timestamp: 1958505087099n,
+      regexp: '{"pattern":"test","options":"i"}',
+      minKey: null,
+      maxKey: null
     });
   }
 
@@ -220,21 +232,25 @@ describe('mongo data types', () => {
       const schema = await adapter.getConnectionSchema();
       const dbSchema = schema.filter((s) => s.name == TEST_CONNECTION_OPTIONS.database)[0];
       expect(dbSchema).not.toBeNull();
-      expect(dbSchema.tables).toEqual([
+      expect(dbSchema.tables).toMatchObject([
         {
           name: 'test_data',
           columns: [
             { name: '_id', sqlite_type: 4, internal_type: 'Integer' },
             { name: 'bool', sqlite_type: 4, internal_type: 'Boolean' },
             { name: 'bytea', sqlite_type: 1, internal_type: 'Binary' },
             { name: 'date', sqlite_type: 2, internal_type: 'Date' },
+            { name: 'decimal', sqlite_type: 2, internal_type: 'Decimal' },
             { name: 'float', sqlite_type: 8, internal_type: 'Double' },
             { name: 'int2', sqlite_type: 4, internal_type: 'Integer' },
             { name: 'int4', sqlite_type: 4, internal_type: 'Integer' },
             { name: 'int8', sqlite_type: 4, internal_type: 'Long' },
+            { name: 'maxKey', sqlite_type: 0, internal_type: 'MaxKey' },
+            { name: 'minKey', sqlite_type: 0, internal_type: 'MinKey' },
             { name: 'nested', sqlite_type: 2, internal_type: 'Object' },
             { name: 'null', sqlite_type: 0, internal_type: 'Null' },
             { name: 'objectId', sqlite_type: 2, internal_type: 'ObjectId' },
+            { name: 'regexp', sqlite_type: 2, internal_type: 'RegExp' },
             { name: 'text', sqlite_type: 2, internal_type: 'String' },
             { name: 'timestamp', sqlite_type: 4, internal_type: 'Timestamp' },
             { name: 'uuid', sqlite_type: 2, internal_type: 'UUID' }

modules/module-mongodb/vitest.config.ts

@@ -4,6 +4,12 @@ import tsconfigPaths from 'vite-tsconfig-paths';
 export default defineConfig({
   plugins: [tsconfigPaths()],
   test: {
-    setupFiles: './test/src/setup.ts'
+    setupFiles: './test/src/setup.ts',
+    poolOptions: {
+      threads: {
+        singleThread: true
+      }
+    },
+    pool: 'threads'
   }
 });