Merge pull request #78 from obeliss-nlesc/feature/use_sqlitedb

Feature/use sqlitedb

.github/workflows/unit-tests.yml

@@ -8,4 +8,5 @@ jobs:
       - uses: actions/setup-node@v4
         with:
           node-version: "20"
+      - run: npm install
       - run: npm run test

README.md

@@ -47,8 +47,19 @@ POSTGRES_PASSWORD=somepassword
 OTREE_REST_KEY=somepassword
 SECRET_KEY="some secret in quotes"
 API_KEY="some other secret in quotes"
+URL_PASS="some SHA256 hashed secret in quotes"
 ```
 
+- `POSTGRES_IPS`: The POSTGRES server IPs comma separated.
+- `OTREE_IPS`: The OTREE server IPs comma separated.
+- `POSTGRES_USER`: The POSTGRES database username.
+- `POSTGRES_DB`: The POSTGRES database name used by oTree.
+- `POSTGRES_PASSWORD`: The POSTGRES user password.
+- `OTREE_REST_KEY`: The oTree REST KEY to access the oTree API.
+- `SECRET_KEY`: A secret key used to generate a waiting room URL signiture token.
+- `API_KEY`: An API key to access the admin REST API for the waiting room.
+- `URL_PASS`: A password used to access the waiting room info/helper pages. This password is created using the helper script `create-url-password-hash.js`
+
 ## Setup experiments config.json file
 
 The Server needs to know some details of the oTree experiments being hosted on the servers.
@@ -92,7 +103,7 @@ Group by Arrival Time scheduler i.e. it will group people on a first come first
 
 ## File database
 
-By default the waiting room will save user details in a json file database `data/userdb.json`.
+By default the waiting room will save user details in a sqlite file database `data/userdb.sqlite`.
 A different file can be passed as a parameter using `--db-file` option when starting the server.
 The database stores the used urls i.e. urls that have alreaddy been used by a user.
 On restarting the server waiting room, these urls are removed from the list of available urls so that
@@ -112,7 +123,13 @@ start the server with --reset-db flag.
 
 ## Generate test URLs
 
-Use the encode-url.js helper script to generate test urls.
+Use the helper URL path to generate tests URLS:
+
+```
+http://localhost:8060/urls/[EXPIMENT_NAME]/[NO_OF_URLS]?secret=[SECRET_USED_IN_URL_PASS]
+```
+
+Or use the encode-url.js helper script to generate test urls.
 
 ```shell
 node encode-url.js -n 5 -h localhost:8080
@@ -150,6 +167,18 @@ To create session URLs for an experiment use the helper command `sessions.js`
 node sessions.js create [NAME] --num [NUM OF PARTICIPANTS]
 ```
 
+## Simple info dashboard
+
+The server comes with a simple info page showing the number of available URLs, number of users and user ids per session.
+The info page is accessed through the following path:
+
+```
+http://localhost:8060/info/[EXPERIMENT_NAME]?secret=[SECRET_USED_IN_URL_PASS]
+```
+
+E.g. for Experiment `DropOutTest`
+![DropOutTest](./dropouttest_info.png)
+
 ## Starting the server using PM2
 
 To manage the server with pm2, first install pm2

UserDb.js → UserJsonDb.js

@@ -3,16 +3,17 @@
 const murmurhash = require("murmurhash")
 const fs = require("fs").promises
 const User = require("./user.js")
-const { SHA1 } = require("crypto-js")
+const UserMap = require("./UserMap.js")
 
-class UserDb extends Map {
+class UserJsonDb extends UserMap {
   constructor(file) {
     super()
     this.seed = 42
     this.file = file
     this.lastHash = 0
     this.writeCounter = 0
   }
+
   load() {
     let data = []
     return fs
@@ -29,6 +30,7 @@ class UserDb extends Map {
             user.experimentUrl = u.experimentUrl
             user.groupId = u.groupId
             user.oTreeId = u.oTreeId
+            user.server = u.server
             user.tokenParams = u.tokenParams
             user.state = u.redirectedUrl ? u.state : user.state
             const compoundKey = `${user.userId}:${user.experimentId}`
@@ -41,28 +43,12 @@ class UserDb extends Map {
         )
       })
   }
-  find(userId) {
-    return Array.from(this.values()).filter((u) => {
-      return u.userId == userId
-    })
-  }
-  getUsedUrls() {
-    return Array.from(this.values())
-      .filter((u) => {
-        return u.experimentUrl
-      })
-      .map((u) => {
-        return u.experimentUrl
-      })
-  }
-  dump() {
-    const data = []
-    this.forEach((v, k) => {
-      data.push(v.serialize())
-    })
 
-    return JSON.stringify(data)
+  upsert() {
+    // Implement interface method
+    return this.save()
   }
+
   save() {
     this.writeCounter += 1
     const currentCounter = this.writeCounter
@@ -73,15 +59,20 @@ class UserDb extends Map {
       this.#save()
     }, 500)
   }
+
+  saveAll() {
+    return this.save()
+  }
+
   forceSave() {
     const data = []
     this.forEach((v, k) => {
       data.push(v.serialize())
     })
     const dump = JSON.stringify(data, null, 2)
-
     return fs.writeFile(this.file, dump)
   }
+
   #save() {
     const data = []
     this.forEach((v, k) => {
@@ -104,4 +95,4 @@ class UserDb extends Map {
   }
 }
 
-module.exports = UserDb
+module.exports = UserJsonDb

UserMap.js

@@ -0,0 +1,68 @@
+const User = require("./user.js")
+
+class UserMap extends Map {
+  constructor(file) {
+    super()
+  }
+
+  load() {
+    throw new Error("Method not implemented.")
+  }
+
+  find(userId) {
+    return Array.from(this.values()).filter((u) => {
+      return u.userId == userId
+    })
+  }
+
+  getUsersInSession(session) {
+    return Array.from(this.values()).filter((u) => {
+      if (!u.server) {
+        return false
+      }
+      const userSession = u.server.split("#")[1]
+      return userSession == session
+    })
+  }
+
+  getUsedUrls() {
+    return Array.from(this.values())
+      .filter((u) => {
+        return u.experimentUrl
+      })
+      .map((u) => {
+        return u.experimentUrl
+      })
+  }
+
+  dump() {
+    const data = []
+    this.forEach((v, k) => {
+      data.push(v.serialize())
+    })
+
+    return JSON.stringify(data)
+  }
+
+  upsert(user) {
+    throw new Error("Method not implemented.")
+  }
+
+  save(user) {
+    throw new Error("Method not implemented.")
+  }
+
+  saveAll() {
+    throw new Error("Method not implemented.")
+  }
+
+  findAll() {
+    throw new Error("Method not implemented.")
+  }
+
+  forceSave() {
+    throw new Error("Method not implemented.")
+  }
+}
+
+module.exports = UserMap

UserSqliteDb.js

@@ -0,0 +1,143 @@
+const sqlite3 = require("sqlite3").verbose()
+const User = require("./user.js")
+const UserMap = require("./UserMap.js")
+
+class UserSqliteDb extends UserMap {
+  constructor(file) {
+    super()
+    this.writeCounter = 0
+    this.db = new sqlite3.Database(file)
+    this.db.serialize(() => {
+      this.db.run(`
+        CREATE TABLE IF NOT EXISTS users (
+          userId TEXT PRIMARY KEY,
+          jsonObj TEXT
+        )
+      `)
+    })
+  }
+
+  _getTableNames() {
+    return new Promise((resolve, reject) => {
+      this.db.all(
+        "SELECT name FROM sqlite_master WHERE type='table'",
+        [],
+        (err, rows) => {
+          if (err) {
+            reject(err)
+          } else {
+            const tableNames = rows.map((row) => row.name)
+            resolve(tableNames)
+          }
+        },
+      )
+    }) // Promise
+  }
+
+  load() {
+    return new Promise((resolve, reject) => {
+      const query = `SELECT * FROM users`
+      this.db.all(query, (err, rows) => {
+        if (err) {
+          console.error(err)
+          reject(err)
+        } else {
+          rows
+            .map((r) => {
+              return JSON.parse(r.jsonObj)
+            })
+            .forEach((u) => {
+              const user = new User(u.userId, u.experimentId)
+              user.redirectedUrl = u.redirectedUrl
+              user.experimentUrl = u.experimentUrl
+              user.groupId = u.groupId
+              user.server = u.server
+              user.oTreeId = u.oTreeId
+              user.tokenParams = u.tokenParams
+              user.state = u.redirectedUrl ? u.state : user.state
+              const compoundKey = `${user.userId}:${user.experimentId}`
+              this.set(compoundKey, user)
+            })
+          resolve(this)
+        }
+      })
+    }) //Promise
+  }
+
+  findAll() {
+    return new Promise((resolve, reject) => {
+      const query = `SELECT * FROM users`
+      this.db.all(query, (err, rows) => {
+        if (err) {
+          reject(err)
+        } else {
+          resolve(rows)
+        }
+      })
+    }) // Promise
+  }
+
+  upsert(user) {
+    return new Promise((resolve, reject) => {
+      const query = `
+        INSERT INTO users (userId, jsonObj) 
+        VALUES (?, ?)
+        ON CONFLICT(userId) DO UPDATE SET 
+          jsonObj = excluded.jsonObj
+      `
+      const compoundKey = `${user.userId}:${user.experimentId}`
+      this.db.run(
+        query,
+        [compoundKey, JSON.stringify(user.serialize())],
+        function (err) {
+          if (err) {
+            reject(err)
+          } else {
+            // console.log(`${compoundKey} upsert`)
+            resolve(compoundKey)
+          }
+        },
+      )
+    })
+  }
+
+  save(user) {
+    return new Promise((resolve, reject) => {
+      const query = `INSERT INTO users (userId, jsonObj) VALUES (?, ?)`
+      const compoundKey = `${user.userId}:${user.experimentId}`
+      this.db.run(
+        query,
+        [compoundKey, JSON.stringify(user.serialize())],
+        function (err) {
+          if (err) {
+            reject(err)
+          } else {
+            resolve(compoundKey)
+          }
+        },
+      )
+    }) // Promise
+  }
+
+  saveAll() {
+    this.writeCounter += 1
+    const currentCounter = this.writeCounter
+    setTimeout(() => {
+      if (currentCounter != this.writeCounter || this.writeCounter === 0) {
+        return
+      }
+      this.forEach((u, k) => {
+        this.upsert(u)
+      })
+    }, 500)
+  }
+
+  forceSave() {
+    const allUpserts = [...this.values()].map((u) => {
+      return this.upsert(u)
+    })
+    return Promise.all(allUpserts)
+  }
+}
+
+module.exports = UserSqliteDb

config.json

@@ -24,7 +24,7 @@
     {
       "name": "DropOutTest",
       "enabled": true,
-      "agreementTimeout": 10000,
+      "agreementTimeout": 100,
       "scheduler": {
         "type": "GatScheduler",
         "params": {