redis scaling

Signed-off-by: Hoang Pham <[email protected]>

Author: hweihwang

.env.example

@@ -21,7 +21,24 @@ TLS_CERT=
 # Turn off SSL certificate validation in development mode for easier testing
 IS_DEV=false
 
+# Storage strategy for whiteboard data and socket-related temporary data
+# Valid values are: 'redis' or 'lru' (Least Recently Used cache)
+# This strategy is used for:
+#  1. Whiteboard data storage
+#  2. Socket-related temporary data (e.g., cached tokens, bound data for each socket ID)
+#  3. Scaling the socket server across multiple nodes (when using 'redis')
+# We strongly recommend using 'redis' for production environments
+# 'lru' provides a balance of performance and memory usage for single-node setups
+STORAGE_STRATEGY=lru
+
+# Redis connection URL for data storage and socket server scaling
+# Required when STORAGE_STRATEGY is set to 'redis'
+# This URL is used for both persistent data and temporary socket-related data
+# Format: redis://[username:password@]host[:port][/database_number]
+# Example: redis://user:[email protected]:6379/0
+REDIS_URL=redis://localhost:6379
+
 # Prometheus metrics endpoint
 # Set this to access the monitoring endpoint at /metrics
 # either providing it as Bearer token or as ?token= query parameter
-# METRICS_TOKEN=
+# METRICS_TOKEN=

package-lock.json

@@ -13,8 +13,8 @@
     "lint:fix": "eslint --ext .js,.ts,.tsx,.vue src websocket_server --fix",
     "stylelint": "stylelint 'src/**/*.{css,scss,sass}'",
     "stylelint:fix": "stylelint 'src/**/*.{css,scss,sass}' --fix",
-    "server:start": "node websocket_server/server.js",
-    "server:watch": "nodemon websocket_server/server.js"
+    "server:start": "node websocket_server/main.js",
+    "server:watch": "nodemon websocket_server/main.js"
   },
   "dependencies": {
     "@excalidraw/excalidraw": "^0.17.6",
@@ -27,6 +27,7 @@
     "@nextcloud/l10n": "^3.1.0",
     "@nextcloud/router": "^3.0.1",
     "@nextcloud/vue": "^8.16.0",
+    "@socket.io/redis-streams-adapter": "^0.2.2",
     "dotenv": "^16.4.5",
     "express": "^4.19.2",
     "jsonwebtoken": "^9.0.2",
@@ -78,6 +79,6 @@
   },
   "engines": {
     "node": "^20",
-    "npm": "^9"
+    "npm": "^10"
   }
 }

package.json

@@ -8,16 +8,16 @@
 import fetch from 'node-fetch'
 import https from 'https'
 import dotenv from 'dotenv'
-import Utils from './utils.js'
+import Utils from './Utils.js'
 dotenv.config()
 
-class ApiService {
+export default class ApiService {
 
-	constructor(authManager) {
+	constructor(tokenGenerator) {
 		this.NEXTCLOUD_URL = process.env.NEXTCLOUD_URL
 		this.IS_DEV = Utils.parseBooleanFromEnv(process.env.IS_DEV)
 		this.agent = this.IS_DEV ? new https.Agent({ rejectUnauthorized: false }) : null
-		this.authManager = authManager
+		this.tokenGenerator = tokenGenerator
 	}
 
 	fetchOptions(method, token, body = null, roomId = null, lastEditedUser = null) {
@@ -27,7 +27,7 @@ class ApiService {
 				'Content-Type': 'application/json',
 				...(method === 'GET' && { Authorization: `Bearer ${token}` }),
 				...(method === 'PUT' && {
-					'X-Whiteboard-Auth': this.authManager.generateSharedToken(roomId),
+					'X-Whiteboard-Auth': this.tokenGenerator.handle(roomId),
 					'X-Whiteboard-User': lastEditedUser || 'unknown',
 				}),
 			},
@@ -65,5 +65,3 @@ class ApiService {
 	}
 
 }
-
-export default ApiService

websocket_server/apiService.js renamed to websocket_server/ApiService.js

@@ -5,16 +5,16 @@
 
 import dotenv from 'dotenv'
 import express from 'express'
-import { PrometheusMetrics } from './prom-metrics.js'
+import PrometheusDataManager from './PrometheusDataManager.js'
 
 dotenv.config()
 
-class AppManager {
+export default class AppManager {
 
 	constructor(storageManager) {
 		this.app = express()
 		this.storageManager = storageManager
-		this.metricsManager = new PrometheusMetrics(storageManager)
+		this.metricsManager = new PrometheusDataManager(storageManager)
 		this.METRICS_TOKEN = process.env.METRICS_TOKEN
 		this.setupRoutes()
 	}
@@ -44,5 +44,3 @@ class AppManager {
 	}
 
 }
-
-export default AppManager

websocket_server/app.js renamed to websocket_server/AppManager.js

@@ -0,0 +1,57 @@
+/* eslint-disable no-console */
+
+/**
+ * SPDX-FileCopyrightText: 2024 Nextcloud GmbH and Nextcloud contributors
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+
+import StorageStrategy from './StorageStrategy.js'
+import { LRUCache } from 'lru-cache'
+
+export default class LRUCacheStrategy extends StorageStrategy {
+
+	constructor(apiService) {
+		super()
+		this.apiService = apiService
+		this.cache = new LRUCache({
+			max: 1000,
+			ttl: 30 * 60 * 1000,
+			ttlAutopurge: true,
+			dispose: async (value, key) => {
+				console.log('Disposing room', key)
+				if (value?.data && value?.lastEditedUser) {
+					try {
+						await this.apiService.saveRoomDataToServer(
+							key,
+							value.data,
+							value.lastEditedUser,
+						)
+					} catch (error) {
+						console.error(`Failed to save room ${key} data:`, error)
+					}
+				}
+			},
+		})
+	}
+
+	async get(key) {
+		return this.cache.get(key)
+	}
+
+	async set(key, value) {
+		this.cache.set(key, value)
+	}
+
+	async delete(key) {
+		this.cache.delete(key)
+	}
+
+	async clear() {
+		this.cache.clear()
+	}
+
+	getRooms() {
+		return this.cache
+	}
+
+}

websocket_server/LRUCacheStrategy.js

@@ -4,9 +4,9 @@
  */
 
 import { register, Gauge } from 'prom-client'
-import { SystemMonitor } from './monitoring.js'
+import SystemMonitor from './SystemMonitor.js'
 
-export class PrometheusMetrics {
+export default class PrometheusDataManager {
 
 	constructor(storageManager) {
 		this.systemMonitor = new SystemMonitor(storageManager)

websocket_server/prom-metrics.js renamed to websocket_server/PrometheusDataManager.js

@@ -0,0 +1,119 @@
+/* eslint-disable no-console */
+
+/**
+ * SPDX-FileCopyrightText: 2024 Nextcloud GmbH and Nextcloud contributors
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+
+import StorageStrategy from './StorageStrategy.js'
+import { createClient } from 'redis'
+import Room from './Room.js'
+
+export default class RedisStrategy extends StorageStrategy {
+
+	constructor(apiService) {
+		super()
+		this.apiService = apiService
+		this.client = createClient({
+			url: process.env.REDIS_URL || 'redis://localhost:6379',
+			retry_strategy: (options) => {
+				if (options.error && options.error.code === 'ECONNREFUSED') {
+					return new Error('The server refused the connection')
+				}
+				if (options.total_retry_time > 1000 * 60 * 60) {
+					return new Error('Retry time exhausted')
+				}
+				if (options.attempt > 10) {
+					return undefined
+				}
+				return Math.min(options.attempt * 100, 3000)
+			},
+		})
+		this.client.on('error', (err) =>
+			console.error('Redis Client Error', err),
+		)
+		this.connect()
+	}
+
+	async connect() {
+		try {
+			await this.client.connect()
+		} catch (error) {
+			console.error('Failed to connect to Redis:', error)
+			throw error
+		}
+	}
+
+	async get(key) {
+		try {
+			const data = await this.client.get(key)
+			if (!data) return null
+			return this.deserialize(data)
+		} catch (error) {
+			console.error(`Error getting data for key ${key}:`, error)
+			return null
+		}
+	}
+
+	async set(key, value) {
+		try {
+			const serializedData = this.serialize(value)
+			await this.client.set(key, serializedData, { EX: 30 * 60 })
+		} catch (error) {
+			console.error(`Error setting data for key ${key}:`, error)
+		}
+	}
+
+	async delete(key) {
+		try {
+			const room = await this.get(key)
+			if (room?.data && room?.lastEditedUser) {
+				await this.apiService.saveRoomDataToServer(
+					key,
+					room.data,
+					room.lastEditedUser,
+				)
+			}
+			await this.client.del(key)
+		} catch (error) {
+			console.error(`Error deleting key ${key}:`, error)
+		}
+	}
+
+	async clear() {
+		try {
+			await this.client.flushDb()
+		} catch (error) {
+			console.error('Error clearing Redis database:', error)
+		}
+	}
+
+	async getRooms() {
+		try {
+			const keys = await this.client.keys('*')
+			const rooms = new Map()
+			for (const key of keys) {
+				const room = await this.get(key)
+				if (room) rooms.set(key, room)
+			}
+			return rooms
+		} catch (error) {
+			console.error('Error getting rooms:', error)
+			return new Map()
+		}
+	}
+
+	serialize(value) {
+		return value instanceof Room
+			? JSON.stringify(value.toJSON())
+			: JSON.stringify(value)
+	}
+
+	deserialize(data) {
+		const parsedData = JSON.parse(data)
+		return parsedData.id && parsedData.users
+			? Room.fromJSON(parsedData)
+			: parsedData
+	}
+
+}

websocket_server/RedisStrategy.js

@@ -0,0 +1,49 @@
+/**
+ * SPDX-FileCopyrightText: 2024 Nextcloud GmbH and Nextcloud contributors
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+
+export default class Room {
+
+	constructor(id, data = null, users = new Set(), lastEditedUser = null) {
+		this.id = id
+		this.data = data
+		this.users = new Set(users)
+		this.lastEditedUser = lastEditedUser
+	}
+
+	setUsers(users) {
+		this.users = new Set(users)
+	}
+
+	updateLastEditedUser(userId) {
+		this.lastEditedUser = userId
+	}
+
+	setData(data) {
+		this.data = data
+	}
+
+	isEmpty() {
+		return this.users.size === 0
+	}
+
+	toJSON() {
+		return {
+			id: this.id,
+			data: this.data,
+			users: Array.from(this.users),
+			lastEditedUser: this.lastEditedUser,
+		}
+	}
+
+	static fromJSON(json) {
+		return new Room(
+			json.id,
+			json.data,
+			new Set(json.users),
+			json.lastEditedUser,
+		)
+	}
+
+}