ISSUE #5412 migration script working in principle

Author: carmenfan

backend/src/scripts/migrations/5.16/index.js

@@ -0,0 +1,24 @@
+/**
+ *  Copyright (C) 2023 3D Repo Ltd
+ *
+ *  This program is free software: you can redistribute it and/or modify
+ *  it under the terms of the GNU Affero General Public License as
+ *  published by the Free Software Foundation, either version 3 of the
+ *  License, or (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU Affero General Public License for more details.
+ *
+ *  You should have received a copy of the GNU Affero General Public License
+ *  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+const migrateUsers = require('./migrateUsersToFrontegg');
+
+const scripts = [
+	{ script: migrateUsers, desc: 'migrate users and teamspaces to Frontegg' },
+];
+
+module.exports = scripts;

backend/src/scripts/migrations/5.16/migrateUsersToFrontegg.js

@@ -0,0 +1,85 @@
+/**
+ *  Copyright (C) 2023 3D Repo Ltd
+ *
+ *  This program is free software: you can redistribute it and/or modify
+ *  it under the terms of the GNU Affero General Public License as
+ *  published by the Free Software Foundation, either version 3 of the
+ *  License, or (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU Affero General Public License for more details.
+ *
+ *  You should have received a copy of the GNU Affero General Public License
+ *  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+const { v5Path } = require('../../../interop');
+
+const { getTeamspaceList } = require('../../utils');
+
+const { getTeamspaceRefId, setTeamspaceRefId, getAllUsersInTeamspace } = require(`${v5Path}/models/teamspaceSettings`);
+const { addUserToAccount, createAccount, createUser, getAllUsersInAccount } = require(`${v5Path}/services/sso/frontegg`);
+
+const { logger } = require(`${v5Path}/utils/logger`);
+const { updateUserId } = require(`${v5Path}/models/users`);
+
+const processTeamspace = async (ts) => {
+	let refId = await getTeamspaceRefId(ts);
+	const isNew = !refId;
+	if (isNew) {
+		logger.logInfo('\t creating Frontegg account...');
+		refId = await createAccount(ts);
+		await setTeamspaceRefId(ts, refId);
+	}
+
+	const projection = {
+		user: 1,
+		'customData.email': 1,
+		'customData.userId': 1,
+		'customData.firstName': 1,
+		'customData.lastName': 1,
+	};
+
+	const [membersKnown, membersInTs] = await Promise.all([
+		isNew ? Promise.resolve([]) : getAllUsersInAccount(refId),
+		getAllUsersInTeamspace(ts, projection),
+	]);
+
+	const emailToUserId = {};
+
+	membersKnown.forEach(({ id, email }) => {
+		emailToUserId[email] = id;
+	});
+
+	await Promise.all(membersInTs.map(async ({ user, customData: { email, userId, firstName, lastName } }) => {
+		if (emailToUserId[email]) {
+			// This user is a registered user of the frontegg account
+
+			if (emailToUserId[email] !== userId) {
+				throw new Error(`User ID mismatched for ${user}. Expected ${userId}, found ${emailToUserId}`);
+			}
+		} else if (!userId) {
+			// user does not exist in frontegg, we need to create an entry
+			logger.logInfo(`\tCreating ${user}...`);
+			const newUserId = await createUser(refId, email, [firstName, lastName].join(' '));
+			console.log(newUserId);
+			if (newUserId) await updateUserId(user, newUserId);
+		} else {
+			logger.logInfo(`\tAdding ${user} to account...`);
+			await addUserToAccount(refId, userId, false);
+		}
+	}));
+};
+
+const run = async () => {
+	const teamspaces = await getTeamspaceList();
+	for (const teamspace of teamspaces) {
+		logger.logInfo(`-${teamspace}`);
+		// eslint-disable-next-line no-await-in-loop
+		await processTeamspace(teamspace);
+	}
+};
+
+module.exports = run;

backend/src/v5/handler/db.js

@@ -320,9 +320,9 @@ DBHandler.storeFileInGridFS = async (database, collection, filename, buffer) =>
 	});
 };
 
-DBHandler.createIndex = async (database, colName, indexDef, { runInBackground: background } = {}) => {
+DBHandler.createIndex = async (database, colName, indexDef, { runInBackground: background, unique } = {}) => {
 	const collection = await getCollection(database, colName);
-	const options = deleteIfUndefined({ background });
+	const options = deleteIfUndefined({ background, unique });
 	await collection.createIndex(indexDef, options);
 };
 

backend/src/v5/models/teamspaceSettings.js

@@ -219,6 +219,10 @@ TeamspaceSetting.createTeamspaceSettings = async (teamspace, teamspaceId) => {
 	await db.insertOne(teamspace, TEAMSPACE_SETTINGS_COL, settings);
 };
 
+TeamspaceSetting.setTeamspaceRefId = async (teamspace, refId) => {
+	await teamspaceSettingUpdate(teamspace, { _id: teamspace }, { $set: { refId } });
+};
+
 TeamspaceSetting.getTeamspaceRefId = async (teamspace) => {
 	const { refId } = await teamspaceSettingQuery(teamspace, { _id: teamspace }, { refId: 1 });
 	return refId;
@@ -232,11 +236,11 @@ const grantPermissionToUser = async (teamspace, username, permission) => {
 TeamspaceSetting.grantAdminToUser = (teamspace, username) => grantPermissionToUser(teamspace,
 	username, TEAMSPACE_ADMIN);
 
-TeamspaceSetting.getAllUsersInTeamspace = async (teamspace) => {
+TeamspaceSetting.getAllUsersInTeamspace = async (teamspace, projection) => {
 	const query = { 'roles.db': teamspace, 'roles.role': TEAM_MEMBER };
-	const users = await findMany(query, { user: 1 });
+	const users = await findMany(query, projection ?? { user: 1 });
 
-	return users.map(({ user }) => user);
+	return projection ? users : users.map(({ user }) => user);
 };
 
 TeamspaceSetting.removeUserFromAdminPrivilege = async (teamspace, user) => {

backend/src/v5/models/users.js

@@ -161,4 +161,8 @@ User.addUser = async (newUserData) => {
 User.removeUser = (user) => db.deleteOne(USERS_DB_NAME, USERS_COL, { user });
 User.removeUsers = (users) => db.deleteMany(USERS_DB_NAME, USERS_COL, { user: { $in: users } });
 
+User.ensureIndicesExist = async () => {
+	await db.createIndex(USERS_DB_NAME, USERS_COL, { 'customData.userId': 1 }, { runInBackground: true, unique: true });
+};
+
 module.exports = User;

backend/src/v5/services/sessions.js

@@ -23,10 +23,12 @@ const { events } = require('./eventsManager/eventsManager.constants');
 const expressSession = require('express-session');
 const { generateUUID } = require('../utils/helper/uuids');
 const { publish } = require('./eventsManager/eventsManager');
+const { ensureIndicesExist } = require('../models/users');
 
 const Sessions = { };
 const initialiseSession = async () => {
 	const store = await db.getSessionStore(expressSession);
+	await ensureIndicesExist();
 	const secure = config.public_protocol === 'https';
 	const { secret, maxAge, domain } = config.cookie;
 

backend/src/v5/services/sso/frontegg/index.js

@@ -19,6 +19,7 @@ const { HEADER_TENANT_ID, META_LABEL_TEAMSPACE } = require('./frontegg.constants
 const { get, delete: httpDelete, post } = require('../../../utils/webRequests');
 const { IdentityClient } = require('@frontegg/client');
 const Yup = require('yup');
+const { deleteIfUndefined } = require('../../../utils/helper/objects');
 const { generateUUIDString } = require('../../../utils/helper/uuids');
 const { logger } = require('../../../utils/logger');
 const { sso: { frontegg } } = require('../../../utils/config');
@@ -123,6 +124,27 @@ Frontegg.getUserById = async (userId) => {
 	}
 };
 
+Frontegg.createUser = async (accountId, email, name, userData, privateUserData, bypassVerification = false) => {
+	try {
+		const payload = deleteIfUndefined({
+			email,
+			name,
+			tenantId: accountId,
+			metadata: userData,
+			vendorMetadata: privateUserData,
+			roleIds: ['APP_USER'],
+
+		});
+
+		// using the migration endpoint will automatically activate the user
+		const url = bypassVerification ? `${config.vendorDomain}/identity/resources/migrations/v1/local` : `${config.vendorDomain}/identity/resources/vendor-only/users/v1`;
+		const { data } = await post(url, payload, { headers: await getBearerHeader() });
+		return data.id;
+	} catch (err) {
+		throw new Error(`Failed to create user(${email}) on Frontegg: ${err.message}`);
+	}
+};
+
 Frontegg.getTeamspaceByAccount = async (accountId) => {
 	try {
 		const { data: { metadata } } = await get(`${config.vendorDomain}/tenants/resources/tenants/v2/${accountId}`, await getBearerHeader());
@@ -159,12 +181,48 @@ Frontegg.createAccount = async (name) => {
 	}
 };
 
-Frontegg.addUserToAccount = async (accountId, userId) => {
+Frontegg.getAllUsersInAccount = async (accountId) => {
+	try {
+		const header = {
+			...await getBearerHeader(),
+			'frontegg-tenant-id': accountId,
+
+		};
+
+		const initialQuery = {
+			_limit: 200,
+			_offset: 0,
+			_sortBy: 'email',
+			_order: 'ASC',
+		};
+
+		let query = new URLSearchParams(initialQuery).toString();
+		const entries = [];
+
+		while (query?.length) {
+			// eslint-disable-next-line no-await-in-loop
+			const { data: { items, _links } } = await get(`${config.vendorDomain}/identity/resources/users/v3?${query}`,
+				header);
+
+			items.forEach(({ id, email }) => {
+				entries.push({ id, email });
+			});
+
+			query = _links.next;
+		}
+
+		return entries;
+	} catch (err) {
+		throw new Error(`Failed to get users from account(${accountId}) from Frontegg: ${err.message}`);
+	}
+};
+
+Frontegg.addUserToAccount = async (accountId, userId, sendInvite = true) => {
 	try {
 		const payload = {
 			tenantId: accountId,
 			validateTenantExist: true,
-			skipInviteEmail: false,
+			skipInviteEmail: !sendInvite,
 		};
 		await post(`${config.vendorDomain}/identity/resources/users/v1/${userId}/tenant`, payload, { headers: await getBearerHeader() });
 	} catch (err) {