Merge pull request #5329 from 3drepo/ISSUE_5244

Issue 5244 - Custom Tickets Query Filters

Author: carmenfan

backend/src/v5/handler/db.js

@@ -190,10 +190,10 @@ DBHandler.findOne = async (database, colName, query, projection, sort) => {
 	return collection.findOne(query, options);
 };
 
-DBHandler.find = async (database, colName, query, projection, sort, limit) => {
+DBHandler.find = async (database, colName, query, projection, sort, limit, skip = 0) => {
 	const collection = await getCollection(database, colName);
 	const options = deleteIfUndefined({ projection, sort });
-	const cmd = collection.find(query, options);
+	const cmd = collection.find(query, options).skip(skip);
 	return limit ? cmd.limit(limit).toArray() : cmd.toArray();
 };
 

backend/src/v5/middleware/dataConverter/inputs/teamspaces/projects/models/commons/ticketQueryFilters.js

@@ -0,0 +1,51 @@
+/**
+ *  Copyright (C) 2024 3D Repo Ltd
+ *
+ *  This program is free software: you can redistribute it and/or modify
+ *  it under the terms of the GNU Affero General Public License as
+ *  published by the Free Software Foundation, either version 3 of the
+ *  License, or (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU Affero General Public License for more details.
+ *
+ *  You should have received a copy of the GNU Affero General Public License
+ *  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+const { createResponseCode, templates } = require('../../../../../../../utils/responseCodes');
+const { queryParamSchema, querySchema } = require('../../../../../../../schemas/tickets/tickets.filters');
+const { respond } = require('../../../../../../../utils/responder');
+
+const TicketQueryFilters = {};
+
+TicketQueryFilters.validateQueryString = async (req, res, next) => {
+	if (req.query.query) {
+		try {
+			const queryString = await querySchema.validate(decodeURIComponent(req.query.query));
+			const queryParams = queryString.slice(1, -1).split('&&');
+			const queryFilters = [];
+
+			await Promise.all(queryParams.map(async (param) => {
+				const [propertyName, operator, value] = param.split('::');
+				try {
+					const validatedQuery = await queryParamSchema.validate({ propertyName, operator, value });
+					queryFilters.push(validatedQuery);
+				} catch (err) {
+					throw createResponseCode(templates.invalidArguments, `Error at '${propertyName}' query filter: ${err.message}`);
+				}
+			}));
+
+			req.listOptions.queryFilters = queryFilters;
+		} catch (err) {
+			respond(req, res, createResponseCode(templates.invalidArguments, err.message));
+			return;
+		}
+	}
+
+	await next();
+};
+
+module.exports = TicketQueryFilters;

backend/src/v5/middleware/dataConverter/inputs/teamspaces/projects/models/commons/utils.js

@@ -35,6 +35,8 @@ Utils.validateListSortAndFilter = async (req, res, next) => {
 				otherwise: (s) => s.default(true),
 			}),
 			updatedSince: types.date,
+			limit: Yup.number().integer().min(1),
+			skip: Yup.number().integer().min(0).default(0),
 		});
 		try {
 			req.listOptions = await schema.validate(req.query, { stripUnknown: true });

backend/src/v5/middleware/dataConverter/inputs/teamspaces/projects/models/drawings/calibrations.js

@@ -63,8 +63,7 @@ const validateNewCalibrationData = async (req, res, next) => {
 			model: positionArray(YupHelper.types.position).required(),
 			drawing: positionArray(YupHelper.types.position2d).required(),
 		}).required(),
-		verticalRange: Yup.array().of(Yup.number()).length(2).required()
-			.test('valid-verticalRange', 'The second number of the range must be larger than the first', (value) => value[0] <= value[1]),
+		verticalRange: YupHelper.types.range.required(),
 		units: YupHelper.types.strings.unit.required(),
 	}).required().noUnknown();
 

backend/src/v5/models/tickets.constants.js

@@ -15,8 +15,69 @@
  *  along with this program.  If not, see <http://www.gnu.org/licenses/>.
  */
 
+const { isNumberString } = require('../utils/helper/typeCheck');
+const { queryOperators } = require('../schemas/tickets/tickets.filters');
+const { toBoolean } = require('../utils/helper/strings');
+
 const Tickets = {};
 
 Tickets.TICKETS_RESOURCES_COL = 'tickets.resources';
 
+Tickets.operatorToQuery = {
+	[queryOperators.EXISTS]: (propertyName) => ({
+		[propertyName]: { $exists: true },
+	}),
+	[queryOperators.NOT_EXISTS]: (propertyName) => ({
+		[propertyName]: { $not: { $exists: true } },
+	}),
+	[queryOperators.IS]: (propertyName, value) => ({
+		[propertyName]: { $in: value },
+	}),
+	[queryOperators.NOT_IS]: (propertyName, value) => ({
+		[propertyName]: { $not: { $in: value } },
+	}),
+	[queryOperators.EQUALS]: (propertyName, value) => ({
+		[propertyName]: { $in: value.flatMap((v) => {
+			if (isNumberString(v)) return [Number(v), new Date(Number(v))];
+			return toBoolean(v);
+		}) },
+	}),
+	[queryOperators.NOT_EQUALS]: (propertyName, value) => ({
+		[propertyName]: { $not: { $in: value.flatMap((v) => {
+			if (isNumberString(v)) return [Number(v), new Date(Number(v))];
+			return toBoolean(v);
+		}) } },
+	}),
+	[queryOperators.CONTAINS]: (propertyName, value) => ({
+		$or: value.map((val) => ({ [propertyName]: { $regex: val, $options: 'i' } })),
+	}),
+	[queryOperators.NOT_CONTAINS]: (propertyName, value) => ({
+		$nor: value.map((val) => ({ [propertyName]: { $regex: val, $options: 'i' } })),
+	}),
+	[queryOperators.RANGE]: (propertyName, value) => ({
+		$or: value.flatMap((range) => [
+			{ [propertyName]: { $gte: range[0], $lte: range[1] } },
+			{ [propertyName]: { $gte: new Date(range[0]), $lte: new Date(range[1]) } },
+		]),
+	}),
+	[queryOperators.NOT_IN_RANGE]: (propertyName, value) => ({
+		$nor: value.flatMap((range) => [
+			{ [propertyName]: { $gte: range[0], $lte: range[1] } },
+			{ [propertyName]: { $gte: new Date(range[0]), $lte: new Date(range[1]) } },
+		]),
+	}),
+	[queryOperators.GREATER_OR_EQUAL_TO]: (propertyName, value) => ({
+		$or: [
+			{ [propertyName]: { $gte: value } },
+			{ [propertyName]: { $gte: new Date(value) } },
+		],
+	}),
+	[queryOperators.LESSER_OR_EQUAL_TO]: (propertyName, value) => ({
+		$or: [
+			{ [propertyName]: { $lte: value } },
+			{ [propertyName]: { $lte: new Date(value) } },
+		],
+	}),
+};
+
 module.exports = Tickets;

backend/src/v5/models/tickets.js

@@ -27,6 +27,7 @@ const { Long } = DbHandler.dataTypes;
 
 const Tickets = {};
 const TICKETS_COL = 'tickets';
+const TEMPLATES_COL = 'templates';
 const TICKETS_COUNTER_COL = 'tickets.counters';
 
 const reserveTicketNumbers = async (teamspace, project, model, type, nToReserve) => {
@@ -161,6 +162,48 @@ Tickets.getTicketById = async (
 Tickets.getTicketsByQuery = (teamspace, project, model, query, projection) => DbHandler.find(teamspace,
 	TICKETS_COL, { teamspace, project, model, ...query }, projection);
 
+Tickets.getTicketsByFilter = (
+	teamspace,
+	project,
+	model,
+	{
+		query,
+		ticketCodeQuery,
+		projection = { teamspace: 0, project: 0, model: 0 },
+		updatedSince,
+		sort = { [`properties.${basePropertyLabels.CREATED_AT}`]: -1 },
+		limit,
+		skip = 0,
+	} = {},
+) => {
+	const formattedQuery = { teamspace, project, model, ...query };
+
+	if (updatedSince) {
+		formattedQuery[`properties.${basePropertyLabels.UPDATED_AT}`] = { $gt: updatedSince };
+	}
+
+	if (ticketCodeQuery) {
+		const pipelines = [
+			{ $match: formattedQuery },
+			{ $lookup: { from: TEMPLATES_COL, localField: 'type', foreignField: '_id', as: 'templateDetails' } },
+			{ $unwind: '$templateDetails' },
+			{ $addFields: { ticketCode: { $concat: ['$templateDetails.code', ':', { $toString: '$number' }] } } },
+			{ $match: ticketCodeQuery },
+			{ $project: projection },
+			{ $sort: sort },
+			{ $skip: skip },
+		];
+
+		if (limit) {
+			pipelines.push({ $limit: limit });
+		}
+
+		return DbHandler.aggregate(teamspace, TICKETS_COL, pipelines);
+	}
+
+	return DbHandler.find(teamspace, TICKETS_COL, formattedQuery, projection, sort, limit, skip);
+};
+
 Tickets.getAllTickets = (
 	teamspace,
 	project,
@@ -169,16 +212,17 @@ Tickets.getAllTickets = (
 		projection = { teamspace: 0, project: 0, model: 0 },
 		updatedSince,
 		sort = { [`properties.${basePropertyLabels.Created_AT}`]: -1 },
+		limit,
+		skip = 0,
 	} = {},
-
 ) => {
 	const query = { teamspace, project, model };
 
 	if (updatedSince) {
 		query[`properties.${basePropertyLabels.UPDATED_AT}`] = { $gt: updatedSince };
 	}
 
-	return DbHandler.find(teamspace, TICKETS_COL, query, projection, sort);
+	return DbHandler.find(teamspace, TICKETS_COL, query, projection, sort, limit, skip);
 };
 
 module.exports = Tickets;

backend/src/v5/processors/teamspaces/projects/models/commons/tickets.js

@@ -15,9 +15,10 @@
  *  along with this program.  If not, see <http://www.gnu.org/licenses/>.
  */
 
+const { TICKETS_RESOURCES_COL, operatorToQuery } = require('../../../../../models/tickets.constants');
 const { UUIDToString, generateUUID, stringToUUID } = require('../../../../../utils/helper/uuids');
 const { addGroups, deleteGroups, getGroupsByIds } = require('./tickets.groups');
-const { addTicketsWithTemplate, getAllTickets, getTicketById, updateTickets } = require('../../../../../models/tickets');
+const { addTicketsWithTemplate, getAllTickets, getTicketById, getTicketsByFilter, updateTickets } = require('../../../../../models/tickets');
 const {
 	basePropertyLabels,
 	modulePropertyLabels,
@@ -28,15 +29,15 @@ const {
 const { createResponseCode, templates } = require('../../../../../utils/responseCodes');
 const { deleteIfUndefined, isEmpty } = require('../../../../../utils/helper/objects');
 const { generateFullSchema, getClosedStatuses } = require('../../../../../schemas/tickets/templates');
+const { getAllTemplates, getTemplatesByQuery } = require('../../../../../models/tickets.templates');
 const { getFileWithMetaAsStream, removeFiles, storeFiles } = require('../../../../../services/filesManager');
 const { getNestedProperty, setNestedProperty } = require('../../../../../utils/helper/objects');
 const { isBuffer, isUUID } = require('../../../../../utils/helper/typeCheck');
-const { TICKETS_RESOURCES_COL } = require('../../../../../models/tickets.constants');
 const { events } = require('../../../../../services/eventsManager/eventsManager.constants');
-const { getAllTemplates } = require('../../../../../models/tickets.templates');
 const { getArrayDifference } = require('../../../../../utils/helper/arrays');
 const { importComments } = require('./tickets.comments');
 const { publish } = require('../../../../../services/eventsManager/eventsManager');
+const { specialQueryFields } = require('../../../../../schemas/tickets/tickets.filters');
 
 const Tickets = {};
 
@@ -326,8 +327,34 @@ const filtersToProjection = (filters) => {
 	return projectionObject;
 };
 
-Tickets.getTicketList = (teamspace, project, model,
-	{ filters = [], updatedSince, sortBy, sortDesc }) => {
+const getQueryInfoFromQueryFilters = async (teamspace, queryFilters) => {
+	const queries = [];
+	let templateQuery;
+	let ticketCodeQuery;
+
+	queryFilters.forEach(({ propertyName, operator, value }) => {
+		if (propertyName === specialQueryFields.TEMPLATE) {
+			templateQuery = operatorToQuery[operator]('code', value);
+		} else if (propertyName === specialQueryFields.TICKET_CODE) {
+			ticketCodeQuery = operatorToQuery[operator](propertyName, value);
+		} else {
+			queries.push({ ...operatorToQuery[operator](propertyName, value) });
+		}
+	});
+
+	if (templateQuery) {
+		const temps = await getTemplatesByQuery(teamspace, templateQuery, { _id: 1 });
+		queries.push({ type: { $in: temps.map(({ _id }) => _id) } });
+	}
+
+	return {
+		query: queries.length ? { $and: queries } : {},
+		ticketCodeQuery,
+	};
+};
+
+Tickets.getTicketList = async (teamspace, project, model,
+	{ filters = [], queryFilters = [], updatedSince, sortBy, sortDesc, limit, skip }) => {
 	const { SAFETIBASE, SEQUENCING } = presetModules;
 	const {
 		[SAFETIBASE]: safetibaseProps,
@@ -361,7 +388,14 @@ Tickets.getTicketList = (teamspace, project, model,
 		sort = { [propertyToFilterName(sortBy)]: sortDesc ? -1 : 1 };
 	}
 
-	return getAllTickets(teamspace, project, model, deleteIfUndefined({ projection, updatedSince, sort }));
+	if (queryFilters.length) {
+		const queryInfo = await getQueryInfoFromQueryFilters(teamspace, queryFilters);
+		return getTicketsByFilter(teamspace, project, model,
+			deleteIfUndefined({ projection, updatedSince, sort, limit, skip, ...queryInfo }));
+	}
+
+	return getAllTickets(teamspace, project, model,
+		deleteIfUndefined({ projection, updatedSince, sort, limit, skip }));
 };
 
 Tickets.getOpenTicketsCount = async (teamspace, project, model) => {

backend/src/v5/routes/teamspaces/projects/models/common/tickets.js

@@ -48,6 +48,7 @@ const { getAllTemplates: getAllTemplatesInProject } = require('../../../../../pr
 const { getUserFromSession } = require('../../../../../utils/sessions');
 const { templates } = require('../../../../../utils/responseCodes');
 const { validateListSortAndFilter } = require('../../../../../middleware/dataConverter/inputs/teamspaces/projects/models/commons/utils');
+const { validateQueryString } = require('../../../../../middleware/dataConverter/inputs/teamspaces/projects/models/commons/ticketQueryFilters');
 
 const createTicket = (isFed) => async (req, res) => {
 	const { teamspace, project, model } = req.params;
@@ -576,19 +577,19 @@ const establishRoutes = (isFed) => {
 	 *         schema:
 	 *           type: string
 	 *       - name: updatedSince
-	 *         description: only return tickets that have been updated since a certain time (in epoch timestamp)
+	 *         description: Only return tickets that have been updated since a certain time (in epoch timestamp)
 	 *         in: query
 	 *         required: false
 	 *         schema:
 	 *           type: number
 	 *       - name: sortBy
-	 *         description: specify what property the tickets should be sorted by (default is created at)
+	 *         description: Specify what property the tickets should be sorted by (default is created at)
 	 *         in: query
 	 *         required: false
 	 *         schema:
 	 *           type: string
 	 *       - name: sortDesc
-	 *         description: specify whether the tickets should be sorted in descending order (default is true)
+	 *         description: Specify whether the tickets should be sorted in descending order (default is true)
 	 *         in: query
 	 *         required: false
 	 *         schema:
@@ -599,6 +600,24 @@ const establishRoutes = (isFed) => {
 	 *         required: false
 	 *         schema:
 	 *           type: string
+	 *       - name: query
+	 *         description: Query string that defies tickets to be included in the response. More information here https://github.com/3drepo/3drepo.io/wiki/Custom-Ticket-Query-Filters
+	 *         in: query
+	 *         required: false
+	 *         schema:
+	 *           type: string
+	 *       - name: skip
+	 *         description: Skip the first x tickets to be returned
+	 *         in: query
+	 *         required: false
+	 *         schema:
+	 *           type: number
+	 *       - name: limit
+	 *         description: Limit the amount of tickets to be returned
+	 *         in: query
+	 *         required: false
+	 *         schema:
+	 *           type: number
 	 *     responses:
 	 *       401:
 	 *         $ref: "#/components/responses/notLoggedIn"
@@ -643,7 +662,7 @@ const establishRoutes = (isFed) => {
 	 *                         description: ticket modules and their properties
 	 *
 	 */
-	router.get('/', hasReadAccess, validateListSortAndFilter, getTicketsInModel(isFed), serialiseTicketList);
+	router.get('/', hasReadAccess, validateListSortAndFilter, validateQueryString, getTicketsInModel(isFed), serialiseTicketList);
 
 	/**
 	 * @openapi

backend/src/v5/schemas/tickets/templates.js

@@ -56,7 +56,7 @@ const pinColSchema = Yup.lazy((val) => {
 	});
 });
 
-const blackListedChrsRegex = /^[^.,[\]]*$/;
+const blackListedChrsRegex = /^(?!\$)(?!.*&&)[^.,[\]":]*$/;
 
 const uniqueTypeBlackList = [
 	propTypes.LONG_TEXT,