From 779bb11e831eb902330db3ed9056f90aeba4234c Mon Sep 17 00:00:00 2001 From: NickOvt Date: Tue, 24 Oct 2023 11:07:12 +0300 Subject: [PATCH 1/5] feat(mailbox-count-limit): Set a limit for maximum number of mailbox folders ZMS-93 (#542) * add max mailboxes to settings and consts * rewrite mailbox handler create function, convert it to async as well as add check for max mailboxes * mailboxes.js add support for new createAsync function, refactor. tools.js add support for new error code * make userDate check the first check * fix error message, make it clearer. Remove OVERQUOTA error code and replace with CANNOT. Remove OVERQUOTA error in the tools.js as well * fix createAsync wrapper, strict ordering. Settings handler remove unnecessary second param --- lib/api/mailboxes.js | 10 +-- lib/consts.js | 5 +- lib/mailbox-handler.js | 156 +++++++++++++++++++--------------------- lib/settings-handler.js | 9 +++ 4 files changed, 89 insertions(+), 91 deletions(-) diff --git a/lib/api/mailboxes.js b/lib/api/mailboxes.js index f72310f2..b93e8a15 100644 --- a/lib/api/mailboxes.js +++ b/lib/api/mailboxes.js @@ -12,15 +12,7 @@ module.exports = (db, server, mailboxHandler) => { const getMailboxCounter = util.promisify(tools.getMailboxCounter); const updateMailbox = util.promisify(mailboxHandler.update.bind(mailboxHandler)); const deleteMailbox = util.promisify(mailboxHandler.del.bind(mailboxHandler)); - const createMailbox = util.promisify((...args) => { - let callback = args.pop(); - mailboxHandler.create(...args, (err, status, id) => { - if (err) { - return callback(err); - } - return callback(null, { status, id }); - }); - }); + const createMailbox = mailboxHandler.createAsync.bind(mailboxHandler); server.get( '/users/:user/mailboxes', diff --git a/lib/consts.js b/lib/consts.js index d6c971b2..7eda33a5 100644 --- a/lib/consts.js +++ b/lib/consts.js @@ -134,5 +134,8 @@ module.exports = { MAX_IMAP_UPLOAD: 10 * 1024 * 1024 * 1024, // maximum number of filters per account - MAX_FILTERS: 400 + MAX_FILTERS: 400, + + // maximum amount of mailboxes per user + MAX_MAILBOXES: 1500 }; diff --git a/lib/mailbox-handler.js b/lib/mailbox-handler.js index da3b5b68..641b2c41 100644 --- a/lib/mailbox-handler.js +++ b/lib/mailbox-handler.js @@ -3,6 +3,7 @@ const ObjectId = require('mongodb').ObjectId; const ImapNotifier = require('./imap-notifier'); const { publish, MAILBOX_CREATED, MAILBOX_RENAMED, MAILBOX_DELETED } = require('./events'); +const { SettingsHandler } = require('./settings-handler'); class MailboxHandler { constructor(options) { @@ -19,99 +20,92 @@ class MailboxHandler { redis: this.redis, pushOnly: true }); + + this.settingsHandler = new SettingsHandler({ db: this.database }); } create(user, path, opts, callback) { - this.database.collection('mailboxes').findOne( - { - user, - path - }, - (err, mailboxData) => { - if (err) { - return callback(err); - } - if (mailboxData) { - const err = new Error('Mailbox creation failed with code MailboxAlreadyExists'); - err.code = 'ALREADYEXISTS'; - err.responseCode = 400; - return callback(err, 'ALREADYEXISTS'); - } + this.createAsync(user, path, opts) + .then(mailboxData => callback(null, ...[mailboxData.status, mailboxData.id])) + .catch(err => callback(err)); + } - this.users.collection('users').findOne( - { - _id: user - }, - { - projection: { - retention: true - } - }, - (err, userData) => { - if (err) { - return callback(err); - } + async createAsync(user, path, opts) { + const userData = await this.database.collection('users').findOne({ _id: user }, { projection: { retention: true } }); - if (!userData) { - const err = new Error('This user does not exist'); - err.code = 'UserNotFound'; - err.responseCode = 404; - return callback(err, 'UserNotFound'); - } + if (!userData) { + const err = new Error('This user does not exist'); + err.code = 'UserNotFound'; + err.responseCode = 404; + throw err; + } - mailboxData = { - _id: new ObjectId(), - user, - path, - uidValidity: Math.floor(Date.now() / 1000), - uidNext: 1, - modifyIndex: 0, - subscribed: true, - flags: [], - retention: userData.retention - }; + let mailboxData = await this.database.collection('mailboxes').findOne({ user, path }); - Object.keys(opts || {}).forEach(key => { - if (!['_id', 'user', 'path'].includes(key)) { - mailboxData[key] = opts[key]; - } - }); + if (mailboxData) { + const err = new Error('Mailbox creation failed with code MailboxAlreadyExists'); + err.code = 'ALREADYEXISTS'; + err.responseCode = 400; + throw err; + } - this.database.collection('mailboxes').insertOne(mailboxData, { writeConcern: 'majority' }, (err, r) => { - if (err) { - if (err.code === 11000) { - const err = new Error('Mailbox creation failed with code MailboxAlreadyExists'); - err.code = 'ALREADYEXISTS'; - err.responseCode = 400; - return callback(err, 'ALREADYEXISTS'); - } - return callback(err); - } + const mailboxCountForUser = await this.database.collection('mailboxes').countDocuments({ user }); - publish(this.redis, { - ev: MAILBOX_CREATED, - user, - mailbox: r.insertedId, - path: mailboxData.path - }).catch(() => false); + if (mailboxCountForUser > (await this.settingsHandler.get('const:max:mailboxes'))) { + const err = new Error('Mailbox creation failed with code ReachedMailboxCountLimit. Max mailboxes count reached.'); + err.code = 'CANNOT'; + err.responseCode = 400; + throw err; + } - return this.notifier.addEntries( - mailboxData, - { - command: 'CREATE', - mailbox: r.insertedId, - path - }, - () => { - this.notifier.fire(user); - return callback(null, true, mailboxData._id); - } - ); - }); - } - ); + mailboxData = { + _id: new ObjectId(), + user, + path, + uidValidity: Math.floor(Date.now() / 1000), + uidNext: 1, + modifyIndex: 0, + subscribed: true, + flags: [], + retention: userData.retention + }; + + Object.keys(opts || {}).forEach(key => { + if (!['_id', 'user', 'path'].includes(key)) { + mailboxData[key] = opts[key]; + } + }); + + const r = this.database.collection('mailboxes').insertOne(mailboxData, { writeConcern: 'majority' }); + + try { + await publish(this.redis, { + ev: MAILBOX_CREATED, + user, + mailbox: r.insertedId, + path: mailboxData.path + }); + } catch { + // ignore + } + + await this.notifier.addEntries( + mailboxData, + { + command: 'CREATE', + mailbox: r.insertedId, + path + }, + () => { + this.notifier.fire(user); + return; } ); + + return { + status: true, + id: mailboxData._id + }; } rename(user, mailbox, newname, opts, callback) { diff --git a/lib/settings-handler.js b/lib/settings-handler.js index e68cc043..84018e61 100644 --- a/lib/settings-handler.js +++ b/lib/settings-handler.js @@ -35,6 +35,15 @@ const SETTING_KEYS = [ schema: Joi.number() }, + { + key: 'const:max:mailboxes', + name: 'Max mailboxes', + description: 'Maximum amount of mailboxes for a user', + type: 'number', + constKey: 'MAX_MAILBOXES', + schema: Joi.number() + }, + { key: 'const:max:rcpt_to', name: 'Max message recipients', From aae91bb6ace5fab3b4e6991fa08d0766a5fb14a1 Mon Sep 17 00:00:00 2001 From: NickOvt Date: Thu, 26 Oct 2023 13:11:45 +0300 Subject: [PATCH 2/5] Zms 91 (#545) * added new file to test git repo setup * add limit_allowed and limit_sent to loggelf in messages in API. Also if multiple recipients log each message separately * typo fix * messages.js refactor code. Make cleaner * fix _limit_sent. Fix earlier issue too * session -> sess. Session doesn't exist on request objs, sess exists * return pback the general SUBMIT log message to track API endpoint usage --- lib/api/messages.js | 26 +++++++++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) diff --git a/lib/api/messages.js b/lib/api/messages.js index 8ab79e13..59f673f3 100644 --- a/lib/api/messages.js +++ b/lib/api/messages.js @@ -1910,7 +1910,7 @@ module.exports = (db, server, messageHandler, userHandler, storageHandler, setti envelope, files }, - session: result.value.session, + session: result.value.sess, date, verificationResults, flags: [] @@ -2316,7 +2316,7 @@ module.exports = (db, server, messageHandler, userHandler, storageHandler, setti }); } - let maxRecipients = Number(userData.maxRecipients) || (await settingsHandler.get('const:max:recipients')); + let maxRecipients = Number(userData.recipients) || (await settingsHandler.get('const:max:recipients')); let maxRptsTo = await settingsHandler.get('const:max:rcpt_to'); // Trying to send more than allowed recipients count per email @@ -2464,6 +2464,26 @@ module.exports = (db, server, messageHandler, userHandler, storageHandler, setti } } + for (const to of Array.isArray(envelope.to) ? envelope.to : [envelope.to]) { + server.loggelf({ + short_message: `[RCPT TO: ${to}] ${result.value.sess}`, + _mail_action: 'rcpt_to', + _user: userData._id.toString(), + _queue_id: queueId, + _sent_mailbox: response.message && response.message.mailbox, + _sent_message: response.message && response.message.id, + _send_time: sendTime && sendTime.toISOString && sendTime.toISOString(), + _from: envelope.from, + _to: to, + _message_id: messageData.msgid, + _subject: messageData.subject, + _sess: result.value.sess, + _ip: result.value.ip, + _limit_allowed: userData.recipients, + _limit_sent: messagesSent + envelope.to.length + }); + } + server.loggelf({ short_message: '[SUBMIT] draft', _mail_action: 'submit_draft', @@ -2476,7 +2496,7 @@ module.exports = (db, server, messageHandler, userHandler, storageHandler, setti _to: envelope.to && envelope.to.join(','), _message_id: messageData.msgid, _subject: messageData.subject, - _sess: result.value.session, + _sess: result.value.sess, _ip: result.value.ip }); From 48b9efb8ca4b300597b2e8f5ef4aa307ac97dcfe Mon Sep 17 00:00:00 2001 From: Andris Reinman Date: Mon, 30 Oct 2023 10:55:39 +0200 Subject: [PATCH 3/5] fix(audit): Fixed `find()` query for expired audits (#547) --- lib/audit-handler.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/audit-handler.js b/lib/audit-handler.js index f9df3a37..c11319d9 100644 --- a/lib/audit-handler.js +++ b/lib/audit-handler.js @@ -262,7 +262,8 @@ class AuditHandler { let expiredAudits = await this.database .collection('audits') .find({ - expires: { $lt: new Date(), deleted: false } + expires: { $lt: new Date() }, + deleted: false }) .toArray(); From 4434cb5e1ff4414da874b62997da5ea41892a286 Mon Sep 17 00:00:00 2001 From: Andris Reinman Date: Mon, 6 Nov 2023 12:30:23 +0200 Subject: [PATCH 4/5] fix(mailbox-create): Use correct database for loading User data when creating mailboxes (#550) --- lib/mailbox-handler.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/mailbox-handler.js b/lib/mailbox-handler.js index 641b2c41..aa8be17d 100644 --- a/lib/mailbox-handler.js +++ b/lib/mailbox-handler.js @@ -31,7 +31,7 @@ class MailboxHandler { } async createAsync(user, path, opts) { - const userData = await this.database.collection('users').findOne({ _id: user }, { projection: { retention: true } }); + const userData = await this.users.collection('users').findOne({ _id: user }, { projection: { retention: true } }); if (!userData) { const err = new Error('This user does not exist'); From ea24b9328b6984db841de86309f1712f100acb97 Mon Sep 17 00:00:00 2001 From: NickOvt Date: Fri, 10 Nov 2023 17:55:16 +0200 Subject: [PATCH 5/5] feat(apidocs): Autogenerate OpenAPI docs ZMS-100 (#552) * api.js added endpoint for generating openapi docs. added new info to one route in mailboxes.js and messages.js files so that the api docs generation can be done at all * try to first generate json representation of the api docs * add initial Joi Object parsing * api.js make generation dynamic. messages.js add schemas from separate file. messages-schemas.js used for messages endpoint schemas * add additions to schemas. Add new schemas to messages.js and also add response object there. Add response object parsing functionality to api.js * add initial openapi doc yml file generation * remove manual yaml parsing with js-yaml JSON -> YAML parsing * fix replaceWithRefs and parseComponentsDecoupled functions, refactor, remove unnecessary comments and logs * add support for another endpoint * move big code from api.js to tools * fix array type representation, fix response objects, add necessary data and changes to endpoints * redo include logic into exclude login * fix api generation in tools.js to accomodate new naming of objects * fix messages.js, add structuredClone check in tools.js * fix structured clone definition * add one endpoint in messages.js to the api generation * messages.js add one more endpoint to API generation * add response to prev commit. Add new endpoint to API generation. Archive message and archive messages * finish with post endpoints in messages.js * added general request and response schemas. Also added req and res schemas for messages * add multiple GET endpoints to API generation and changed them to new design. Use general schemas made earlier * fix incorrect import of successRes * fix mailboxes.js * refactor general-schemas.js. Fix searchSchema in messages.js. Mailboxes.js fix response * tools.js rename methodObj in API generation to operationObj * tools.js api generation remove string fallbacks * messages.js finish with GET endpoints, addition to API doc generation * for openApi doc generation use JSON now instead of YAML --- api.js | 11 + lib/api/mailboxes.js | 48 +- lib/api/messages.js | 1041 +++++++++++++++++----- lib/schemas.js | 13 +- lib/schemas/request/general-schemas.js | 13 + lib/schemas/request/messages-schemas.js | 60 ++ lib/schemas/response/general-schemas.js | 9 + lib/schemas/response/messages-schemas.js | 18 + lib/tools.js | 386 ++++++++ 9 files changed, 1339 insertions(+), 260 deletions(-) create mode 100644 lib/schemas/request/general-schemas.js create mode 100644 lib/schemas/request/messages-schemas.js create mode 100644 lib/schemas/response/general-schemas.js create mode 100644 lib/schemas/response/messages-schemas.js diff --git a/api.js b/api.js index 96302417..e5ca651b 100644 --- a/api.js +++ b/api.js @@ -573,6 +573,17 @@ module.exports = done => { ); } + server.get( + { path: '/openapi', name: 'openapi-docs-generation' }, + tools.responseWrapper(async (req, res) => { + res.charSet('utf-8'); + + const routes = server.router.getRoutes(); + + tools.generateAPiDocs(routes); + }) + ); + server.on('error', err => { if (!started) { started = true; diff --git a/lib/api/mailboxes.js b/lib/api/mailboxes.js index b93e8a15..e1f3d105 100644 --- a/lib/api/mailboxes.js +++ b/lib/api/mailboxes.js @@ -7,6 +7,8 @@ const tools = require('../tools'); const roles = require('../roles'); const util = require('util'); const { sessSchema, sessIPSchema, booleanSchema } = require('../schemas'); +const { userId, mailboxId } = require('../schemas/request/general-schemas'); +const { successRes } = require('../schemas/response/general-schemas'); module.exports = (db, server, mailboxHandler) => { const getMailboxCounter = util.promisify(tools.getMailboxCounter); @@ -238,19 +240,45 @@ module.exports = (db, server, mailboxHandler) => { ); server.post( - '/users/:user/mailboxes', + { + path: '/users/:user/mailboxes', + summary: 'Create new Mailbox', + validationObjs: { + pathParams: { user: userId }, + requestBody: { + path: Joi.string() + .regex(/\/{2,}|\/$/, { invert: true }) + .required() + .description('Full path of the mailbox, folders are separated by slashes, ends with the mailbox name (unicode string)'), + hidden: booleanSchema.default(false).description('Is the folder hidden or not. Hidden folders can not be opened in IMAP.'), + retention: Joi.number() + .min(0) + .description('Retention policy for the created Mailbox. Milliseconds after a message added to mailbox expires. Set to 0 to disable.'), + sess: sessSchema, + ip: sessIPSchema + }, + queryParams: {}, + response: { + 200: { + description: 'Success', + model: Joi.object({ + success: successRes, + id: mailboxId + }) + } + } + }, + tags: ['Mailboxes'] + }, tools.responseWrapper(async (req, res) => { res.charSet('utf-8'); - const schema = Joi.object().keys({ - user: Joi.string().hex().lowercase().length(24).required(), - path: Joi.string() - .regex(/\/{2,}|\/$/, { invert: true }) - .required(), - hidden: booleanSchema.default(false), - retention: Joi.number().min(0), - sess: sessSchema, - ip: sessIPSchema + const { pathParams, requestBody, queryParams } = req.route.spec.validationObjs; + + const schema = Joi.object({ + ...pathParams, + ...requestBody, + ...queryParams }); const result = schema.validate(req.params, { diff --git a/lib/api/messages.js b/lib/api/messages.js index 59f673f3..c56e2745 100644 --- a/lib/api/messages.js +++ b/lib/api/messages.js @@ -25,6 +25,10 @@ const { getMongoDBQuery /*, getElasticSearchQuery*/ } = require('../search-query //const { getClient } = require('../elasticsearch'); const BimiHandler = require('../bimi-handler'); +const { Address, AddressOptionalNameArray, Header, Attachment, ReferenceWithAttachments, Bimi } = require('../schemas/request/messages-schemas'); +const { userId, mailboxId, messageId } = require('../schemas/request/general-schemas'); +const { MsgEnvelope } = require('../schemas/response/messages-schemas'); +const { successRes } = require('../schemas/response/general-schemas'); module.exports = (db, server, messageHandler, userHandler, storageHandler, settingsHandler) => { let maildrop = new Maildropper({ @@ -336,23 +340,113 @@ module.exports = (db, server, messageHandler, userHandler, storageHandler, setti }; server.get( - { name: 'messages', path: '/users/:user/mailboxes/:mailbox/messages' }, + { + name: 'messages', + path: '/users/:user/mailboxes/:mailbox/messages', + summary: 'List messages in a Mailbox', + description: 'Lists all messages in a mailbox', + validationObjs: { + requestBody: {}, + pathParams: { + user: Joi.string().hex().lowercase().length(24).required().description('ID of the User'), + mailbox: Joi.string().hex().lowercase().length(24).required().description('ID of the Mailbox') + }, + queryParams: { + unseen: booleanSchema.description('If true, then returns only unseen messages'), + metaData: booleanSchema.default(false).description('If true, then includes metaData in the response'), + threadCounters: booleanSchema + .default(false) + .description('If true, then includes threadMessageCount in the response. Counters come with some overhead'), + limit: Joi.number().empty('').default(20).min(1).max(250).description('How many records to return'), + order: Joi.any().empty('').allow('asc', 'desc').default('desc').description('Ordering of the records by insert date'), + next: nextPageCursorSchema, + previous: previousPageCursorSchema, + page: pageNrSchema, + sess: sessSchema, + ip: sessIPSchema, + includeHeaders: Joi.alternatives() + .try(Joi.string(), booleanSchema) + .description('Comma separated list of header keys to include in the response') + }, + response: { + 200: { + description: 'Success', + model: Joi.object({ + success: booleanSchema.description('Indicates successful response').required(), + total: Joi.number().description('How many results were found').required(), + page: Joi.number().description('Current page number. Derived from page query argument').required(), + previousCursor: Joi.alternatives() + .try(Joi.string(), booleanSchema) + .description('Either a cursor string or false if there are not any previous results') + .required(), + nextCursor: Joi.alternatives() + .try(Joi.string(), booleanSchema) + .description('Either a cursor string or false if there are not any next results') + .required(), + specialUse: Joi.string().description('Special use. If available').required(), + results: Joi.array() + .items( + Joi.object({ + id: Joi.number().required().description('ID of the Message'), + mailbox: Joi.string().required().description('ID of the Mailbox'), + thread: Joi.string().required().description('ID of the Thread'), + threadMessageCount: Joi.number().description( + 'Amount of messages in the Thread. Included if threadCounters query argument was true' + ), + from: Address.description('Sender in From: field'), + to: Joi.array().items(Address).required().description('Recipients in To: field'), + cc: Joi.array().items(Address).required().description('Recipients in Cc: field'), + bcc: Joi.array().items(Address).required().description('Recipients in Bcc: field. Usually only available for drafts'), + messageId: Joi.string().required().description('Message ID'), + subject: Joi.string().required().description('Message subject'), + date: Joi.date().required().description('Date string from header'), + idate: Joi.date().description('Date string of receive time'), + intro: Joi.string().required().description('First 128 bytes of the message'), + attachments: booleanSchema.required().description('Does the message have attachments'), + size: Joi.number().required().description('Message size in bytes'), + seen: booleanSchema.required().description('Is this message alread seen or not'), + deleted: booleanSchema + .required() + .description( + 'Does this message have a Deleted flag (should not have as messages are automatically deleted once this flag is set)' + ), + flagged: booleanSchema.required().description('Does this message have a Flagged flag'), + draft: booleanSchema.required().description('is this message a draft'), + answered: booleanSchema.required().description('Does this message have a Answered flag'), + forwarded: booleanSchema.required().description('Does this message have a $Forwarded flag'), + references: Joi.array().items(ReferenceWithAttachments).required().description('References'), + bimi: Bimi.required().description( + 'Marks BIMI verification as passed for a domain. NB! BIMI record and logo files for the domain must be valid.' + ), + contentType: Joi.object({ + value: Joi.string().required().description('MIME type of the message, eg. "multipart/mixed"'), + params: Joi.object().required().description('An object with Content-Type params as key-value pairs') + }) + .$_setFlag('objectName', 'ContentType') + .required() + .description('Parsed Content-Type header. Usually needed to identify encrypted messages and such'), + encrypted: booleanSchema.description('Specifies whether the message is encrypted'), + metaData: Joi.object().description('Custom metadata value. Included if metaData query argument was true'), + headers: Joi.object().description('Header object keys requested with the includeHeaders argument') + }).$_setFlag('objectName', 'GetMessagesResult') + ) + .required() + .description('Message listing') + }) + } + } + }, + tags: ['Messages'] + }, tools.responseWrapper(async (req, res) => { res.charSet('utf-8'); - const schema = Joi.object().keys({ - user: Joi.string().hex().lowercase().length(24).required(), - mailbox: Joi.string().hex().lowercase().length(24).required(), - unseen: booleanSchema, - metaData: booleanSchema.default(false), - threadCounters: booleanSchema.default(false), - limit: Joi.number().empty('').default(20).min(1).max(250), - order: Joi.any().empty('').allow('asc', 'desc').default('desc'), - next: nextPageCursorSchema, - previous: previousPageCursorSchema, - page: pageNrSchema, - sess: sessSchema, - ip: sessIPSchema + const { requestBody, pathParams, queryParams } = req.route.spec.validationObjs; + + const schema = Joi.object({ + ...requestBody, + ...pathParams, + ...queryParams }); const result = schema.validate(req.params, { @@ -520,61 +614,119 @@ module.exports = (db, server, messageHandler, userHandler, storageHandler, setti }) ); - const searchSchema = Joi.object().keys({ - user: Joi.string().hex().lowercase().length(24).required(), - q: Joi.string().trim().empty('').max(1024).optional(), - - mailbox: Joi.string().hex().length(24).empty(''), - thread: Joi.string().hex().length(24).empty(''), - - or: Joi.object().keys({ - query: Joi.string().trim().max(255).empty(''), - from: Joi.string().trim().empty(''), - to: Joi.string().trim().empty(''), - subject: Joi.string().trim().empty('') - }), - - query: Joi.string().trim().max(255).empty(''), - datestart: Joi.date().label('Start time').empty(''), - dateend: Joi.date().label('End time').empty(''), - from: Joi.string().trim().empty(''), - to: Joi.string().trim().empty(''), - subject: Joi.string().trim().empty(''), - minSize: Joi.number().empty(''), - maxSize: Joi.number().empty(''), - attachments: booleanSchema, - flagged: booleanSchema, - unseen: booleanSchema, + const searchSchema = { + q: Joi.string().trim().empty('').max(1024).optional().description('Additional query string'), + + mailbox: Joi.string().hex().length(24).empty('').description('ID of the Mailbox'), + thread: Joi.string().hex().length(24).empty('').description('Thread ID'), + + or: Joi.object({ + query: Joi.string() + .trim() + .max(255) + .empty('') + .description('Search string, uses MongoDB fulltext index. Covers data from mesage body and also common headers like from, to, subject etc.'), + from: Joi.string().trim().empty('').description('Partial match for the From: header line'), + to: Joi.string().trim().empty('').description('Partial match for the To: and Cc: header lines'), + subject: Joi.string().trim().empty('').description('Partial match for the Subject: header line') + }).description('At least onOne of the included terms must match'), + + query: Joi.string() + .trim() + .max(255) + .empty('') + .description('Search string, uses MongoDB fulltext index. Covers data from mesage body and also common headers like from, to, subject etc.'), + datestart: Joi.date().label('Start time').empty('').description('Datestring for the earliest message storing time'), + dateend: Joi.date().label('End time').empty('').description('Datestring for the latest message storing time'), + from: Joi.string().trim().empty('').description('Partial match for the From: header line'), + to: Joi.string().trim().empty('').description('Partial match for the To: and Cc: header lines'), + subject: Joi.string().trim().empty('').description('Partial match for the Subject: header line'), + minSize: Joi.number().empty('').description('Minimal message size in bytes'), + maxSize: Joi.number().empty('').description('Maximal message size in bytes'), + attachments: booleanSchema.description('If true, then matches only messages with attachments'), + flagged: booleanSchema.description('If true, then matches only messages with \\Flagged flags'), + unseen: booleanSchema.description('If true, then matches only messages without \\Seen flags'), includeHeaders: Joi.string() .max(1024) .trim() .empty('') .example('List-ID, MIME-Version') .description('Comma separated list of header keys to include in the response'), - searchable: booleanSchema, + searchable: booleanSchema.description('If true, then matches messages not in Junk or Trash'), sess: sessSchema, ip: sessIPSchema - }); + }; server.get( - { name: 'search', path: '/users/:user/search' }, + { + name: 'search', + path: '/users/:user/search', + validationObjs: { + queryParams: { + ...searchSchema, + ...{ + threadCounters: booleanSchema + .default(false) + .description('If true, then includes threadMessageCount in the response. Counters come with some overhead'), + limit: Joi.number().default(20).min(1).max(250).description('How many records to return'), + order: Joi.any() + .empty('') + .allow('asc', 'desc') + .optional() + .description('Ordering of the records by insert date. If no order is supplied, results are sorted by heir mongoDB ObjectId.'), + includeHeaders: Joi.string() + .max(1024) + .trim() + .empty('') + .example('List-ID, MIME-Version') + .description('Comma separated list of header keys to include in the response'), + next: nextPageCursorSchema, + previous: previousPageCursorSchema, + page: pageNrSchema + } + }, + pathParams: { user: userId }, + requestBody: {}, + response: { + 200: { + description: 'Success', + model: Joi.object({ + success: booleanSchema.required().description('Indicates successful response'), + query: Joi.string().required('Query'), + total: Joi.number().required('How many results were found'), + page: Joi.number().required('Current page number. Derived from page query argument'), + previousCursor: Joi.alternatives() + .try(booleanSchema, Joi.string()) + .required() + .description('Either a cursor string or false if there are not any previous results'), + nextCursor: Joi.alternatives() + .try(booleanSchema, Joi.string()) + .required() + .description('Either a cursor string or false if there are not any next results'), + results: Joi.array() + .items( + Joi.object({ + id: Joi.string().required().description('ID of the Domain Alias'), + alias: Joi.string().required().description('Domain Alias'), + domain: Joi.string().required().description('The domain this alias applies to') + }).$_setFlag('objectName', 'GetDomainAliasesResult') + ) + .required() + .description('Aliases listing') + }) + } + } + }, + summary: 'Search for messages', + description: 'This method allows searching for matching messages.', + tags: ['Messages'] + }, tools.responseWrapper(async (req, res) => { res.charSet('utf-8'); - const schema = searchSchema.keys({ - threadCounters: booleanSchema.default(false), - limit: Joi.number().default(20).min(1).max(250), - order: Joi.any().empty('').allow('asc', 'desc').optional(), - includeHeaders: Joi.string() - .max(1024) - .trim() - .empty('') - .example('List-ID, MIME-Version') - .description('Comma separated list of header keys to include in the response'), - next: nextPageCursorSchema, - previous: previousPageCursorSchema, - page: pageNrSchema - }); + const { requestBody, queryParams, pathParams } = req.route.spec.validationObjs; + + const schema = Joi.object({ ...requestBody, ...queryParams, ...pathParams }); const result = schema.validate(req.params, { abortEarly: false, @@ -738,20 +890,50 @@ module.exports = (db, server, messageHandler, userHandler, storageHandler, setti ); server.post( - { name: 'searchApply', path: '/users/:user/search' }, + { + name: 'searchApply', + path: '/users/:user/search', + summary: 'Search and update messages', + description: + 'This method allows applying an action to all matching messages. This is an async method so that it will return immediately. Actual modifications are run in the background.', + tags: ['Messages'], + validationObjs: { + requestBody: { + ...searchSchema, + ...{ + // actions to take on matching messages + action: Joi.object() + .keys({ + moveTo: Joi.string().hex().lowercase().length(24).description('ID of the target Mailbox if you want to move messages'), + seen: booleanSchema.description('State of the \\Seen flag'), + flagged: booleanSchema.description('State of the \\Flagged flag') + }) + .required() + .description('Define actions to take with matching messages') + } + }, + queryParams: {}, + pathParams: { + user: Joi.string().hex().lowercase().length(24).required() + }, + response: { + 200: { + description: 'Success', + model: Joi.object({ + success: booleanSchema.required().description('Indicates if the action succeeded or not'), + scheduled: Joi.string().required().description('ID of the scheduled operation'), + existing: booleanSchema.required().description('Indicates if the scheduled operation already exists') + }) + } + } + } + }, tools.responseWrapper(async (req, res) => { res.charSet('utf-8'); - const schema = searchSchema.keys({ - // actions to take on matching messages - action: Joi.object() - .keys({ - moveTo: Joi.string().hex().lowercase().length(24), - seen: booleanSchema, - flagged: booleanSchema - }) - .required() - }); + const { pathParams, requestBody, queryParams } = req.route.spec.validationObjs; + + const schema = Joi.object({ ...pathParams, ...requestBody, ...queryParams }); const result = schema.validate(req.params, { abortEarly: false, @@ -805,19 +987,126 @@ module.exports = (db, server, messageHandler, userHandler, storageHandler, setti ); server.get( - { name: 'message', path: '/users/:user/mailboxes/:mailbox/messages/:message' }, + { + name: 'message', + path: '/users/:user/mailboxes/:mailbox/messages/:message', + summary: 'Request Message information', + validationObjs: { + queryParams: { + replaceCidLinks: booleanSchema.default(false).description('If true then replaces cid links'), + markAsSeen: booleanSchema.default(false).description('If true then marks message as seen'), + sess: sessSchema, + ip: sessIPSchema + }, + pathParams: { + user: userId, + mailbox: mailboxId, + message: messageId + }, + requestBody: {}, + response: { + 200: { + description: 'Success', + model: Joi.object({ + success: successRes, + id: messageId, + mailbox: mailboxId, + user: userId, + envelope: MsgEnvelope.required(), + thread: Joi.string().required().description('ID of the Thread'), + from: Address.required(), + to: Address, + cc: Address, + bcc: Address, + subject: Joi.string().required().description('Message subject'), + messageId: Joi.string().required().description('Message-ID header'), + date: Joi.date().required().description('Date string from header'), + idate: Joi.date().description('Date string of receive time'), + list: Joi.object({ + id: Joi.string().required().description('Value from List-ID header'), + unsubscribe: Joi.string().required().description('Value from List-Unsubscribe header') + }) + .description('If set then this message is from a mailing list') + .$_setFlag('objectName', 'List'), + expires: Joi.string().description('Datestring, if set then indicates the time after this message is automatically deleted'), + seen: booleanSchema.required().description('Does this message have a \\Seen flag'), + deleted: booleanSchema.required().description('Does this message have a \\Deleted flag'), + flagged: booleanSchema.required().description('Does this message have a \\Flagged flag'), + draft: booleanSchema.required().description('Does this message have a \\Draft flag'), + html: Joi.array() + .items(Joi.string()) + .description( + 'An array of HTML string. Every array element is from a separate mime node, usually you would just join these to a single string' + ), + text: Joi.string().description('Plaintext content of the message'), + attachments: Joi.array() + .items( + Joi.object({ + id: Joi.string().required().description('Attachment ID'), + hash: Joi.string().description('SHA-256 hash of the contents of the attachment'), + filename: Joi.string().required().description('Filename of the attachment'), + contentType: Joi.string().required().description('MIME type'), + disposition: Joi.string().required().description('Attachment disposition'), + transferEncoding: Joi.string() + .required() + .description('Which transfer encoding was used (actual content when fetching attachments is not encoded)'), + related: booleanSchema + .required() + .description( + 'Was this attachment found from a multipart/related node. This usually means that this is an embedded image' + ), + sizeKb: Joi.number().required().description('Approximate size of the attachment in kilobytes') + }) + ) + .description('Attachments for the message'), + verificationResults: Joi.object({ + tls: Joi.object({ + name: Joi.object().required().description('Cipher name, eg "ECDHE-RSA-AES128-GCM-SHA256"'), + version: Joi.object().required().description('TLS version, eg "TLSv1/SSLv3"') + }) + .$_setFlag('objectName', 'Tls') + .required() + .description('TLS information. Value is false if TLS was not used'), + spf: Joi.object({}) + .required() + .description('Domain name (either MFROM or HELO) of verified SPF or false if no SPF match was found'), + dkim: Joi.object({}).required().description('Domain name of verified DKIM signature or false if no valid signature was found') + }).description( + 'Security verification info if message was received from MX. If this property is missing then do not automatically assume invalid TLS, SPF or DKIM.' + ), + bimi: Joi.object({ + certified: booleanSchema.description('If true, then this logo is from a VMC file'), + url: Joi.string().description('URL of the resource the logo was retrieved from'), + image: Joi.string().description('Data URL for the SVG image') + }).description('BIMI logo info. If logo validation failed in any way, then this property is not set'), + contentType: Joi.object({ + value: Joi.string().required().description('MIME type of the message, eg. "multipart/mixed'), + params: Joi.object({}).required().description('An object with Content-Type params as key-value pairs') + }) + .required() + .description('Parsed Content-Type header. Usually needed to identify encrypted messages and such'), + metaData: Joi.object({}).description('Custom metadata object set for this message'), + references: Joi.object({}), + files: Joi.object({}).description( + 'List of files added to this message as attachments. Applies to Drafts, normal messages do not have this property. Needed to prevent uploading the same attachment every time a draft is updated' + ), + outbound: Joi.array().items(Joi.object({})).description('Outbound queue entries'), + forwardTargets: Joi.object({}), + reference: Joi.object({}).description('Referenced message info'), + answered: booleanSchema.required(), + forwarded: booleanSchema.required() + }) + } + } + }, + tags: ['Messages'] + }, tools.responseWrapper(async (req, res) => { res.charSet('utf-8'); - const schema = Joi.object().keys({ - user: Joi.string().hex().lowercase().length(24).required(), - mailbox: Joi.string().hex().lowercase().length(24).required(), - message: Joi.number().min(1).required(), - replaceCidLinks: booleanSchema.default(false), - markAsSeen: booleanSchema.default(false), - sess: sessSchema, - ip: sessIPSchema - }); + const { requestBody, queryParams, pathParams } = req.route.spec.validationObjs; + + const schema = Joi.object({ ...requestBody, ...queryParams, ...pathParams }); const result = schema.validate(req.params, { abortEarly: false, @@ -1104,14 +1393,41 @@ module.exports = (db, server, messageHandler, userHandler, storageHandler, setti ); server.get( - { name: 'raw', path: '/users/:user/mailboxes/:mailbox/messages/:message/message.eml' }, + { + name: 'raw', + path: '/users/:user/mailboxes/:mailbox/messages/:message/message.eml', + summary: 'Get Message source', + description: 'This method returns the full RFC822 formatted source of the stored message', + validationObjs: { + pathParams: { + user: userId, + mailbox: mailboxId, + message: messageId + }, + queryParams: { + sess: sessSchema, + ip: sessIPSchema + }, + requestBody: {}, + response: { + 200: { + description: 'Success', + model: Joi.object({ + success: Joi.object({}).description('Success') + }) + } + } + }, + responseType: 'message/rfc822', + tags: ['Messages'] + }, tools.responseWrapper(async (req, res) => { - const schema = Joi.object().keys({ - user: Joi.string().hex().lowercase().length(24).required(), - mailbox: Joi.string().hex().lowercase().length(24).required(), - message: Joi.number().min(1).required(), - sess: sessSchema, - ip: sessIPSchema + const { pathParams, queryParams, requestBody } = req.route.spec.validationObjs; + + const schema = Joi.object({ + ...pathParams, + ...queryParams, + ...requestBody }); const result = schema.validate(req.params, { @@ -1200,16 +1516,43 @@ module.exports = (db, server, messageHandler, userHandler, storageHandler, setti ); server.get( - { name: 'attachment', path: '/users/:user/mailboxes/:mailbox/messages/:message/attachments/:attachment' }, + { + name: 'attachment', + path: '/users/:user/mailboxes/:mailbox/messages/:message/attachments/:attachment', + summary: 'Download Attachment', + description: 'This method returns attachment file contents in binary form', + validationObjs: { + queryParams: {}, + pathParams: { + user: userId, + mailbox: mailboxId, + message: messageId, + attachment: Joi.string() + .regex(/^ATT\d+$/i) + .uppercase() + .required() + .description('ID of the Attachment') + }, + requestBody: {}, + response: { + 200: { + description: 'Success', + model: Joi.object({ + success: Joi.binary() + }) + } + } + }, + responseType: 'application/octet-stream', + tags: ['Messages'] + }, tools.responseWrapper(async (req, res) => { - const schema = Joi.object().keys({ - user: Joi.string().hex().lowercase().length(24).required(), - mailbox: Joi.string().hex().lowercase().length(24).required(), - message: Joi.number().min(1).required(), - attachment: Joi.string() - .regex(/^ATT\d+$/i) - .uppercase() - .required() + const { requestBody, queryParams, pathParams } = req.route.spec.validationObjs; + + const schema = Joi.object({ + ...requestBody, + ...queryParams, + ...pathParams }); const result = schema.validate(req.params, { @@ -1529,114 +1872,109 @@ module.exports = (db, server, messageHandler, userHandler, storageHandler, setti ); server.post( - '/users/:user/mailboxes/:mailbox/messages', + { + path: '/users/:user/mailboxes/:mailbox/messages', + summary: 'Upload Message', + description: + 'This method allows to upload either an RFC822 formatted message or a message structure to a mailbox. Raw message is stored unmodified, no headers are added or removed. If you want to generate the uploaded message from structured data fields, then do not use the raw property.', + validationObjs: { + pathParams: { + user: userId, + mailbox: mailboxId + }, + requestBody: { + date: Joi.date(), + unseen: booleanSchema.default(false).description('Is the message unseen or not'), + flagged: booleanSchema.default(false).description('Is the message flagged or not'), + draft: booleanSchema.default(false).description('Is the message a draft or not'), + + raw: Joi.binary() + .max(consts.MAX_ALLOWED_MESSAGE_SIZE) + .empty('') + .description( + 'base64 encoded message source. Alternatively, you can provide this value as POST body by using message/rfc822 MIME type. If raw message is provided then it overrides any other mail configuration' + ), + + from: Address.description('Addres for the From: header'), + + replyTo: Address.description('Address for the Reply-To: header'), + + to: AddressOptionalNameArray.description('Addresses for the To: header'), + + cc: AddressOptionalNameArray.description('Addresses for the Cc: header'), + + bcc: AddressOptionalNameArray.description('Addresses for the Bcc: header'), + + headers: Joi.array() + .items(Header) + .description( + 'Custom headers for the message. If reference message is set then In-Reply-To and References headers are set automatically' + ), + + subject: Joi.string() + .empty('') + .max(2 * 1024) + .description('Message subject. If not then resolved from Reference message'), + text: Joi.string() + .empty('') + .max(1024 * 1024) + .description('Plaintext message'), + html: Joi.string() + .empty('') + .max(1024 * 1024) + .description('HTML formatted message'), + + files: Joi.array() + .items(Joi.string().hex().lowercase().length(24)) + .description( + 'Attachments as storage file IDs. NB! When retrieving message info then an array of objects is returned. When uploading a message then an array of IDs is used.' + ), + + attachments: Joi.array().items(Attachment).description('Attachments for the message'), + + metaData: metaDataSchema.label('metaData').description('Optional metadata, must be an object or JSON formatted string'), + + reference: ReferenceWithAttachments.description( + 'Optional referenced email. If uploaded message is a reply draft and relevant fields are not provided then these are resolved from the message to be replied to' + ), + + replacePrevious: Joi.object({ + mailbox: Joi.string().hex().lowercase().length(24), + id: Joi.number().required() + }).description('If set, then deletes a previous message when storing the new one. Useful when uploading a new Draft message.'), + + bimi: Bimi.description('Marks BIMI verification as passed for a domain. NB! BIMI record and logo files for the domain must be valid.'), + + sess: sessSchema, + ip: sessIPSchema + }, + queryParams: {}, + response: { + 200: { + description: 'Success', + model: Joi.object({ + success: Joi.boolean().description('Indicates successful response'), + message: Joi.object({ + id: Joi.number(), + malbox: Joi.string(), + size: Joi.number() + }).description('Message information'), + previousDeleted: Joi.boolean().description('Set if replacing a previous message was requested') + }) + } + } + }, + tags: ['Messages'] + }, tools.responseWrapper(async (req, res) => { res.charSet('utf-8'); - const schema = Joi.object().keys({ - user: Joi.string().hex().lowercase().length(24).required(), - mailbox: Joi.string().hex().lowercase().length(24).required(), - date: Joi.date(), - unseen: booleanSchema.default(false), - flagged: booleanSchema.default(false), - draft: booleanSchema.default(false), - - raw: Joi.binary().max(consts.MAX_ALLOWED_MESSAGE_SIZE).empty(''), - - from: Joi.object().keys({ - name: Joi.string().empty('').max(255), - address: Joi.string().email({ tlds: false }).required() - }), - - replyTo: Joi.object().keys({ - name: Joi.string().empty('').max(255), - address: Joi.string().email({ tlds: false }).required() - }), - - to: Joi.array().items( - Joi.object().keys({ - name: Joi.string().empty('').max(255), - address: Joi.string().email({ tlds: false }).required() - }) - ), + const { pathParams, requestBody, queryParams } = req.route.spec.validationObjs; - cc: Joi.array().items( - Joi.object().keys({ - name: Joi.string().empty('').max(255), - address: Joi.string().email({ tlds: false }).required() - }) - ), - - bcc: Joi.array().items( - Joi.object().keys({ - name: Joi.string().empty('').max(255), - address: Joi.string().email({ tlds: false }).required() - }) - ), - - headers: Joi.array().items( - Joi.object().keys({ - key: Joi.string().empty('').max(255), - value: Joi.string() - .empty('') - .max(100 * 1024) - }) - ), - - subject: Joi.string() - .empty('') - .max(2 * 1024), - text: Joi.string() - .empty('') - .max(1024 * 1024), - html: Joi.string() - .empty('') - .max(1024 * 1024), - - files: Joi.array().items(Joi.string().hex().lowercase().length(24)), - - attachments: Joi.array().items( - Joi.object().keys({ - filename: Joi.string().empty('').max(255), - - contentType: Joi.string().empty('').max(255), - contentTransferEncoding: Joi.string().empty(''), - contentDisposition: Joi.string().empty('').trim().lowercase().valid('inline', 'attachment'), - cid: Joi.string().empty('').max(255), - - encoding: Joi.string().empty('').default('base64'), - content: Joi.string().required() - }) - ), - - metaData: metaDataSchema.label('metaData'), - - reference: Joi.object().keys({ - mailbox: Joi.string().hex().lowercase().length(24).required(), - id: Joi.number().required(), - action: Joi.string().valid('reply', 'replyAll', 'forward').required(), - attachments: Joi.alternatives().try( - booleanSchema, - Joi.array().items( - Joi.string() - .regex(/^ATT\d+$/i) - .uppercase() - ) - ) - }), - - replacePrevious: Joi.object({ - mailbox: Joi.string().hex().lowercase().length(24), - id: Joi.number().required() - }), - - bimi: Joi.object().keys({ - domain: Joi.string().domain().required(), - selector: Joi.string().empty('').max(255) - }), - - sess: sessSchema, - ip: sessIPSchema + const schema = Joi.object({ + ...pathParams, + ...requestBody, + ...queryParams }); if (!req.params.raw && req.body && (Buffer.isBuffer(req.body) || typeof req.body === 'string')) { @@ -1983,18 +2321,56 @@ module.exports = (db, server, messageHandler, userHandler, storageHandler, setti ); server.post( - '/users/:user/mailboxes/:mailbox/messages/:message/forward', + { + path: '/users/:user/mailboxes/:mailbox/messages/:message/forward', + validationObjs: { + pathParams: { + user: userId, + mailbox: mailboxId, + message: messageId + }, + queryParams: {}, + requestBody: { + target: Joi.number().min(1).max(1000).description('Number of original forwarding target'), + addresses: Joi.array() + .items(Joi.string().email({ tlds: false })) + .description('An array of additional forward targets'), + sess: sessSchema, + ip: sessIPSchema + }, + response: { + 200: { + description: 'Success', + model: Joi.object({ + success: Joi.boolean().description('Indicates successful response'), + queueId: Joi.string().description('Message ID in outbound queue'), + forwarded: Joi.array() + .items( + Joi.object({ + seq: Joi.string(), + type: Joi.string(), + value: Joi.string() + }).$_setFlag('objectName', 'Forwarded') + ) + .description('Information about forwarding targets') + }) + } + } + }, + summary: 'Forward stored Message', + description: + 'This method allows either to re-forward a message to an original forward target or forward it to some other address. This is useful if a user had forwarding turned on but the message was not delivered so you can try again. Forwarding does not modify the original message.', + tags: ['Messages'] + }, tools.responseWrapper(async (req, res) => { res.charSet('utf-8'); - const schema = Joi.object().keys({ - user: Joi.string().hex().lowercase().length(24).required(), - mailbox: Joi.string().hex().lowercase().length(24).required(), - message: Joi.number().required(), - target: Joi.number().min(1).max(1000), - addresses: Joi.array().items(Joi.string().email({ tlds: false })), - sess: sessSchema, - ip: sessIPSchema + const { pathParams, requestBody, queryParams } = req.route.spec.validationObjs; + + const schema = Joi.object({ + ...pathParams, + ...requestBody, + ...queryParams }); const result = schema.validate(req.params, { @@ -2154,18 +2530,51 @@ module.exports = (db, server, messageHandler, userHandler, storageHandler, setti ); server.post( - '/users/:user/mailboxes/:mailbox/messages/:message/submit', + { + path: '/users/:user/mailboxes/:mailbox/messages/:message/submit', + validationObjs: { + pathParams: { + user: userId, + mailbox: mailboxId, + message: messageId + }, + queryParams: {}, + requestBody: { + deleteFiles: booleanSchema.description('If true then deletes attachment files listed in metaData.files array'), + sendTime: Joi.date().description('Datestring for delivery if message should be sent some later time'), + sess: sessSchema, + ip: sessIPSchema + }, + response: { + 200: { + description: 'Success', + model: Joi.object({ + success: booleanSchema.description('Indicates successful response').required(), + queueId: Joi.string().description('Message ID in outbound queue').required(), + message: Joi.object({ + id: Joi.number().description('Message ID in mailbox').required(), + mailbox: Joi.string().description('Mailbox ID the message was stored into').required(), + size: Joi.number().description('Size of the RFC822 formatted email') + }) + .description('Message information') + .$_setFlag('objectName', 'Message') + }) + } + } + }, + summary: 'Submit Draft for delivery', + description: 'This method allows to submit a draft message for delivery. Draft is moved to Sent mail folder.', + tags: ['Messages'] + }, tools.responseWrapper(async (req, res) => { res.charSet('utf-8'); - const schema = Joi.object().keys({ - user: Joi.string().hex().lowercase().length(24).required(), - mailbox: Joi.string().hex().lowercase().length(24).required(), - message: Joi.number().required(), - deleteFiles: booleanSchema, - sendTime: Joi.date(), - sess: sessSchema, - ip: sessIPSchema + const { pathParams, requestBody, queryParams } = req.route.spec.validationObjs; + + const schema = Joi.object({ + ...pathParams, + ...queryParams, + ...requestBody }); const result = schema.validate(req.params, { @@ -2547,25 +2956,110 @@ module.exports = (db, server, messageHandler, userHandler, storageHandler, setti ); server.get( - { name: 'archived', path: '/users/:user/archived/messages' }, + { + name: 'archived', + path: '/users/:user/archived/messages', + summary: 'List archived messages', + description: 'Archive contains all recently deleted messages besides Drafts etc.', + validationObjs: { + pathParams: { + user: userId + }, + queryParams: { + limit: Joi.number().empty('').default(20).min(1).max(250).description('How many records to return'), + next: nextPageCursorSchema, + previous: previousPageCursorSchema, + order: Joi.any().empty('').allow('asc', 'desc').default('desc').description('Ordering of the records by insert date'), + includeHeaders: Joi.string() + .max(1024) + .trim() + .empty('') + .example('List-ID, MIME-Version') + .description('Comma separated list of header keys to include in the response'), + page: pageNrSchema, + sess: sessSchema, + ip: sessIPSchema + }, + requestBody: {}, + response: { + 200: { + description: 'Success', + model: Joi.object({ + success: booleanSchema.description('Indicates successful response').required(), + total: Joi.number().description('How many results were found').required(), + page: Joi.number().description('Current page number. Derived from page query argument').required(), + previousCursor: Joi.alternatives() + .try(Joi.string(), booleanSchema) + .description('Either a cursor string or false if there are not any previous results') + .required(), + nextCursor: Joi.alternatives() + .try(Joi.string(), booleanSchema) + .description('Either a cursor string or false if there are not any next results') + .required(), + specialUse: Joi.string().description('Special use. If available').required(), + results: Joi.array() + .items( + Joi.object({ + id: Joi.number().required().description('ID of the Message'), + mailbox: Joi.string().required().description('ID of the Mailbox'), + thread: Joi.string().required().description('ID of the Thread'), + threadMessageCount: Joi.number().description( + 'Amount of messages in the Thread. Included if threadCounters query argument was true' + ), + from: Address.description('Sender in From: field'), + to: Joi.array().items(Address).required().description('Recipients in To: field'), + cc: Joi.array().items(Address).required().description('Recipients in Cc: field'), + bcc: Joi.array().items(Address).required().description('Recipients in Bcc: field. Usually only available for drafts'), + messageId: Joi.string().required().description('Message ID'), + subject: Joi.string().required().description('Message subject'), + date: Joi.date().required().description('Date string from header'), + idate: Joi.date().description('Date string of receive time'), + intro: Joi.string().required().description('First 128 bytes of the message'), + attachments: booleanSchema.required().description('Does the message have attachments'), + size: Joi.number().required().description('Message size in bytes'), + seen: booleanSchema.required().description('Is this message alread seen or not'), + deleted: booleanSchema + .required() + .description( + 'Does this message have a Deleted flag (should not have as messages are automatically deleted once this flag is set)' + ), + flagged: booleanSchema.required().description('Does this message have a Flagged flag'), + draft: booleanSchema.required().description('is this message a draft'), + answered: booleanSchema.required().description('Does this message have a Answered flag'), + forwarded: booleanSchema.required().description('Does this message have a $Forwarded flag'), + references: Joi.array().items(ReferenceWithAttachments).required().description('References'), + bimi: Bimi.required().description( + 'Marks BIMI verification as passed for a domain. NB! BIMI record and logo files for the domain must be valid.' + ), + contentType: Joi.object({ + value: Joi.string().required().description('MIME type of the message, eg. "multipart/mixed"'), + params: Joi.object().required().description('An object with Content-Type params as key-value pairs') + }) + .$_setFlag('objectName', 'ContentType') + .required() + .description('Parsed Content-Type header. Usually needed to identify encrypted messages and such'), + encrypted: booleanSchema.description('Specifies whether the message is encrypted'), + metaData: Joi.object().description('Custom metadata value. Included if metaData query argument was true'), + headers: Joi.object().description('Header object keys requested with the includeHeaders argument') + }).$_setFlag('objectName', 'GetMessagesResult') + ) + .required() + .description('Message listing') + }) + } + } + }, + tags: ['Archive'] + }, tools.responseWrapper(async (req, res) => { res.charSet('utf-8'); - const schema = Joi.object().keys({ - user: Joi.string().hex().lowercase().length(24).required(), - limit: Joi.number().empty('').default(20).min(1).max(250), - next: nextPageCursorSchema, - previous: previousPageCursorSchema, - order: Joi.any().empty('').allow('asc', 'desc').default('desc'), - includeHeaders: Joi.string() - .max(1024) - .trim() - .empty('') - .example('List-ID, MIME-Version') - .description('Comma separated list of header keys to include in the response'), - page: pageNrSchema, - sess: sessSchema, - ip: sessIPSchema + const { pathParams, queryParams, requestBody } = req.route.spec.validationObjs; + + const schema = Joi.object({ + ...pathParams, + ...queryParams, + ...requestBody }); const result = schema.validate(req.params, { @@ -2691,16 +3185,44 @@ module.exports = (db, server, messageHandler, userHandler, storageHandler, setti ); server.post( - { name: 'create_restore_task', path: '/users/:user/archived/restore' }, + { + name: 'create_restore_task', + path: '/users/:user/archived/restore', + tags: ['Archive'], + summary: 'Restore archived messages', + description: + 'Initiates a restore task to move archived messages of a date range back to the mailboxes the messages were deleted from. If target mailbox does not exist, then the messages are moved to INBOX.', + validationObjs: { + pathParams: { + user: userId + }, + requestBody: { + start: Joi.date().label('Start time').required().description('Datestring'), + end: Joi.date().label('End time').required().description('Datestring'), + sess: sessSchema, + ip: sessIPSchema + }, + queryParams: {}, + response: { + 200: { + description: 'Success', + model: Joi.object({ + success: booleanSchema.required().description('Indicates successful response'), + task: Joi.string().required().description('Task ID') + }) + } + } + } + }, tools.responseWrapper(async (req, res) => { res.charSet('utf-8'); - const schema = Joi.object().keys({ - user: Joi.string().hex().lowercase().length(24).required(), - start: Joi.date().label('Start time').required(), - end: Joi.date().label('End time').required(), - sess: sessSchema, - ip: sessIPSchema + const { pathParams, requestBody, queryParams } = req.route.spec.validationObjs; + + const schema = Joi.object({ + ...pathParams, + ...queryParams, + ...requestBody }); const result = schema.validate(req.params, { @@ -2775,16 +3297,45 @@ module.exports = (db, server, messageHandler, userHandler, storageHandler, setti ); server.post( - { name: 'archived_restore', path: '/users/:user/archived/messages/:message/restore' }, + { + name: 'archived_restore', + path: '/users/:user/archived/messages/:message/restore', + summary: 'Restore archived messages ', + description: + 'Initiates a restore task to move archived messages of a date range back to the mailboxes the messages were deleted from. If target mailbox does not exist, then the messages are moved to INBOX.', + tags: ['Archive'], + validationObjs: { + requestBody: { + mailbox: Joi.string().hex().lowercase().length(24).description('ID of the target Mailbox. If not set then original mailbox is used.'), + sess: sessSchema, + ip: sessIPSchema + }, + queryParams: {}, + pathParams: { + user: userId, + message: messageId + }, + response: { + 200: { + description: 'Success', + model: Joi.object({ + success: booleanSchema.required().description('Indicates successful response'), + mailbox: Joi.string().required().description('Maibox ID the message was moved to'), + id: Joi.number().required().description('New ID for the Message') + }) + } + } + } + }, tools.responseWrapper(async (req, res) => { res.charSet('utf-8'); - const schema = Joi.object().keys({ - user: Joi.string().hex().lowercase().length(24).required(), - message: Joi.string().hex().lowercase().length(24).required(), - mailbox: Joi.string().hex().lowercase().length(24), - sess: sessSchema, - ip: sessIPSchema + const { pathParams, requestBody, queryParams } = req.route.spec.validationObjs; + + const schema = Joi.object({ + ...pathParams, + ...requestBody, + ...queryParams }); const result = schema.validate(req.params, { diff --git a/lib/schemas.js b/lib/schemas.js index 33ec4f09..1aa7d454 100644 --- a/lib/schemas.js +++ b/lib/schemas.js @@ -3,13 +3,14 @@ const EJSON = require('mongodb-extended-json'); const Joi = require('joi'); -const sessSchema = Joi.string().max(255).label('Session identifier'); +const sessSchema = Joi.string().max(255).label('Session identifier').description('Session identifier for the logs'); const sessIPSchema = Joi.string() .ip({ version: ['ipv4', 'ipv6'], cidr: 'forbidden' }) - .label('Client IP'); + .label('Client IP') + .description('IP address for the logs '); /* const tagSchema = Joi.string().max(); @@ -83,9 +84,11 @@ const metaDataValidator = () => (value, helpers) => { const mongoCursorSchema = Joi.string().trim().empty('').custom(mongoCursorValidator({}), 'Cursor validation').max(1024); const pageLimitSchema = Joi.number().default(20).min(1).max(250).label('Page size'); -const pageNrSchema = Joi.number().default(1).label('Page number'); -const nextPageCursorSchema = mongoCursorSchema.label('Next page cursor'); -const previousPageCursorSchema = mongoCursorSchema.label('Previous page cursor'); +const pageNrSchema = Joi.number().default(1).label('Page number').description('Current page number. Informational only, page numbers start from 1'); +const nextPageCursorSchema = mongoCursorSchema.label('Next page cursor').description('Cursor value for next page, retrieved from nextCursor response value'); +const previousPageCursorSchema = mongoCursorSchema + .label('Previous page cursor') + .description('Cursor value for previous page, retrieved from previousCursor response value'); const booleanSchema = Joi.boolean().empty('').truthy('Y', 'true', 'yes', 'on', '1', 1).falsy('N', 'false', 'no', 'off', '0', 0); const metaDataSchema = Joi.any().custom(metaDataValidator({}), 'metadata validation'); diff --git a/lib/schemas/request/general-schemas.js b/lib/schemas/request/general-schemas.js new file mode 100644 index 00000000..2eef368e --- /dev/null +++ b/lib/schemas/request/general-schemas.js @@ -0,0 +1,13 @@ +'use strict'; + +const Joi = require('joi'); + +const userId = Joi.string().hex().lowercase().length(24).required().description('ID of the User'); +const mailboxId = Joi.string().hex().lowercase().length(24).required().description('ID of the Mailbox'); +const messageId = Joi.number().min(1).required().description('Message ID'); + +module.exports = { + userId, + mailboxId, + messageId +}; diff --git a/lib/schemas/request/messages-schemas.js b/lib/schemas/request/messages-schemas.js new file mode 100644 index 00000000..d4226e60 --- /dev/null +++ b/lib/schemas/request/messages-schemas.js @@ -0,0 +1,60 @@ +'use strict'; + +const Joi = require('joi'); +const { booleanSchema } = require('../../schemas'); + +const Address = Joi.object({ + name: Joi.string().empty('').max(255), + address: Joi.string().email({ tlds: false }).required() +}).$_setFlag('objectName', 'Address'); + +const AddressOptionalName = Joi.object({ + name: Joi.string().empty('').max(255), + address: Joi.string().email({ tlds: false }).required() +}).$_setFlag('objectName', 'AddressOptionalName'); + +const AddressOptionalNameArray = Joi.array().items(AddressOptionalName); + +const Header = Joi.object({ + key: Joi.string().empty('').max(255), + value: Joi.string() + .empty('') + .max(100 * 1024) +}).$_setFlag('objectName', 'Header'); + +const Attachment = Joi.object({ + filename: Joi.string().empty('').max(255), + contentType: Joi.string().empty('').max(255), + encoding: Joi.string().empty('').default('base64'), + contentTransferEncoding: Joi.string().empty(''), + content: Joi.string().required(), + cid: Joi.string().empty('').max(255) +}).$_setFlag('objectName', 'Attachment'); + +const ReferenceWithAttachments = Joi.object({ + mailbox: Joi.string().hex().lowercase().length(24).required(), + id: Joi.number().required(), + action: Joi.string().valid('reply', 'replyAll', 'forward').required(), + attachments: Joi.alternatives().try( + booleanSchema, + Joi.array().items( + Joi.string() + .regex(/^ATT\d+$/i) + .uppercase() + ) + ) +}).$_setFlag('objectName', 'ReferenceWithAttachments'); + +const Bimi = Joi.object({ + domain: Joi.string().domain().required(), + selector: Joi.string().empty('').max(255) +}).$_setFlag('objectName', 'Bimi'); + +module.exports = { + Address, + AddressOptionalNameArray, + Header, + Attachment, + ReferenceWithAttachments, + Bimi +}; diff --git a/lib/schemas/response/general-schemas.js b/lib/schemas/response/general-schemas.js new file mode 100644 index 00000000..208eebd7 --- /dev/null +++ b/lib/schemas/response/general-schemas.js @@ -0,0 +1,9 @@ +'use strict'; + +const { booleanSchema } = require('../../schemas'); + +const successRes = booleanSchema.required().description('Indicates successful response'); + +module.exports = { + successRes +}; diff --git a/lib/schemas/response/messages-schemas.js b/lib/schemas/response/messages-schemas.js new file mode 100644 index 00000000..bc0045b2 --- /dev/null +++ b/lib/schemas/response/messages-schemas.js @@ -0,0 +1,18 @@ +'use strict'; +const Joi = require('joi'); + +const Rcpt = Joi.object({ + value: Joi.string().required().description('RCPT TO address as provided by SMTP client'), + formatted: Joi.string().required().description('Normalized RCPT address') +}).$_setFlag('objectName', 'Rcpt'); + +const MsgEnvelope = Joi.object({ + from: Joi.string().required().description('Address from MAIL FROM'), + rcpt: Joi.array().items(Rcpt).description('Array of addresses from RCPT TO (should have just one normally)') +}) + .description('SMTP envelope (if available)') + .$_setFlag('objectName', 'Envelope'); + +module.exports = { + MsgEnvelope +}; diff --git a/lib/tools.js b/lib/tools.js index 98a4df68..85f59d3c 100644 --- a/lib/tools.js +++ b/lib/tools.js @@ -19,6 +19,8 @@ const addressparser = require('nodemailer/lib/addressparser'); let templates = false; +const structuredCloneWrapper = typeof structuredClone === 'function' ? structuredClone : obj => JSON.parse(JSON.stringify(obj)); + function checkRangeQuery(uids, ne) { // check if uids is a straight continous array and if such then return a range query, // otherwise retrun a $in query @@ -605,6 +607,176 @@ function buildCertChain(cert, ca) { .join('\n'); } +// ignore function and symbol types +const joiTypeToOpenApiTypeMap = { + any: 'object', + number: 'number', + link: 'string', + boolean: 'boolean', + date: 'string', + string: 'string', + binary: 'string' +}; + +function replaceWithRefs(reqBodyData) { + if (reqBodyData.type === 'array') { + const obj = reqBodyData.items; + + replaceWithRefs(obj); + } else if (reqBodyData.type === 'object') { + if (reqBodyData.objectName) { + const objectName = reqBodyData.objectName; + Object.keys(reqBodyData).forEach(key => { + if (key !== '$ref') { + delete reqBodyData[key]; + } + }); + reqBodyData.$ref = `#/components/schemas/${objectName}`; + } else { + for (const key in reqBodyData.properties) { + replaceWithRefs(reqBodyData.properties[key]); + } + } + } else if (reqBodyData.type === 'alternatives') { + for (const obj in reqBodyData.oneOf) { + replaceWithRefs(obj); + } + } +} + +function parseComponetsDecoupled(component, components) { + if (component.type === 'array') { + const obj = structuredCloneWrapper(component.items); // copy + + if (obj.objectName) { + for (const key in obj.properties) { + parseComponetsDecoupled(obj.properties[key], components); + } + + // in case the Array itself is marked as a separate object > + const objectName = obj.objectName; + components[objectName] = obj; + delete components[objectName].objectName; + // ^ + } + } else if (component.type === 'object') { + const obj = structuredCloneWrapper(component); // copy + const objectName = obj.objectName; + + for (const key in obj.properties) { + parseComponetsDecoupled(obj.properties[key], components); + } + + if (objectName) { + components[objectName] = obj; + delete components[objectName].objectName; + } + } else if (component.oneOf) { + // Joi object is of 'alternatives' types + for (const obj in component.oneOf) { + parseComponetsDecoupled({ ...obj }, components); + } + } +} + +/** + * Parse Joi Objects + */ +function parseJoiObject(path, joiObject, requestBodyProperties) { + if (joiObject.type === 'object') { + const fieldsMap = joiObject._ids._byKey; + + const data = { + type: joiObject.type, + description: joiObject._flags.description, + properties: {}, + required: [] + }; + + if (joiObject._flags.objectName) { + data.objectName = joiObject._flags.objectName; + } + + if (path) { + requestBodyProperties[path] = data; + } else if (Array.isArray(requestBodyProperties)) { + requestBodyProperties.push(data); + } else { + requestBodyProperties.items = data; + } + + for (const [key, value] of fieldsMap) { + if (value.schema._flags.presence === 'required') { + data.required.push(key); + } + parseJoiObject(key, value.schema, data.properties); + } + } else if (joiObject.type === 'alternatives') { + const matches = joiObject.$_terms.matches; + + const data = { + oneOf: [], + description: joiObject._flags.description + }; + + if (path) { + requestBodyProperties[path] = data; + } else if (Array.isArray(requestBodyProperties)) { + requestBodyProperties.push(data); + } else { + requestBodyProperties.items = data; + } + + for (const alternative of matches) { + parseJoiObject(null, alternative.schema, data.oneOf); + } + } else if (joiObject.type === 'array') { + const elems = joiObject?.$_terms.items; + + const data = { + type: 'array', + items: {}, + description: joiObject._flags.description + }; + + if (path) { + requestBodyProperties[path] = data; + } else if (Array.isArray(requestBodyProperties)) { + requestBodyProperties.push(data); + } else { + requestBodyProperties.items = data; + } + parseJoiObject(null, elems[0], data); + } else { + const openApiType = joiTypeToOpenApiTypeMap[joiObject.type]; // even if type is object here then ignore and do not go recursive + const isRequired = joiObject._flags.presence === 'required'; + const description = joiObject._flags.description; + let format = undefined; + + if (!openApiType) { + throw new Error('Unsupported type! Check API endpoint!'); + } + + if (joiObject.type !== openApiType) { + // type has changed, so probably string, acquire format + format = joiObject.type; + } + + const data = { type: openApiType, description, required: isRequired }; + if (format) { + data.format = format; + } + + if (path) { + requestBodyProperties[path] = data; + } else if (Array.isArray(requestBodyProperties)) { + requestBodyProperties.push(data); + } else { + requestBodyProperties.items = data; + } + } +} + module.exports = { normalizeAddress, normalizeDomain, @@ -694,5 +866,219 @@ module.exports = { res.json(data); } }; + }, + + async generateAPiDocs(routes) { + let docs = { + openapi: '3.0.0', + info: { + title: 'WildDuck API', + description: 'WildDuck API docs', + version: '1.0.0', + contact: { + url: 'https://github.com/nodemailer/wildduck' + } + }, + servers: [{ url: 'https://api.wildduck.email' }], + tags: [ + { name: 'Addresses' }, + { name: 'ApplicationPasswords' }, + { + name: 'Archive', + description: + 'Archive includes all deleted messages. Once messages are old enough then these are permanenetly deleted from the archive as well. Until then you can restore the deleted messages.' + }, + { + name: 'Audit', + description: + 'Auditing allows to monitor an email account. All existing, deleted and new emails are copied to the auditing system. See also https://github.com/nodemailer/wildduck-audit-manager' + }, + { name: 'Authentication' }, + { name: 'Autoreplies' }, + { + name: 'Certs', + description: + 'WildDuck allows to register TLS certificates to be used with SNI connections. These certificates are used by IMAP, POP3, API and SMTP servers when a SNI capable client establishes a TLS connection. This does not apply for MX servers.' + }, + { + name: 'DKIM', + description: + 'Whenever an email is sent WildDuck checks if there is a DKIM key registered for the domain name of the sender address and uses it to sign the message.' + }, + { + name: 'DomainAccess', + description: 'Add sender domain names to allowlist (messages are all accepted) or blocklist (messages are sent to Spam folder)' + }, + { name: 'DomainAliases' }, + { name: 'Filters' }, + { name: 'Mailboxes' }, + { name: 'Messages' }, + { name: 'Settings' }, + { + name: 'Storage', + description: + 'Storage allows easier attachment handling when composing Draft messages. Instead of uploading the attachmnent with every draft update, you store the attachment to the Storage and then link stored file for the Draft.' + }, + { name: 'Submission' }, + { name: 'TwoFactorAuth' }, + { name: 'Users' }, + { name: 'Webhooks' } + ] + }; + + const mapPathToMethods = {}; // map -> {path -> {post -> {}, put -> {}, delete -> {}, get -> {}}} + + for (const routePath in routes) { + const route = routes[routePath]; + const { spec } = route; + + if (spec.exclude) { + continue; + } + + if (!mapPathToMethods[spec.path]) { + mapPathToMethods[spec.path] = {}; + } + + mapPathToMethods[spec.path][spec.method.toLowerCase()] = {}; + const operationObj = mapPathToMethods[spec.path][spec.method.toLowerCase()]; + // 1) add tags + operationObj.tags = spec.tags; + + // 2) add summary + operationObj.summary = spec.summary; + + // 3) add description + operationObj.description = spec.description; + + // 4) add operationId + operationObj.operationId = spec.name || route.name; + + // 5) add requestBody + const applicationType = spec.applicationType || 'application/json'; + operationObj.requestBody = { + content: { + [applicationType]: { + schema: { + type: 'object', + properties: {} + } + } + }, + required: true + }; + + for (const reqBodyKey in spec.validationObjs?.requestBody) { + const reqBodyKeyData = spec.validationObjs.requestBody[reqBodyKey]; + + parseJoiObject(reqBodyKey, reqBodyKeyData, operationObj.requestBody.content[applicationType].schema.properties); + } + + // 6) add parameters (queryParams + pathParams). + operationObj.parameters = []; + for (const paramKey in spec.validationObjs?.pathParams) { + const paramKeyData = spec.validationObjs.pathParams[paramKey]; + + const obj = {}; + obj.name = paramKey; + obj.in = 'path'; + obj.description = paramKeyData._flags.description; + obj.required = paramKeyData._flags.presence === 'required'; + obj.schema = { type: paramKeyData.type }; + operationObj.parameters.push(obj); + } + + for (const paramKey in spec.validationObjs?.queryParams) { + const paramKeyData = spec.validationObjs.queryParams[paramKey]; + + const obj = {}; + obj.name = paramKey; + obj.in = 'query'; + obj.description = paramKeyData._flags.description; + obj.required = paramKeyData._flags.presence === 'required'; + obj.schema = { type: paramKeyData.type }; + operationObj.parameters.push(obj); + } + + // 7) add responses + const responseType = spec.responseType || 'application/json'; + operationObj.responses = {}; + + for (const resHttpCode in spec.validationObjs?.response) { + const resBodyData = spec.validationObjs.response[resHttpCode]; + + operationObj.responses[resHttpCode] = { + description: resBodyData.description, + content: { + [responseType]: { + schema: {} + } + } + }; + + const obj = operationObj.responses[resHttpCode]; + + parseJoiObject('schema', resBodyData.model, obj.content[responseType]); + } + } + + const components = { components: { schemas: {} } }; + + for (const path in mapPathToMethods) { + // for every path + const pathData = mapPathToMethods[path]; + + for (const httpMethod in pathData) { + // for every http method (post, put, get, delete) + const innerData = pathData[httpMethod]; + + // for every requestBody obj + for (const key in innerData.requestBody.content[Object.keys(innerData.requestBody.content)[0]].schema.properties) { + const reqBodyData = innerData.requestBody.content[Object.keys(innerData.requestBody.content)[0]].schema.properties[key]; + + parseComponetsDecoupled(reqBodyData, components.components.schemas); + replaceWithRefs(reqBodyData); + } + + // for every response object + for (const key in innerData.responses) { + // key here is http method (2xx, 4xx, 5xx) + const obj = innerData.responses[key].content[Object.keys(innerData.responses[key].content)[0]].schema; + parseComponetsDecoupled(obj, components.components.schemas); + replaceWithRefs(obj); + } + } + } + + // refify components that use other components + for (const obj of Object.values(components.components.schemas)) { + replaceWithRefs(obj); + } + + const finalObj = { paths: mapPathToMethods }; + + docs = { ...docs, ...finalObj }; + docs = { ...docs, ...components }; + + docs = { + ...docs, + securitySchemes: { + AccessTokenAuth: { + name: 'X-Access-Token', + type: 'apiKey', + in: 'header', + description: `If authentication is enabled in the WildDuck configuration, you will need to supply an access token in the \`X-Access-Token\` header. + \`\`\`json + { + "X-Access-Token": "59fc66a03e54454869460e45" + } + \`\`\` + ` + } + }, + security: [{ AccessTokenAuth: [] }] + }; + + await fs.promises.writeFile(__dirname + '/../openapidocs.json', JSON.stringify(docs)); } };