From e4c30ca1ea4c4ac8635b5c7b923fd1b59e0075ba Mon Sep 17 00:00:00 2001 From: Nikolai Ovtsinnikov Date: Thu, 28 Sep 2023 12:33:41 +0300 Subject: [PATCH 01/30] api.js added endpoint for generating openapi docs. added new info to one route in mailboxes.js and messages.js files so that the api docs generation can be done at all --- api.js | 96 ++++++++++++++++++++++++++++++++++++++++++++ lib/api/mailboxes.js | 28 ++++++++----- lib/api/messages.js | 34 ++++++++++++---- 3 files changed, 142 insertions(+), 16 deletions(-) diff --git a/api.js b/api.js index 96302417..9f846f48 100644 --- a/api.js +++ b/api.js @@ -573,6 +573,102 @@ module.exports = done => { ); } + server.get( + { path: '/openapi', name: 'openapi-docs-generation' }, + tools.responseWrapper(async (req, res) => { + res.charSet('utf-8'); + + const testRoute = server.router.getRoutes().postusersusermailboxesmailboxmessages; + + console.log(testRoute.spec.pathParams); + console.log(testRoute.spec.requestBody); + console.log(testRoute.spec.queryParams); + + // const docs = ` + // openapi: 3.0.0 + // info: + // title: WildDuck API + // description: WildDuck API docs + // version: 1.0.0 + // contact: + // url: 'https://github.com/nodemailer/wildduck' + + // servers: + // - url: 'https://api.wildduck.email' + + // tags: + // - name: Addresses + // - name: ApplicationPasswords + // - name: Archive + // description: Archive includes all deleted messages. Once messages are old enough then these are permanenetly deleted from the archive as well. Until then you can restore the deleted messages. + // - name: Audit + // description: 'Auditing allows to monitor an email account. All existing, deleted and new emails are copied to the auditing system. See also https://github.com/nodemailer/wildduck-audit-manager' + // - name: Authentication + // - name: Autoreplies + // - name: Certs + // description: WildDuck allows to register TLS certificates to be used with SNI connections. These certificates are used by IMAP, POP3, API and SMTP servers when a SNI capable client establishes a TLS connection. This does not apply for MX servers. + // - name: DKIM + // description: Whenever an email is sent WildDuck checks if there is a DKIM key registered for the domain name of the sender address and uses it to sign the message. + // - name: DomainAccess + // description: Add sender domain names to allowlist (messages are all accepted) or blocklist (messages are sent to Spam folder) + // - name: DomainAliases + // - name: Filters + // - name: Mailboxes + // - name: Messages + // - name: Settings + // - name: Storage + // description: Storage allows easier attachment handling when composing Draft messages. Instead of uploading the attachmnent with every draft update, you store the attachment to the Storage and then link stored file for the Draft. + // - name: Submission + // - name: TwoFactorAuth + // - name: Users + // - name: Webhooks`; + // console.log(docs); + + // console.info(testRoute.spec.pathParams.user._flags, testRoute.spec.pathParams.user._singleRules); + // const isRequired = testRoute.spec.pathParams.user._flags.presence === 'required'; + // const description = testRoute.spec.pathParams.user._flags.description || null; + // const originalType = testRoute.spec.pathParams.user.type; + + /** + * tags: tags + * summary: description + * operationId: name? + * method: spec.method + * url: spec.path + * parameters: if query then in: query, if path then in: path + * name: name of key + * description: joi description + * schema: + * type: joi type + * default: get from joi + * + * requestBody: spec.requestBody + * content: + * application/json: + * schema: + * + * required: + * + * type: joi type + * properties: + * + * + * responses: + * + */ + + // console.log( + // isRequired, + // description, + // originalType, + // testRoute.spec.description, + // testRoute.spec.method.toLowerCase(), + // testRoute.spec.path, + // testRoute.spec.tags + // ); + }) + ); + server.on('error', err => { if (!started) { started = true; diff --git a/lib/api/mailboxes.js b/lib/api/mailboxes.js index 3aed88cc..25ad8967 100644 --- a/lib/api/mailboxes.js +++ b/lib/api/mailboxes.js @@ -246,19 +246,29 @@ module.exports = (db, server, mailboxHandler) => { ); server.post( - '/users/:user/mailboxes', - tools.responseWrapper(async (req, res) => { - res.charSet('utf-8'); - - const schema = Joi.object().keys({ - user: Joi.string().hex().lowercase().length(24).required(), + { + path: '/users/:user/mailboxes', + pathParams: { user: Joi.string().hex().lowercase().length(24).required().description('This is the user path param') }, + description: 'Some description for the API path', + requestBody: { path: Joi.string() .regex(/\/{2,}|\/$/, { invert: true }) .required(), hidden: booleanSchema.default(false), - retention: Joi.number().min(0), - sess: sessSchema, - ip: sessIPSchema + retention: Joi.number().min(0) + }, + queryParams: { sess: sessSchema, ip: sessIPSchema }, + tags: ['Mailboxes'] + }, + tools.responseWrapper(async (req, res) => { + res.charSet('utf-8'); + + const { pathParams, requestBody, queryParams } = req.route.spec; + + const schema = Joi.object({ + ...pathParams, + ...requestBody, + ...queryParams }); const result = schema.validate(req.params, { diff --git a/lib/api/messages.js b/lib/api/messages.js index 03e99c3a..718fe77b 100644 --- a/lib/api/messages.js +++ b/lib/api/messages.js @@ -1529,13 +1529,16 @@ module.exports = (db, server, messageHandler, userHandler, storageHandler, setti ); server.post( - '/users/:user/mailboxes/:mailbox/messages', - tools.responseWrapper(async (req, res) => { - res.charSet('utf-8'); - - const schema = Joi.object().keys({ + { + path: '/users/:user/mailboxes/:mailbox/messages', + pathParams: { user: Joi.string().hex().lowercase().length(24).required(), - mailbox: Joi.string().hex().lowercase().length(24).required(), + mailbox: Joi.string().hex().lowercase().length(24).required() + }, + summary: 'Upload Message', + description: + 'This method allows to upload either an RFC822 formatted message or a message structure to a mailbox. Raw message is stored unmodified, no headers are added or removed. If you want to generate the uploaded message from structured data fields, then do not use the raw property.', + requestBody: { date: Joi.date(), unseen: booleanSchema.default(false), flagged: booleanSchema.default(false), @@ -1629,11 +1632,28 @@ module.exports = (db, server, messageHandler, userHandler, storageHandler, setti bimi: Joi.object().keys({ domain: Joi.string().domain().required(), - selector: Joi.string().empty('').max(255) + selector: Joi.string().empty('').max(255), + bimi: Joi.object().keys({ + domain: Joi.string().domain().required(), + selector: Joi.string().empty('').max(255) + }) }), sess: sessSchema, ip: sessIPSchema + }, + queryParams: {}, + tags: ['Messages'] + }, + tools.responseWrapper(async (req, res) => { + res.charSet('utf-8'); + + const { pathParams, requestBody, queryParams } = req.route.spec; + + const schema = Joi.object({ + ...pathParams, + ...requestBody, + ...queryParams }); if (!req.params.raw && req.body && (Buffer.isBuffer(req.body) || typeof req.body === 'string')) { From d2caad5095132c6ad3b2e5f2483ce9630330e909 Mon Sep 17 00:00:00 2001 From: Nikolai Ovtsinnikov Date: Thu, 28 Sep 2023 12:34:23 +0300 Subject: [PATCH 02/30] try to first generate json representation of the api docs --- api.js | 99 +++++++++++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 94 insertions(+), 5 deletions(-) diff --git a/api.js b/api.js index 9f846f48..8029179f 100644 --- a/api.js +++ b/api.js @@ -580,9 +580,9 @@ module.exports = done => { const testRoute = server.router.getRoutes().postusersusermailboxesmailboxmessages; - console.log(testRoute.spec.pathParams); - console.log(testRoute.spec.requestBody); - console.log(testRoute.spec.queryParams); + // console.log(testRoute.spec.pathParams); + // console.log(testRoute.spec.requestBody.bimi); + // console.log(testRoute.spec.queryParams); // const docs = ` // openapi: 3.0.0 @@ -631,7 +631,8 @@ module.exports = done => { /** * tags: tags - * summary: description + * summary: summary + * descriptiom: description * operationId: name? * method: spec.method * url: spec.path @@ -654,9 +655,97 @@ module.exports = done => { * * * responses: - * + * + * + * + * */ + // const { spec } = testRoute; + // let docStringForPath = ``; + + // // 1) add tags + // docStringForPath += 'tags:\n'; + // for (const tag of spec.tags) { + // docStringForPath += `\t- ${tag}\n`; + // } + // // 2) add summary + // docStringForPath += `summary: ${spec.summary || ''}\n`; + + // // 3) add description + // docStringForPath += `description: ${spec.description || ''}\n`; + + // // 4) add operationId + // docStringForPath += `operationId: ${spec.name || testRoute.name}\n`; + + // // 5) add requestBody + // docStringForPath += `requestBody:\n\tcontent:\n\tapplication/json:\n\tschema:\n\t\n`; + // docStringForPath += 'required: true\n'; + // console.log(docStringForPath); + + const mapPathToMethods = {}; // map -> {post, put, delete, get} + + const { spec } = testRoute; + + if (!mapPathToMethods[spec.path]) { + mapPathToMethods[spec.path] = {}; + } + + mapPathToMethods[spec.path][spec.method.toLowerCase()] = {}; + const methodObj = mapPathToMethods[spec.path][spec.method.toLowerCase()]; + // 1) add tags + methodObj.tags = spec.tags; + + // 2) add summary + methodObj.summary = spec.summary || ''; + + // 3) add description + methodObj.description = spec.description || ''; + + // 4) add operationId + methodObj.operationId = spec.name || testRoute.name; + + // 5) add requestBody, if object use recursion + // if object then fields are in _ids._byKey.get() + methodObj.requestBody = {}; + for (const reqBodyKey in spec.requestBody) { + const reqBodyKeyData = spec.requestBody[reqBodyKey]; + + if (reqBodyKey === 'reference') { + console.log(reqBodyKeyData._ids._byKey.get('attachments')); + } + } + + // 6) add parameters (queryParams + pathParams). TODO: ADD FORMAT key in schema BASED ON FIELD ADDITIONAL RULES IN JOI + methodObj.parameters = {}; + for (const paramKey in spec.pathParams) { + const paramKeyData = spec.pathParams[paramKey]; + + methodObj.parameters[paramKey] = {}; + const obj = methodObj.parameters[paramKey]; + obj.in = 'path'; + obj.description = paramKeyData._flags.description || ''; + obj.required = paramKeyData._flags.presence === 'required'; + obj.schema = { type: paramKeyData.type }; + + // console.log(paramKeyData); + } + + for (const paramKey in spec.queryParams) { + const paramKeyData = spec.pathParams[paramKey]; + + methodObj.parameters[paramKey] = {}; + const obj = methodObj.parameters[paramKey]; + obj.in = 'query'; + obj.description = paramKeyData._flags.description || ''; + obj.required = paramKeyData._flags.presence === 'required'; + obj.schema = { type: paramKeyData.type }; + } + + // 7) add responses + methodObj.responses = {}; + + // console.log(mapPathToMethods['/users/:user/mailboxes/:mailbox/messages'].post.parameters); // console.log( // isRequired, // description, From a69c6adf7f1c6edb433b27d8bdca11c484d19474 Mon Sep 17 00:00:00 2001 From: Nikolai Ovtsinnikov Date: Mon, 2 Oct 2023 18:00:08 +0300 Subject: [PATCH 03/30] add initial Joi Object parsing --- api.js | 131 +++++++++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 127 insertions(+), 4 deletions(-) diff --git a/api.js b/api.js index 8029179f..dfb66035 100644 --- a/api.js +++ b/api.js @@ -707,15 +707,33 @@ module.exports = done => { // 5) add requestBody, if object use recursion // if object then fields are in _ids._byKey.get() - methodObj.requestBody = {}; + methodObj.requestBody = { + content: { + 'application/json': { + schema: { + type: 'object', + properties: {} + } + } + }, + required: true + }; for (const reqBodyKey in spec.requestBody) { const reqBodyKeyData = spec.requestBody[reqBodyKey]; - if (reqBodyKey === 'reference') { - console.log(reqBodyKeyData._ids._byKey.get('attachments')); - } + // if (reqBodyKey === 'reference') { + // console.log(reqBodyKeyData._ids._byKey.get('attachments').schema.$_terms.matches[0].schema); + // } + // if (reqBodyKeyData.type === 'array') { + // console.log(reqBodyKeyData.$_terms.items[0]._ids); + // break; + // } + + parseJoiObject(reqBodyKey, reqBodyKeyData, methodObj.requestBody.content['application/json'].schema.properties); } + console.log(methodObj.requestBody.content['application/json'].schema.properties.reference); + // 6) add parameters (queryParams + pathParams). TODO: ADD FORMAT key in schema BASED ON FIELD ADDITIONAL RULES IN JOI methodObj.parameters = {}; for (const paramKey in spec.pathParams) { @@ -758,6 +776,111 @@ module.exports = done => { }) ); + // TODO: ignore function and symbol types for now + const joiTypeToOpenApiTypeMap = { + any: 'object', + number: 'number', + link: 'string', + boolean: 'boolean', + date: 'string', + string: 'string', + binary: 'string' + }; + + function parseJoiObject(path, joiObject, requestBodyProperties) { + // console.log(path, joiObject, requestBody); + if (joiObject.type === 'object') { + // recursion + const fieldsMap = joiObject._ids._byKey; + + const data = { + type: joiObject.type, + descrption: joiObject._flags.description || 'OBJECT DESCRIPTION', + properties: {}, + required: [] + }; + if (path) { + requestBodyProperties[path] = data; + } else { + requestBodyProperties.push(data); + } + for (const [key, value] of fieldsMap) { + // console.log(key, value); + if (value.schema._flags.presence === 'required') { + data.required.push(key); + } + parseJoiObject(key, value.schema, data.properties); + } + } else if (joiObject.type === 'alternatives') { + const matches = joiObject.$_terms.matches; + + const data = { + oneOf: [], + description: joiObject._flags.description || 'ALTERNATIVES DESCRIPTION' + }; + + if (path) { + requestBodyProperties[path] = data; + } else { + requestBodyProperties.push(data); + } + + // handle alternatives + recursion if needed + + for (const alternative of matches) { + parseJoiObject(null, alternative.schema, data.oneOf); + } + } else if (joiObject.type === 'array') { + const elems = joiObject?.$_terms.items; + + const data = { + type: 'array', + items: [], + description: joiObject._flags.description || 'ARRAY DESCRIPTION' + }; + + if (path) { + requestBodyProperties[path] = data; + } else { + requestBodyProperties.push(data); + } + parseJoiObject(null, elems[0], data.items); + // handle array + } else { + const openApiType = joiTypeToOpenApiTypeMap[joiObject.type]; // even if object here then ignore and do not go recursive + const isRequired = joiObject._flags.presence === 'required'; + const description = joiObject._flags.description || ''; + let format = undefined; + + if (!openApiType) { + throw new Error('Unsupported type! Check API endpoint!'); + } + + if (joiObject.type !== openApiType) { + // type has changed, so probably string, acquire format + format = joiObject.type; + } + // TODO: if type before and after is string, add additional checks + + const data = { type: openApiType, description, format, required: isRequired }; + if (path) { + requestBodyProperties[path] = data; + } else { + // no path given, expect requestBodyProperties to be an array to append to + requestBodyProperties.push(data); + } + + // console.log(openApiType, isRequired, description, format); + + // all other types + // 1) get type + // 2) get if required + // 3) if not in openapi types -> check if can be string -> openapi type: string + forrmat: check format + // if string check if it has additional check, e.g hex string, base64 etc, email, etc. + // 4) update requestBody at the path + } + } + server.on('error', err => { if (!started) { started = true; From e7a3c722508ae3441f9ff0d3c983bc88571d68c8 Mon Sep 17 00:00:00 2001 From: Nikolai Ovtsinnikov Date: Tue, 3 Oct 2023 14:37:06 +0300 Subject: [PATCH 04/30] api.js make generation dynamic. messages.js add schemas from separate file. messages-schemas.js used for messages endpoint schemas --- api.js | 176 ++++++++++++------------ lib/api/messages.js | 77 ++--------- lib/schemas/request/messages-schemas.js | 60 ++++++++ 3 files changed, 162 insertions(+), 151 deletions(-) create mode 100644 lib/schemas/request/messages-schemas.js diff --git a/api.js b/api.js index dfb66035..8e42370a 100644 --- a/api.js +++ b/api.js @@ -578,8 +578,6 @@ module.exports = done => { tools.responseWrapper(async (req, res) => { res.charSet('utf-8'); - const testRoute = server.router.getRoutes().postusersusermailboxesmailboxmessages; - // console.log(testRoute.spec.pathParams); // console.log(testRoute.spec.requestBody.bimi); // console.log(testRoute.spec.queryParams); @@ -622,6 +620,22 @@ module.exports = done => { // - name: TwoFactorAuth // - name: Users // - name: Webhooks`; + + // securitySchemes: + // AccessTokenAuth: + // name: X-Access-Token + // type: apiKey + // in: header + // description: |- + // If authentication is enabled in the WildDuck configuration, you will need to supply an access token in the `X-Access-Token` header. + + // ```json + // { + // "X-Access-Token": "59fc66a03e54454869460e45" + // } + // ``` + // security: + // - AccessTokenAuth: [] // console.log(docs); // console.info(testRoute.spec.pathParams.user._flags, testRoute.spec.pathParams.user._singleRules); @@ -685,94 +699,82 @@ module.exports = done => { const mapPathToMethods = {}; // map -> {post, put, delete, get} - const { spec } = testRoute; + // const testRoute = server.router.getRoutes().postusersusermailboxesmailboxmessages; - if (!mapPathToMethods[spec.path]) { - mapPathToMethods[spec.path] = {}; - } + const routes = server.router.getRoutes(); + for (const routePath in routes) { + const route = routes[routePath]; + const { spec } = route; - mapPathToMethods[spec.path][spec.method.toLowerCase()] = {}; - const methodObj = mapPathToMethods[spec.path][spec.method.toLowerCase()]; - // 1) add tags - methodObj.tags = spec.tags; - - // 2) add summary - methodObj.summary = spec.summary || ''; + if (!mapPathToMethods[spec.path]) { + mapPathToMethods[spec.path] = {}; + } - // 3) add description - methodObj.description = spec.description || ''; + mapPathToMethods[spec.path][spec.method.toLowerCase()] = {}; + const methodObj = mapPathToMethods[spec.path][spec.method.toLowerCase()]; + // 1) add tags + methodObj.tags = spec.tags; + + // 2) add summary + methodObj.summary = spec.summary || ''; + + // 3) add description + methodObj.description = spec.description || ''; + + // 4) add operationId + methodObj.operationId = spec.name || route.name; + + // 5) add requestBody, if object use recursion + // if object then fields are in _ids._byKey.get() + methodObj.requestBody = { + content: { + 'application/json': { + schema: { + type: 'object', + properties: {} + } + } + }, + required: true + }; + for (const reqBodyKey in spec.requestBody) { + const reqBodyKeyData = spec.requestBody[reqBodyKey]; - // 4) add operationId - methodObj.operationId = spec.name || testRoute.name; + parseJoiObject(reqBodyKey, reqBodyKeyData, methodObj.requestBody.content['application/json'].schema.properties); + } - // 5) add requestBody, if object use recursion - // if object then fields are in _ids._byKey.get() - methodObj.requestBody = { - content: { - 'application/json': { - schema: { - type: 'object', - properties: {} - } - } - }, - required: true - }; - for (const reqBodyKey in spec.requestBody) { - const reqBodyKeyData = spec.requestBody[reqBodyKey]; - - // if (reqBodyKey === 'reference') { - // console.log(reqBodyKeyData._ids._byKey.get('attachments').schema.$_terms.matches[0].schema); - // } - // if (reqBodyKeyData.type === 'array') { - // console.log(reqBodyKeyData.$_terms.items[0]._ids); - // break; - // } - - parseJoiObject(reqBodyKey, reqBodyKeyData, methodObj.requestBody.content['application/json'].schema.properties); - } + console.log(methodObj.requestBody.content['application/json'].schema /*.properties.reference*/); - console.log(methodObj.requestBody.content['application/json'].schema.properties.reference); + // 6) add parameters (queryParams + pathParams). + // TODO: ADD FORMAT key in schema BASED ON FIELD ADDITIONAL RULES IN JOI + methodObj.parameters = {}; + for (const paramKey in spec.pathParams) { + const paramKeyData = spec.pathParams[paramKey]; - // 6) add parameters (queryParams + pathParams). TODO: ADD FORMAT key in schema BASED ON FIELD ADDITIONAL RULES IN JOI - methodObj.parameters = {}; - for (const paramKey in spec.pathParams) { - const paramKeyData = spec.pathParams[paramKey]; + methodObj.parameters[paramKey] = {}; + const obj = methodObj.parameters[paramKey]; + obj.in = 'path'; + obj.description = paramKeyData._flags.description || ''; + obj.required = paramKeyData._flags.presence === 'required'; + obj.schema = { type: paramKeyData.type }; - methodObj.parameters[paramKey] = {}; - const obj = methodObj.parameters[paramKey]; - obj.in = 'path'; - obj.description = paramKeyData._flags.description || ''; - obj.required = paramKeyData._flags.presence === 'required'; - obj.schema = { type: paramKeyData.type }; + // console.log(paramKeyData); + } - // console.log(paramKeyData); - } + for (const paramKey in spec.queryParams) { + const paramKeyData = spec.queryParams[paramKey]; - for (const paramKey in spec.queryParams) { - const paramKeyData = spec.pathParams[paramKey]; + methodObj.parameters[paramKey] = {}; + const obj = methodObj.parameters[paramKey]; + obj.in = 'query'; + obj.description = paramKeyData._flags.description || ''; + obj.required = paramKeyData._flags.presence === 'required'; + obj.schema = { type: paramKeyData.type }; + } - methodObj.parameters[paramKey] = {}; - const obj = methodObj.parameters[paramKey]; - obj.in = 'query'; - obj.description = paramKeyData._flags.description || ''; - obj.required = paramKeyData._flags.presence === 'required'; - obj.schema = { type: paramKeyData.type }; + // 7) add responses + methodObj.responses = {}; } - - // 7) add responses - methodObj.responses = {}; - - // console.log(mapPathToMethods['/users/:user/mailboxes/:mailbox/messages'].post.parameters); - // console.log( - // isRequired, - // description, - // originalType, - // testRoute.spec.description, - // testRoute.spec.method.toLowerCase(), - // testRoute.spec.path, - // testRoute.spec.tags - // ); }) ); @@ -795,7 +797,7 @@ module.exports = done => { const data = { type: joiObject.type, - descrption: joiObject._flags.description || 'OBJECT DESCRIPTION', + descrption: joiObject._flags.description || '', properties: {}, required: [] }; @@ -805,7 +807,6 @@ module.exports = done => { requestBodyProperties.push(data); } for (const [key, value] of fieldsMap) { - // console.log(key, value); if (value.schema._flags.presence === 'required') { data.required.push(key); } @@ -816,7 +817,7 @@ module.exports = done => { const data = { oneOf: [], - description: joiObject._flags.description || 'ALTERNATIVES DESCRIPTION' + description: joiObject._flags.description || '' }; if (path) { @@ -836,7 +837,7 @@ module.exports = done => { const data = { type: 'array', items: [], - description: joiObject._flags.description || 'ARRAY DESCRIPTION' + description: joiObject._flags.description || '' }; if (path) { @@ -862,7 +863,14 @@ module.exports = done => { } // TODO: if type before and after is string, add additional checks - const data = { type: openApiType, description, format, required: isRequired }; + if (openApiType === 'string') { + console.log(joiObject._rules); + } + + const data = { type: openApiType, description, required: isRequired }; + if (format) { + data.format = format; + } if (path) { requestBodyProperties[path] = data; } else { @@ -870,8 +878,6 @@ module.exports = done => { requestBodyProperties.push(data); } - // console.log(openApiType, isRequired, description, format); - // all other types // 1) get type // 2) get if required diff --git a/lib/api/messages.js b/lib/api/messages.js index 718fe77b..d6204035 100644 --- a/lib/api/messages.js +++ b/lib/api/messages.js @@ -25,6 +25,7 @@ const { getMongoDBQuery /*, getElasticSearchQuery*/ } = require('../search-query //const { getClient } = require('../elasticsearch'); const BimiHandler = require('../bimi-handler'); +const { Address, AddressOptionalName, Header, Attachment, ReferenceWithAttachments, Bimi } = require('../schemas/request/messages-schemas'); module.exports = (db, server, messageHandler, userHandler, storageHandler, settingsHandler) => { let maildrop = new Maildropper({ @@ -1546,45 +1547,17 @@ module.exports = (db, server, messageHandler, userHandler, storageHandler, setti raw: Joi.binary().max(consts.MAX_ALLOWED_MESSAGE_SIZE).empty(''), - from: Joi.object().keys({ - name: Joi.string().empty('').max(255), - address: Joi.string().email({ tlds: false }).required() - }), + from: Address, - replyTo: Joi.object().keys({ - name: Joi.string().empty('').max(255), - address: Joi.string().email({ tlds: false }).required() - }), + replyTo: Address, - to: Joi.array().items( - Joi.object().keys({ - name: Joi.string().empty('').max(255), - address: Joi.string().email({ tlds: false }).required() - }) - ), + to: AddressOptionalName, - cc: Joi.array().items( - Joi.object().keys({ - name: Joi.string().empty('').max(255), - address: Joi.string().email({ tlds: false }).required() - }) - ), + cc: AddressOptionalName, - bcc: Joi.array().items( - Joi.object().keys({ - name: Joi.string().empty('').max(255), - address: Joi.string().email({ tlds: false }).required() - }) - ), - - headers: Joi.array().items( - Joi.object().keys({ - key: Joi.string().empty('').max(255), - value: Joi.string() - .empty('') - .max(100 * 1024) - }) - ), + bcc: AddressOptionalName, + + headers: Joi.array().items(Header), subject: Joi.string() .empty('') @@ -1598,46 +1571,18 @@ module.exports = (db, server, messageHandler, userHandler, storageHandler, setti files: Joi.array().items(Joi.string().hex().lowercase().length(24)), - attachments: Joi.array().items( - Joi.object().keys({ - filename: Joi.string().empty('').max(255), - contentType: Joi.string().empty('').max(255), - encoding: Joi.string().empty('').default('base64'), - contentTransferEncoding: Joi.string().empty(''), - content: Joi.string().required(), - cid: Joi.string().empty('').max(255) - }) - ), + attachments: Joi.array().items(Attachment), metaData: metaDataSchema.label('metaData'), - reference: Joi.object().keys({ - mailbox: Joi.string().hex().lowercase().length(24).required(), - id: Joi.number().required(), - action: Joi.string().valid('reply', 'replyAll', 'forward').required(), - attachments: Joi.alternatives().try( - booleanSchema, - Joi.array().items( - Joi.string() - .regex(/^ATT\d+$/i) - .uppercase() - ) - ) - }), + reference: ReferenceWithAttachments, replacePrevious: Joi.object({ mailbox: Joi.string().hex().lowercase().length(24), id: Joi.number().required() }), - bimi: Joi.object().keys({ - domain: Joi.string().domain().required(), - selector: Joi.string().empty('').max(255), - bimi: Joi.object().keys({ - domain: Joi.string().domain().required(), - selector: Joi.string().empty('').max(255) - }) - }), + bimi: Bimi, sess: sessSchema, ip: sessIPSchema diff --git a/lib/schemas/request/messages-schemas.js b/lib/schemas/request/messages-schemas.js new file mode 100644 index 00000000..9a948613 --- /dev/null +++ b/lib/schemas/request/messages-schemas.js @@ -0,0 +1,60 @@ +'use strict'; + +const Joi = require('joi'); +const { booleanSchema } = require('../../schemas'); + +const Address = Joi.object({ + name: Joi.string().empty('').max(255), + address: Joi.string().email({ tlds: false }).required() +}); + +const AddressOptionalName = Joi.array().items( + Joi.object({ + name: Joi.string().empty('').max(255), + address: Joi.string().email({ tlds: false }).required() + }) +); + +const Header = Joi.object({ + key: Joi.string().empty('').max(255), + value: Joi.string() + .empty('') + .max(100 * 1024) +}); + +const Attachment = Joi.object({ + filename: Joi.string().empty('').max(255), + contentType: Joi.string().empty('').max(255), + encoding: Joi.string().empty('').default('base64'), + contentTransferEncoding: Joi.string().empty(''), + content: Joi.string().required(), + cid: Joi.string().empty('').max(255) +}); + +const ReferenceWithAttachments = Joi.object({ + mailbox: Joi.string().hex().lowercase().length(24).required(), + id: Joi.number().required(), + action: Joi.string().valid('reply', 'replyAll', 'forward').required(), + attachments: Joi.alternatives().try( + booleanSchema, + Joi.array().items( + Joi.string() + .regex(/^ATT\d+$/i) + .uppercase() + ) + ) +}); + +const Bimi = Joi.object({ + domain: Joi.string().domain().required(), + selector: Joi.string().empty('').max(255) +}); + +module.exports = { + Address, + AddressOptionalName, + Header, + Attachment, + ReferenceWithAttachments, + Bimi +}; From f3fe6cbeac63a9e3ad5a68b462df0919945e708f Mon Sep 17 00:00:00 2001 From: Nikolai Ovtsinnikov Date: Wed, 4 Oct 2023 19:18:47 +0300 Subject: [PATCH 05/30] add additions to schemas. Add new schemas to messages.js and also add response object there. Add response object parsing functionality to api.js --- api.js | 28 ++++++++++++++++++------- lib/api/messages.js | 21 ++++++++++++++----- lib/schemas/request/messages-schemas.js | 24 ++++++++++----------- 3 files changed, 49 insertions(+), 24 deletions(-) diff --git a/api.js b/api.js index 8e42370a..365ec88f 100644 --- a/api.js +++ b/api.js @@ -737,6 +737,7 @@ module.exports = done => { }, required: true }; + for (const reqBodyKey in spec.requestBody) { const reqBodyKeyData = spec.requestBody[reqBodyKey]; @@ -757,8 +758,6 @@ module.exports = done => { obj.description = paramKeyData._flags.description || ''; obj.required = paramKeyData._flags.presence === 'required'; obj.schema = { type: paramKeyData.type }; - - // console.log(paramKeyData); } for (const paramKey in spec.queryParams) { @@ -774,6 +773,14 @@ module.exports = done => { // 7) add responses methodObj.responses = {}; + + for (const resHttpCode in spec.response) { + const restBodyData = spec.response[resHttpCode]; + + parseJoiObject(resHttpCode, restBodyData, methodObj.responses); + } + + console.log(methodObj.responses); } }) ); @@ -789,18 +796,25 @@ module.exports = done => { binary: 'string' }; + /** + * Parse Joi Objects + */ function parseJoiObject(path, joiObject, requestBodyProperties) { - // console.log(path, joiObject, requestBody); if (joiObject.type === 'object') { // recursion const fieldsMap = joiObject._ids._byKey; const data = { type: joiObject.type, - descrption: joiObject._flags.description || '', + description: joiObject._flags.description || '', properties: {}, required: [] }; + + if (joiObject._flags.objectName) { + data.objectName = joiObject._flags.objectName; + } + if (path) { requestBodyProperties[path] = data; } else { @@ -863,9 +877,9 @@ module.exports = done => { } // TODO: if type before and after is string, add additional checks - if (openApiType === 'string') { - console.log(joiObject._rules); - } + // if (openApiType === 'string') { + // console.log(joiObject._rules); + // } const data = { type: openApiType, description, required: isRequired }; if (format) { diff --git a/lib/api/messages.js b/lib/api/messages.js index d6204035..466adfd2 100644 --- a/lib/api/messages.js +++ b/lib/api/messages.js @@ -25,7 +25,7 @@ const { getMongoDBQuery /*, getElasticSearchQuery*/ } = require('../search-query //const { getClient } = require('../elasticsearch'); const BimiHandler = require('../bimi-handler'); -const { Address, AddressOptionalName, Header, Attachment, ReferenceWithAttachments, Bimi } = require('../schemas/request/messages-schemas'); +const { Address, AddressOptionalNameArray, Header, Attachment, ReferenceWithAttachments, Bimi } = require('../schemas/request/messages-schemas'); module.exports = (db, server, messageHandler, userHandler, storageHandler, settingsHandler) => { let maildrop = new Maildropper({ @@ -1551,11 +1551,11 @@ module.exports = (db, server, messageHandler, userHandler, storageHandler, setti replyTo: Address, - to: AddressOptionalName, + to: AddressOptionalNameArray, - cc: AddressOptionalName, + cc: AddressOptionalNameArray, - bcc: AddressOptionalName, + bcc: AddressOptionalNameArray, headers: Joi.array().items(Header), @@ -1588,7 +1588,18 @@ module.exports = (db, server, messageHandler, userHandler, storageHandler, setti ip: sessIPSchema }, queryParams: {}, - tags: ['Messages'] + tags: ['Messages'], + response: { + 200: Joi.object({ + success: Joi.boolean(), + message: Joi.object({ + id: Joi.number(), + malbox: Joi.string(), + size: Joi.number() + }), + previousDeleted: Joi.boolean() + }) + } }, tools.responseWrapper(async (req, res) => { res.charSet('utf-8'); diff --git a/lib/schemas/request/messages-schemas.js b/lib/schemas/request/messages-schemas.js index 9a948613..d4226e60 100644 --- a/lib/schemas/request/messages-schemas.js +++ b/lib/schemas/request/messages-schemas.js @@ -6,21 +6,21 @@ const { booleanSchema } = require('../../schemas'); const Address = Joi.object({ name: Joi.string().empty('').max(255), address: Joi.string().email({ tlds: false }).required() -}); +}).$_setFlag('objectName', 'Address'); -const AddressOptionalName = Joi.array().items( - Joi.object({ - name: Joi.string().empty('').max(255), - address: Joi.string().email({ tlds: false }).required() - }) -); +const AddressOptionalName = Joi.object({ + name: Joi.string().empty('').max(255), + address: Joi.string().email({ tlds: false }).required() +}).$_setFlag('objectName', 'AddressOptionalName'); + +const AddressOptionalNameArray = Joi.array().items(AddressOptionalName); const Header = Joi.object({ key: Joi.string().empty('').max(255), value: Joi.string() .empty('') .max(100 * 1024) -}); +}).$_setFlag('objectName', 'Header'); const Attachment = Joi.object({ filename: Joi.string().empty('').max(255), @@ -29,7 +29,7 @@ const Attachment = Joi.object({ contentTransferEncoding: Joi.string().empty(''), content: Joi.string().required(), cid: Joi.string().empty('').max(255) -}); +}).$_setFlag('objectName', 'Attachment'); const ReferenceWithAttachments = Joi.object({ mailbox: Joi.string().hex().lowercase().length(24).required(), @@ -43,16 +43,16 @@ const ReferenceWithAttachments = Joi.object({ .uppercase() ) ) -}); +}).$_setFlag('objectName', 'ReferenceWithAttachments'); const Bimi = Joi.object({ domain: Joi.string().domain().required(), selector: Joi.string().empty('').max(255) -}); +}).$_setFlag('objectName', 'Bimi'); module.exports = { Address, - AddressOptionalName, + AddressOptionalNameArray, Header, Attachment, ReferenceWithAttachments, From cccd2c0a977de0bd2065f89cdd58352674355351 Mon Sep 17 00:00:00 2001 From: Nikolai Ovtsinnikov Date: Fri, 6 Oct 2023 13:45:13 +0300 Subject: [PATCH 06/30] add initial openapi doc yml file generation --- api.js | 161 ++++++++++++++++++++++++++++++++++++--------------------- 1 file changed, 101 insertions(+), 60 deletions(-) diff --git a/api.js b/api.js index 365ec88f..7622722f 100644 --- a/api.js +++ b/api.js @@ -48,6 +48,7 @@ const certsRoutes = require('./lib/api/certs'); const webhooksRoutes = require('./lib/api/webhooks'); const settingsRoutes = require('./lib/api/settings'); const { SettingsHandler } = require('./lib/settings-handler'); +const fs = require('fs'); let userHandler; let mailboxHandler; @@ -578,64 +579,44 @@ module.exports = done => { tools.responseWrapper(async (req, res) => { res.charSet('utf-8'); - // console.log(testRoute.spec.pathParams); - // console.log(testRoute.spec.requestBody.bimi); - // console.log(testRoute.spec.queryParams); - - // const docs = ` - // openapi: 3.0.0 - // info: - // title: WildDuck API - // description: WildDuck API docs - // version: 1.0.0 - // contact: - // url: 'https://github.com/nodemailer/wildduck' - - // servers: - // - url: 'https://api.wildduck.email' - - // tags: - // - name: Addresses - // - name: ApplicationPasswords - // - name: Archive - // description: Archive includes all deleted messages. Once messages are old enough then these are permanenetly deleted from the archive as well. Until then you can restore the deleted messages. - // - name: Audit - // description: 'Auditing allows to monitor an email account. All existing, deleted and new emails are copied to the auditing system. See also https://github.com/nodemailer/wildduck-audit-manager' - // - name: Authentication - // - name: Autoreplies - // - name: Certs - // description: WildDuck allows to register TLS certificates to be used with SNI connections. These certificates are used by IMAP, POP3, API and SMTP servers when a SNI capable client establishes a TLS connection. This does not apply for MX servers. - // - name: DKIM - // description: Whenever an email is sent WildDuck checks if there is a DKIM key registered for the domain name of the sender address and uses it to sign the message. - // - name: DomainAccess - // description: Add sender domain names to allowlist (messages are all accepted) or blocklist (messages are sent to Spam folder) - // - name: DomainAliases - // - name: Filters - // - name: Mailboxes - // - name: Messages - // - name: Settings - // - name: Storage - // description: Storage allows easier attachment handling when composing Draft messages. Instead of uploading the attachmnent with every draft update, you store the attachment to the Storage and then link stored file for the Draft. - // - name: Submission - // - name: TwoFactorAuth - // - name: Users - // - name: Webhooks`; - - // securitySchemes: - // AccessTokenAuth: - // name: X-Access-Token - // type: apiKey - // in: header - // description: |- - // If authentication is enabled in the WildDuck configuration, you will need to supply an access token in the `X-Access-Token` header. - - // ```json - // { - // "X-Access-Token": "59fc66a03e54454869460e45" - // } - // ``` - // security: - // - AccessTokenAuth: [] + let docs = ` +openapi: 3.0.0 +info: + title: WildDuck API + description: WildDuck API docs + version: 1.0.0 + contact: + url: 'https://github.com/nodemailer/wildduck' + +servers: + - url: 'https://api.wildduck.email' + +tags: + - name: Addresses + - name: ApplicationPasswords + - name: Archive + description: Archive includes all deleted messages. Once messages are old enough then these are permanenetly deleted from the archive as well. Until then you can restore the deleted messages. + - name: Audit + description: 'Auditing allows to monitor an email account. All existing, deleted and new emails are copied to the auditing system. See also https://github.com/nodemailer/wildduck-audit-manager' + - name: Authentication + - name: Autoreplies + - name: Certs + description: WildDuck allows to register TLS certificates to be used with SNI connections. These certificates are used by IMAP, POP3, API and SMTP servers when a SNI capable client establishes a TLS connection. This does not apply for MX servers. + - name: DKIM + description: Whenever an email is sent WildDuck checks if there is a DKIM key registered for the domain name of the sender address and uses it to sign the message. + - name: DomainAccess + description: Add sender domain names to allowlist (messages are all accepted) or blocklist (messages are sent to Spam folder) + - name: DomainAliases + - name: Filters + - name: Mailboxes + - name: Messages + - name: Settings + - name: Storage + description: Storage allows easier attachment handling when composing Draft messages. Instead of uploading the attachmnent with every draft update, you store the attachment to the Storage and then link stored file for the Draft. + - name: Submission + - name: TwoFactorAuth + - name: Users + - name: Webhooks`; // console.log(docs); // console.info(testRoute.spec.pathParams.user._flags, testRoute.spec.pathParams.user._singleRules); @@ -744,7 +725,7 @@ module.exports = done => { parseJoiObject(reqBodyKey, reqBodyKeyData, methodObj.requestBody.content['application/json'].schema.properties); } - console.log(methodObj.requestBody.content['application/json'].schema /*.properties.reference*/); + // console.log(methodObj.requestBody.content['application/json'].schema /*.properties.reference*/); // 6) add parameters (queryParams + pathParams). // TODO: ADD FORMAT key in schema BASED ON FIELD ADDITIONAL RULES IN JOI @@ -780,8 +761,68 @@ module.exports = done => { parseJoiObject(resHttpCode, restBodyData, methodObj.responses); } - console.log(methodObj.responses); + // console.log(methodObj.responses); } + + docs += `\npaths:\n`; + + let tabLevel; + const tab = '\t\t'; + for (const path in mapPathToMethods) { + tabLevel = 1; + + const data = mapPathToMethods[path]; + + docs += `${tab.repeat(tabLevel)}'${path}':\n`; + + for (const httpMethod in data) { + tabLevel = 2; + docs += `${tab.repeat(tabLevel)}${httpMethod}:\n`; + const innerData = data[httpMethod]; + tabLevel = 3; + + const { tags, summary, description, operationId, requestBody, parameters, responses } = innerData; + docs += `${tab.repeat(tabLevel)}tags:\n`; + for (const tag of tags || []) { + tabLevel = 4; + docs += `${tab.repeat(tabLevel)}- ${tag}\n`; + } + tabLevel = 3; + docs += `${tab.repeat(tabLevel)}summary: ${summary}\n`; + + docs += `${tab.repeat(tabLevel)}operationId: ${operationId}\n`; + + docs += `${tab.repeat(tabLevel)}description: ${description}\n`; + + docs += `${tab.repeat(tabLevel)}requestBody:\n`; + + docs += `${tab.repeat(tabLevel)}parameters:\n`; + + docs += `${tab.repeat(tabLevel)}responses:\n`; + console.log(requestBody, parameters, responses); + } + } + + docs += ` + securitySchemes: + AccessTokenAuth: + name: X-Access-Token + type: apiKey + in: header + description: |- + If authentication is enabled in the WildDuck configuration, you will need to supply an access token in the \`X-Access-Token\` header. + + \`\`\`json + { + "X-Access-Token": "59fc66a03e54454869460e45" + } + \`\`\` +security: + - AccessTokenAuth: [] +`; + + console.log(__dirname); + await fs.promises.writeFile(__dirname + '/openapidocs.yml', docs); }) ); From c58a969dfe07276b05a1aa80bbc6508b173df6f4 Mon Sep 17 00:00:00 2001 From: Nikolai Ovtsinnikov Date: Tue, 10 Oct 2023 10:33:11 +0300 Subject: [PATCH 07/30] remove manual yaml parsing with js-yaml JSON -> YAML parsing --- api.js | 214 +++++++++++++++++++++++++++------------------------------ 1 file changed, 103 insertions(+), 111 deletions(-) diff --git a/api.js b/api.js index 7622722f..ff0bde8c 100644 --- a/api.js +++ b/api.js @@ -49,6 +49,7 @@ const webhooksRoutes = require('./lib/api/webhooks'); const settingsRoutes = require('./lib/api/settings'); const { SettingsHandler } = require('./lib/settings-handler'); const fs = require('fs'); +const yaml = require('js-yaml'); let userHandler; let mailboxHandler; @@ -616,77 +617,18 @@ tags: - name: Submission - name: TwoFactorAuth - name: Users - - name: Webhooks`; - // console.log(docs); - - // console.info(testRoute.spec.pathParams.user._flags, testRoute.spec.pathParams.user._singleRules); - // const isRequired = testRoute.spec.pathParams.user._flags.presence === 'required'; - // const description = testRoute.spec.pathParams.user._flags.description || null; - // const originalType = testRoute.spec.pathParams.user.type; - - /** - * tags: tags - * summary: summary - * descriptiom: description - * operationId: name? - * method: spec.method - * url: spec.path - * parameters: if query then in: query, if path then in: path - * name: name of key - * description: joi description - * schema: - * type: joi type - * default: get from joi - * - * requestBody: spec.requestBody - * content: - * application/json: - * schema: - * - * required: - * - * type: joi type - * properties: - * - * - * responses: - * - * - * - * - */ - - // const { spec } = testRoute; - // let docStringForPath = ``; - - // // 1) add tags - // docStringForPath += 'tags:\n'; - // for (const tag of spec.tags) { - // docStringForPath += `\t- ${tag}\n`; - // } - // // 2) add summary - // docStringForPath += `summary: ${spec.summary || ''}\n`; - - // // 3) add description - // docStringForPath += `description: ${spec.description || ''}\n`; - - // // 4) add operationId - // docStringForPath += `operationId: ${spec.name || testRoute.name}\n`; - - // // 5) add requestBody - // docStringForPath += `requestBody:\n\tcontent:\n\tapplication/json:\n\tschema:\n\t\n`; - // docStringForPath += 'required: true\n'; - // console.log(docStringForPath); - - const mapPathToMethods = {}; // map -> {post, put, delete, get} - - // const testRoute = server.router.getRoutes().postusersusermailboxesmailboxmessages; + - name: Webhooks\n`; + const mapPathToMethods = {}; // map -> {path -> {post, put, delete, get}} const routes = server.router.getRoutes(); for (const routePath in routes) { const route = routes[routePath]; const { spec } = route; + if (!spec.include) { + continue; + } + if (!mapPathToMethods[spec.path]) { mapPathToMethods[spec.path] = {}; } @@ -706,10 +648,10 @@ tags: methodObj.operationId = spec.name || route.name; // 5) add requestBody, if object use recursion - // if object then fields are in _ids._byKey.get() + const applicationType = spec.applicationType || 'application/json'; methodObj.requestBody = { content: { - 'application/json': { + [applicationType]: { schema: { type: 'object', properties: {} @@ -722,11 +664,9 @@ tags: for (const reqBodyKey in spec.requestBody) { const reqBodyKeyData = spec.requestBody[reqBodyKey]; - parseJoiObject(reqBodyKey, reqBodyKeyData, methodObj.requestBody.content['application/json'].schema.properties); + parseJoiObject(reqBodyKey, reqBodyKeyData, methodObj.requestBody.content[applicationType].schema.properties); } - // console.log(methodObj.requestBody.content['application/json'].schema /*.properties.reference*/); - // 6) add parameters (queryParams + pathParams). // TODO: ADD FORMAT key in schema BASED ON FIELD ADDITIONAL RULES IN JOI methodObj.parameters = {}; @@ -764,44 +704,34 @@ tags: // console.log(methodObj.responses); } - docs += `\npaths:\n`; + const components = { components: { schemas: {} } }; - let tabLevel; - const tab = '\t\t'; for (const path in mapPathToMethods) { - tabLevel = 1; + const pathData = mapPathToMethods[path]; - const data = mapPathToMethods[path]; + for (const httpMethod in pathData) { + const innerData = pathData[httpMethod]; - docs += `${tab.repeat(tabLevel)}'${path}':\n`; + for (const key in innerData.requestBody.content[Object.keys(innerData.requestBody.content)[0]].schema.properties) { + const reqBodyData = innerData.requestBody.content[Object.keys(innerData.requestBody.content)[0]].schema.properties[key]; - for (const httpMethod in data) { - tabLevel = 2; - docs += `${tab.repeat(tabLevel)}${httpMethod}:\n`; - const innerData = data[httpMethod]; - tabLevel = 3; - - const { tags, summary, description, operationId, requestBody, parameters, responses } = innerData; - docs += `${tab.repeat(tabLevel)}tags:\n`; - for (const tag of tags || []) { - tabLevel = 4; - docs += `${tab.repeat(tabLevel)}- ${tag}\n`; + parseComponetsDecoupled(reqBodyData, components.components.schemas); + replaceWithRefs(reqBodyData); + console.log(reqBodyData); } - tabLevel = 3; - docs += `${tab.repeat(tabLevel)}summary: ${summary}\n`; - - docs += `${tab.repeat(tabLevel)}operationId: ${operationId}\n`; + } + } - docs += `${tab.repeat(tabLevel)}description: ${description}\n`; + // console.log(components.components.schemas); + // console.log(yaml.dump(components)); - docs += `${tab.repeat(tabLevel)}requestBody:\n`; + const finalObj = { paths: mapPathToMethods }; - docs += `${tab.repeat(tabLevel)}parameters:\n`; + const mapPathToMethodsYaml = yaml.dump(finalObj, { indent: 4, lineWidth: -1 }); + const componentsYaml = yaml.dump(components, { indent: 4, lineWidth: -1 }); - docs += `${tab.repeat(tabLevel)}responses:\n`; - console.log(requestBody, parameters, responses); - } - } + docs += mapPathToMethodsYaml; + docs += componentsYaml; docs += ` securitySchemes: @@ -821,7 +751,6 @@ security: - AccessTokenAuth: [] `; - console.log(__dirname); await fs.promises.writeFile(__dirname + '/openapidocs.yml', docs); }) ); @@ -837,12 +766,86 @@ security: binary: 'string' }; + function replaceWithRefs(reqBodyData) { + if (reqBodyData.type === 'array') { + const obj = reqBodyData.items[0]; + + replaceWithRefs(obj); + } else if (reqBodyData.type === 'object') { + if (reqBodyData.objectName) { + const objectName = reqBodyData.objectName; + Object.keys(reqBodyData).forEach(key => { + if (key !== '$ref') { + delete reqBodyData[key]; + } + }); + reqBodyData.$ref = `#/components/schemas/${objectName}`; + } else { + for (const key in reqBodyData.properties) { + replaceWithRefs(reqBodyData.properties[key]); + } + } + } else if (reqBodyData.type === 'alternatives') { + for (const obj in reqBodyData.oneOf) { + replaceWithRefs(obj); + } + } + } + + function parseComponetsDecoupled(component, components, level = 0) { + if (component.type === 'array') { + const obj = { ...component.items[0] }; // copy + + if (obj.objectName) { + for (const key in obj.properties) { + parseComponetsDecoupled(obj.properties[key], components, level + 1); + } + + const objectName = obj.objectName; + components[objectName] = obj; + delete components[objectName].objectName; + + if (level > 0) { + Object.keys(obj).forEach(key => { + if (key !== '$ref') { + delete obj[key]; + } + }); + obj.$ref = `#/components/schemas/${objectName}`; + } + } + } else if (component.type === 'object' && component.objectName) { + const obj = { ...component }; // copy + const objectName = component.objectName; + + for (const key in obj.properties) { + parseComponetsDecoupled(obj.properties[key], components, level + 1); + } + + components[objectName] = obj; + delete components[objectName].objectName; + + if (level > 0) { + Object.keys(obj).forEach(key => { + if (key !== '$ref') { + delete obj[key]; + } + }); + obj.$ref = `#/components/schemas/${objectName}`; + } + } else if (component.oneOf) { + // alternatives + for (const obj in component.oneOf) { + parseComponetsDecoupled({ ...obj }, components, level); + } + } + } + /** * Parse Joi Objects */ function parseJoiObject(path, joiObject, requestBodyProperties) { if (joiObject.type === 'object') { - // recursion const fieldsMap = joiObject._ids._byKey; const data = { @@ -881,8 +884,6 @@ security: requestBodyProperties.push(data); } - // handle alternatives + recursion if needed - for (const alternative of matches) { parseJoiObject(null, alternative.schema, data.oneOf); } @@ -901,7 +902,6 @@ security: requestBodyProperties.push(data); } parseJoiObject(null, elems[0], data.items); - // handle array } else { const openApiType = joiTypeToOpenApiTypeMap[joiObject.type]; // even if object here then ignore and do not go recursive const isRequired = joiObject._flags.presence === 'required'; @@ -917,7 +917,6 @@ security: format = joiObject.type; } // TODO: if type before and after is string, add additional checks - // if (openApiType === 'string') { // console.log(joiObject._rules); // } @@ -932,13 +931,6 @@ security: // no path given, expect requestBodyProperties to be an array to append to requestBodyProperties.push(data); } - - // all other types - // 1) get type - // 2) get if required - // 3) if not in openapi types -> check if can be string -> openapi type: string + forrmat: check format - // if string check if it has additional check, e.g hex string, base64 etc, email, etc. - // 4) update requestBody at the path } } From d57405ce5efc564b50993a85f2abf873a97eb8d3 Mon Sep 17 00:00:00 2001 From: Nikolai Ovtsinnikov Date: Thu, 12 Oct 2023 13:59:37 +0300 Subject: [PATCH 08/30] fix replaceWithRefs and parseComponentsDecoupled functions, refactor, remove unnecessary comments and logs --- api.js | 81 ++++++++++++++++++++++++++-------------------------------- 1 file changed, 36 insertions(+), 45 deletions(-) diff --git a/api.js b/api.js index ff0bde8c..cf3aa34f 100644 --- a/api.js +++ b/api.js @@ -618,7 +618,7 @@ tags: - name: TwoFactorAuth - name: Users - name: Webhooks\n`; - const mapPathToMethods = {}; // map -> {path -> {post, put, delete, get}} + const mapPathToMethods = {}; // map -> {path -> {post -> {}, put -> {}, delete -> {}, get -> {}}} const routes = server.router.getRoutes(); for (const routePath in routes) { @@ -647,7 +647,7 @@ tags: // 4) add operationId methodObj.operationId = spec.name || route.name; - // 5) add requestBody, if object use recursion + // 5) add requestBody const applicationType = spec.applicationType || 'application/json'; methodObj.requestBody = { content: { @@ -668,7 +668,6 @@ tags: } // 6) add parameters (queryParams + pathParams). - // TODO: ADD FORMAT key in schema BASED ON FIELD ADDITIONAL RULES IN JOI methodObj.parameters = {}; for (const paramKey in spec.pathParams) { const paramKeyData = spec.pathParams[paramKey]; @@ -700,35 +699,45 @@ tags: parseJoiObject(resHttpCode, restBodyData, methodObj.responses); } - - // console.log(methodObj.responses); } const components = { components: { schemas: {} } }; for (const path in mapPathToMethods) { + // for every path const pathData = mapPathToMethods[path]; for (const httpMethod in pathData) { + // for every http method (post, put, get, delete) const innerData = pathData[httpMethod]; + // for every requestBody obj for (const key in innerData.requestBody.content[Object.keys(innerData.requestBody.content)[0]].schema.properties) { const reqBodyData = innerData.requestBody.content[Object.keys(innerData.requestBody.content)[0]].schema.properties[key]; parseComponetsDecoupled(reqBodyData, components.components.schemas); replaceWithRefs(reqBodyData); - console.log(reqBodyData); + } + + // for every response object + for (const key in innerData.responses) { + // key here is http method (2xx, 4xx, 5xx) + const obj = innerData.responses[key]; + parseComponetsDecoupled(obj, components.components.schemas); + replaceWithRefs(obj); } } } - // console.log(components.components.schemas); - // console.log(yaml.dump(components)); + // refify components that use other components + for (const obj of Object.values(components.components.schemas)) { + replaceWithRefs(obj); + } const finalObj = { paths: mapPathToMethods }; - const mapPathToMethodsYaml = yaml.dump(finalObj, { indent: 4, lineWidth: -1 }); - const componentsYaml = yaml.dump(components, { indent: 4, lineWidth: -1 }); + const mapPathToMethodsYaml = yaml.dump(finalObj, { indent: 4, lineWidth: -1, noRefs: true }); + const componentsYaml = yaml.dump(components, { indent: 4, lineWidth: -1, noRefs: true }); docs += mapPathToMethodsYaml; docs += componentsYaml; @@ -755,7 +764,7 @@ security: }) ); - // TODO: ignore function and symbol types for now + // ignore function and symbol types const joiTypeToOpenApiTypeMap = { any: 'object', number: 'number', @@ -792,51 +801,37 @@ security: } } - function parseComponetsDecoupled(component, components, level = 0) { + function parseComponetsDecoupled(component, components) { if (component.type === 'array') { - const obj = { ...component.items[0] }; // copy + const obj = structuredClone(component.items[0]); // copy if (obj.objectName) { for (const key in obj.properties) { - parseComponetsDecoupled(obj.properties[key], components, level + 1); + parseComponetsDecoupled(obj.properties[key], components); } + // in case the Array itself is marked as a separate object > const objectName = obj.objectName; components[objectName] = obj; delete components[objectName].objectName; - - if (level > 0) { - Object.keys(obj).forEach(key => { - if (key !== '$ref') { - delete obj[key]; - } - }); - obj.$ref = `#/components/schemas/${objectName}`; - } + // ^ } - } else if (component.type === 'object' && component.objectName) { - const obj = { ...component }; // copy - const objectName = component.objectName; + } else if (component.type === 'object') { + const obj = structuredClone(component); // copy + const objectName = obj.objectName; for (const key in obj.properties) { - parseComponetsDecoupled(obj.properties[key], components, level + 1); + parseComponetsDecoupled(obj.properties[key], components); } - components[objectName] = obj; - delete components[objectName].objectName; - - if (level > 0) { - Object.keys(obj).forEach(key => { - if (key !== '$ref') { - delete obj[key]; - } - }); - obj.$ref = `#/components/schemas/${objectName}`; + if (objectName) { + components[objectName] = obj; + delete components[objectName].objectName; } } else if (component.oneOf) { - // alternatives + // Joi object is of 'alternatives' types for (const obj in component.oneOf) { - parseComponetsDecoupled({ ...obj }, components, level); + parseComponetsDecoupled({ ...obj }, components); } } } @@ -864,6 +859,7 @@ security: } else { requestBodyProperties.push(data); } + for (const [key, value] of fieldsMap) { if (value.schema._flags.presence === 'required') { data.required.push(key); @@ -903,7 +899,7 @@ security: } parseJoiObject(null, elems[0], data.items); } else { - const openApiType = joiTypeToOpenApiTypeMap[joiObject.type]; // even if object here then ignore and do not go recursive + const openApiType = joiTypeToOpenApiTypeMap[joiObject.type]; // even if type is object here then ignore and do not go recursive const isRequired = joiObject._flags.presence === 'required'; const description = joiObject._flags.description || ''; let format = undefined; @@ -916,10 +912,6 @@ security: // type has changed, so probably string, acquire format format = joiObject.type; } - // TODO: if type before and after is string, add additional checks - // if (openApiType === 'string') { - // console.log(joiObject._rules); - // } const data = { type: openApiType, description, required: isRequired }; if (format) { @@ -928,7 +920,6 @@ security: if (path) { requestBodyProperties[path] = data; } else { - // no path given, expect requestBodyProperties to be an array to append to requestBodyProperties.push(data); } } From e5e72cbb64b2cd97a327821db28979d8bda904db Mon Sep 17 00:00:00 2001 From: Nikolai Ovtsinnikov Date: Thu, 12 Oct 2023 13:59:58 +0300 Subject: [PATCH 09/30] add support for another endpoint --- lib/api/messages.js | 45 ++++++++++++++++++++++++++++++++++++++------- 1 file changed, 38 insertions(+), 7 deletions(-) diff --git a/lib/api/messages.js b/lib/api/messages.js index 466adfd2..e5e81952 100644 --- a/lib/api/messages.js +++ b/lib/api/messages.js @@ -1599,7 +1599,8 @@ module.exports = (db, server, messageHandler, userHandler, storageHandler, setti }), previousDeleted: Joi.boolean() }) - } + }, + include: true }, tools.responseWrapper(async (req, res) => { res.charSet('utf-8'); @@ -1955,18 +1956,48 @@ module.exports = (db, server, messageHandler, userHandler, storageHandler, setti ); server.post( - '/users/:user/mailboxes/:mailbox/messages/:message/forward', - tools.responseWrapper(async (req, res) => { - res.charSet('utf-8'); - - const schema = Joi.object().keys({ + { + path: '/users/:user/mailboxes/:mailbox/messages/:message/forward', + pathParams: { user: Joi.string().hex().lowercase().length(24).required(), mailbox: Joi.string().hex().lowercase().length(24).required(), - message: Joi.number().required(), + message: Joi.number().required() + }, + summary: 'Forward stored Message', + description: + 'This method allows either to re-forward a message to an original forward target or forward it to some other address. This is useful if a user had forwarding turned on but the message was not delivered so you can try again. Forwarding does not modify the original message.', + requestBody: { target: Joi.number().min(1).max(1000), addresses: Joi.array().items(Joi.string().email({ tlds: false })), sess: sessSchema, ip: sessIPSchema + }, + queryParams: {}, + tags: ['Messages'], + response: { + 200: Joi.object({ + success: Joi.boolean(), + queueId: Joi.string(), + forwarded: Joi.array().items( + Joi.object({ + seq: Joi.string(), + type: Joi.string(), + value: Joi.string() + }).$_setFlag('objectName', 'Forwarded') + ) + }) + }, + include: true + }, + tools.responseWrapper(async (req, res) => { + res.charSet('utf-8'); + + const { pathParams, requestBody, queryParams } = req.route.spec; + + const schema = Joi.object({ + ...pathParams, + ...requestBody, + ...queryParams }); const result = schema.validate(req.params, { From 09632a522bfecc46bebbb447ba045e3fb20d3ae0 Mon Sep 17 00:00:00 2001 From: Nikolai Ovtsinnikov Date: Thu, 12 Oct 2023 14:10:50 +0300 Subject: [PATCH 10/30] move big code from api.js to tools --- api.js | 343 +------------------------------------------------- lib/tools.js | 345 +++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 346 insertions(+), 342 deletions(-) diff --git a/api.js b/api.js index cf3aa34f..e5ca651b 100644 --- a/api.js +++ b/api.js @@ -48,8 +48,6 @@ const certsRoutes = require('./lib/api/certs'); const webhooksRoutes = require('./lib/api/webhooks'); const settingsRoutes = require('./lib/api/settings'); const { SettingsHandler } = require('./lib/settings-handler'); -const fs = require('fs'); -const yaml = require('js-yaml'); let userHandler; let mailboxHandler; @@ -580,351 +578,12 @@ module.exports = done => { tools.responseWrapper(async (req, res) => { res.charSet('utf-8'); - let docs = ` -openapi: 3.0.0 -info: - title: WildDuck API - description: WildDuck API docs - version: 1.0.0 - contact: - url: 'https://github.com/nodemailer/wildduck' - -servers: - - url: 'https://api.wildduck.email' - -tags: - - name: Addresses - - name: ApplicationPasswords - - name: Archive - description: Archive includes all deleted messages. Once messages are old enough then these are permanenetly deleted from the archive as well. Until then you can restore the deleted messages. - - name: Audit - description: 'Auditing allows to monitor an email account. All existing, deleted and new emails are copied to the auditing system. See also https://github.com/nodemailer/wildduck-audit-manager' - - name: Authentication - - name: Autoreplies - - name: Certs - description: WildDuck allows to register TLS certificates to be used with SNI connections. These certificates are used by IMAP, POP3, API and SMTP servers when a SNI capable client establishes a TLS connection. This does not apply for MX servers. - - name: DKIM - description: Whenever an email is sent WildDuck checks if there is a DKIM key registered for the domain name of the sender address and uses it to sign the message. - - name: DomainAccess - description: Add sender domain names to allowlist (messages are all accepted) or blocklist (messages are sent to Spam folder) - - name: DomainAliases - - name: Filters - - name: Mailboxes - - name: Messages - - name: Settings - - name: Storage - description: Storage allows easier attachment handling when composing Draft messages. Instead of uploading the attachmnent with every draft update, you store the attachment to the Storage and then link stored file for the Draft. - - name: Submission - - name: TwoFactorAuth - - name: Users - - name: Webhooks\n`; - const mapPathToMethods = {}; // map -> {path -> {post -> {}, put -> {}, delete -> {}, get -> {}}} - const routes = server.router.getRoutes(); - for (const routePath in routes) { - const route = routes[routePath]; - const { spec } = route; - - if (!spec.include) { - continue; - } - - if (!mapPathToMethods[spec.path]) { - mapPathToMethods[spec.path] = {}; - } - - mapPathToMethods[spec.path][spec.method.toLowerCase()] = {}; - const methodObj = mapPathToMethods[spec.path][spec.method.toLowerCase()]; - // 1) add tags - methodObj.tags = spec.tags; - - // 2) add summary - methodObj.summary = spec.summary || ''; - - // 3) add description - methodObj.description = spec.description || ''; - - // 4) add operationId - methodObj.operationId = spec.name || route.name; - - // 5) add requestBody - const applicationType = spec.applicationType || 'application/json'; - methodObj.requestBody = { - content: { - [applicationType]: { - schema: { - type: 'object', - properties: {} - } - } - }, - required: true - }; - - for (const reqBodyKey in spec.requestBody) { - const reqBodyKeyData = spec.requestBody[reqBodyKey]; - - parseJoiObject(reqBodyKey, reqBodyKeyData, methodObj.requestBody.content[applicationType].schema.properties); - } - - // 6) add parameters (queryParams + pathParams). - methodObj.parameters = {}; - for (const paramKey in spec.pathParams) { - const paramKeyData = spec.pathParams[paramKey]; - - methodObj.parameters[paramKey] = {}; - const obj = methodObj.parameters[paramKey]; - obj.in = 'path'; - obj.description = paramKeyData._flags.description || ''; - obj.required = paramKeyData._flags.presence === 'required'; - obj.schema = { type: paramKeyData.type }; - } - - for (const paramKey in spec.queryParams) { - const paramKeyData = spec.queryParams[paramKey]; - - methodObj.parameters[paramKey] = {}; - const obj = methodObj.parameters[paramKey]; - obj.in = 'query'; - obj.description = paramKeyData._flags.description || ''; - obj.required = paramKeyData._flags.presence === 'required'; - obj.schema = { type: paramKeyData.type }; - } - - // 7) add responses - methodObj.responses = {}; - - for (const resHttpCode in spec.response) { - const restBodyData = spec.response[resHttpCode]; - - parseJoiObject(resHttpCode, restBodyData, methodObj.responses); - } - } - - const components = { components: { schemas: {} } }; - - for (const path in mapPathToMethods) { - // for every path - const pathData = mapPathToMethods[path]; - for (const httpMethod in pathData) { - // for every http method (post, put, get, delete) - const innerData = pathData[httpMethod]; - - // for every requestBody obj - for (const key in innerData.requestBody.content[Object.keys(innerData.requestBody.content)[0]].schema.properties) { - const reqBodyData = innerData.requestBody.content[Object.keys(innerData.requestBody.content)[0]].schema.properties[key]; - - parseComponetsDecoupled(reqBodyData, components.components.schemas); - replaceWithRefs(reqBodyData); - } - - // for every response object - for (const key in innerData.responses) { - // key here is http method (2xx, 4xx, 5xx) - const obj = innerData.responses[key]; - parseComponetsDecoupled(obj, components.components.schemas); - replaceWithRefs(obj); - } - } - } - - // refify components that use other components - for (const obj of Object.values(components.components.schemas)) { - replaceWithRefs(obj); - } - - const finalObj = { paths: mapPathToMethods }; - - const mapPathToMethodsYaml = yaml.dump(finalObj, { indent: 4, lineWidth: -1, noRefs: true }); - const componentsYaml = yaml.dump(components, { indent: 4, lineWidth: -1, noRefs: true }); - - docs += mapPathToMethodsYaml; - docs += componentsYaml; - - docs += ` - securitySchemes: - AccessTokenAuth: - name: X-Access-Token - type: apiKey - in: header - description: |- - If authentication is enabled in the WildDuck configuration, you will need to supply an access token in the \`X-Access-Token\` header. - - \`\`\`json - { - "X-Access-Token": "59fc66a03e54454869460e45" - } - \`\`\` -security: - - AccessTokenAuth: [] -`; - - await fs.promises.writeFile(__dirname + '/openapidocs.yml', docs); + tools.generateAPiDocs(routes); }) ); - // ignore function and symbol types - const joiTypeToOpenApiTypeMap = { - any: 'object', - number: 'number', - link: 'string', - boolean: 'boolean', - date: 'string', - string: 'string', - binary: 'string' - }; - - function replaceWithRefs(reqBodyData) { - if (reqBodyData.type === 'array') { - const obj = reqBodyData.items[0]; - - replaceWithRefs(obj); - } else if (reqBodyData.type === 'object') { - if (reqBodyData.objectName) { - const objectName = reqBodyData.objectName; - Object.keys(reqBodyData).forEach(key => { - if (key !== '$ref') { - delete reqBodyData[key]; - } - }); - reqBodyData.$ref = `#/components/schemas/${objectName}`; - } else { - for (const key in reqBodyData.properties) { - replaceWithRefs(reqBodyData.properties[key]); - } - } - } else if (reqBodyData.type === 'alternatives') { - for (const obj in reqBodyData.oneOf) { - replaceWithRefs(obj); - } - } - } - - function parseComponetsDecoupled(component, components) { - if (component.type === 'array') { - const obj = structuredClone(component.items[0]); // copy - - if (obj.objectName) { - for (const key in obj.properties) { - parseComponetsDecoupled(obj.properties[key], components); - } - - // in case the Array itself is marked as a separate object > - const objectName = obj.objectName; - components[objectName] = obj; - delete components[objectName].objectName; - // ^ - } - } else if (component.type === 'object') { - const obj = structuredClone(component); // copy - const objectName = obj.objectName; - - for (const key in obj.properties) { - parseComponetsDecoupled(obj.properties[key], components); - } - - if (objectName) { - components[objectName] = obj; - delete components[objectName].objectName; - } - } else if (component.oneOf) { - // Joi object is of 'alternatives' types - for (const obj in component.oneOf) { - parseComponetsDecoupled({ ...obj }, components); - } - } - } - - /** - * Parse Joi Objects - */ - function parseJoiObject(path, joiObject, requestBodyProperties) { - if (joiObject.type === 'object') { - const fieldsMap = joiObject._ids._byKey; - - const data = { - type: joiObject.type, - description: joiObject._flags.description || '', - properties: {}, - required: [] - }; - - if (joiObject._flags.objectName) { - data.objectName = joiObject._flags.objectName; - } - - if (path) { - requestBodyProperties[path] = data; - } else { - requestBodyProperties.push(data); - } - - for (const [key, value] of fieldsMap) { - if (value.schema._flags.presence === 'required') { - data.required.push(key); - } - parseJoiObject(key, value.schema, data.properties); - } - } else if (joiObject.type === 'alternatives') { - const matches = joiObject.$_terms.matches; - - const data = { - oneOf: [], - description: joiObject._flags.description || '' - }; - - if (path) { - requestBodyProperties[path] = data; - } else { - requestBodyProperties.push(data); - } - - for (const alternative of matches) { - parseJoiObject(null, alternative.schema, data.oneOf); - } - } else if (joiObject.type === 'array') { - const elems = joiObject?.$_terms.items; - - const data = { - type: 'array', - items: [], - description: joiObject._flags.description || '' - }; - - if (path) { - requestBodyProperties[path] = data; - } else { - requestBodyProperties.push(data); - } - parseJoiObject(null, elems[0], data.items); - } else { - const openApiType = joiTypeToOpenApiTypeMap[joiObject.type]; // even if type is object here then ignore and do not go recursive - const isRequired = joiObject._flags.presence === 'required'; - const description = joiObject._flags.description || ''; - let format = undefined; - - if (!openApiType) { - throw new Error('Unsupported type! Check API endpoint!'); - } - - if (joiObject.type !== openApiType) { - // type has changed, so probably string, acquire format - format = joiObject.type; - } - - const data = { type: openApiType, description, required: isRequired }; - if (format) { - data.format = format; - } - if (path) { - requestBodyProperties[path] = data; - } else { - requestBodyProperties.push(data); - } - } - } - server.on('error', err => { if (!started) { started = true; diff --git a/lib/tools.js b/lib/tools.js index 57f64ddd..3d77fc28 100644 --- a/lib/tools.js +++ b/lib/tools.js @@ -16,6 +16,7 @@ const ipaddr = require('ipaddr.js'); const ObjectId = require('mongodb').ObjectId; const log = require('npmlog'); const addressparser = require('nodemailer/lib/addressparser'); +const yaml = require('js-yaml'); let templates = false; @@ -605,6 +606,167 @@ function buildCertChain(cert, ca) { .join('\n'); } +// ignore function and symbol types +const joiTypeToOpenApiTypeMap = { + any: 'object', + number: 'number', + link: 'string', + boolean: 'boolean', + date: 'string', + string: 'string', + binary: 'string' +}; + +function replaceWithRefs(reqBodyData) { + if (reqBodyData.type === 'array') { + const obj = reqBodyData.items[0]; + + replaceWithRefs(obj); + } else if (reqBodyData.type === 'object') { + if (reqBodyData.objectName) { + const objectName = reqBodyData.objectName; + Object.keys(reqBodyData).forEach(key => { + if (key !== '$ref') { + delete reqBodyData[key]; + } + }); + reqBodyData.$ref = `#/components/schemas/${objectName}`; + } else { + for (const key in reqBodyData.properties) { + replaceWithRefs(reqBodyData.properties[key]); + } + } + } else if (reqBodyData.type === 'alternatives') { + for (const obj in reqBodyData.oneOf) { + replaceWithRefs(obj); + } + } +} + +function parseComponetsDecoupled(component, components) { + if (component.type === 'array') { + const obj = structuredClone(component.items[0]); // copy + + if (obj.objectName) { + for (const key in obj.properties) { + parseComponetsDecoupled(obj.properties[key], components); + } + + // in case the Array itself is marked as a separate object > + const objectName = obj.objectName; + components[objectName] = obj; + delete components[objectName].objectName; + // ^ + } + } else if (component.type === 'object') { + const obj = structuredClone(component); // copy + const objectName = obj.objectName; + + for (const key in obj.properties) { + parseComponetsDecoupled(obj.properties[key], components); + } + + if (objectName) { + components[objectName] = obj; + delete components[objectName].objectName; + } + } else if (component.oneOf) { + // Joi object is of 'alternatives' types + for (const obj in component.oneOf) { + parseComponetsDecoupled({ ...obj }, components); + } + } +} + +/** + * Parse Joi Objects + */ +function parseJoiObject(path, joiObject, requestBodyProperties) { + if (joiObject.type === 'object') { + const fieldsMap = joiObject._ids._byKey; + + const data = { + type: joiObject.type, + description: joiObject._flags.description || '', + properties: {}, + required: [] + }; + + if (joiObject._flags.objectName) { + data.objectName = joiObject._flags.objectName; + } + + if (path) { + requestBodyProperties[path] = data; + } else { + requestBodyProperties.push(data); + } + + for (const [key, value] of fieldsMap) { + if (value.schema._flags.presence === 'required') { + data.required.push(key); + } + parseJoiObject(key, value.schema, data.properties); + } + } else if (joiObject.type === 'alternatives') { + const matches = joiObject.$_terms.matches; + + const data = { + oneOf: [], + description: joiObject._flags.description || '' + }; + + if (path) { + requestBodyProperties[path] = data; + } else { + requestBodyProperties.push(data); + } + + for (const alternative of matches) { + parseJoiObject(null, alternative.schema, data.oneOf); + } + } else if (joiObject.type === 'array') { + const elems = joiObject?.$_terms.items; + + const data = { + type: 'array', + items: [], + description: joiObject._flags.description || '' + }; + + if (path) { + requestBodyProperties[path] = data; + } else { + requestBodyProperties.push(data); + } + parseJoiObject(null, elems[0], data.items); + } else { + const openApiType = joiTypeToOpenApiTypeMap[joiObject.type]; // even if type is object here then ignore and do not go recursive + const isRequired = joiObject._flags.presence === 'required'; + const description = joiObject._flags.description || ''; + let format = undefined; + + if (!openApiType) { + throw new Error('Unsupported type! Check API endpoint!'); + } + + if (joiObject.type !== openApiType) { + // type has changed, so probably string, acquire format + format = joiObject.type; + } + + const data = { type: openApiType, description, required: isRequired }; + if (format) { + data.format = format; + } + if (path) { + requestBodyProperties[path] = data; + } else { + requestBodyProperties.push(data); + } + } +} + module.exports = { normalizeAddress, normalizeDomain, @@ -690,5 +852,188 @@ module.exports = { res.json(data); } }; + }, + + async generateAPiDocs(routes) { + let docs = ` +openapi: 3.0.0 +info: + title: WildDuck API + description: WildDuck API docs + version: 1.0.0 + contact: + url: 'https://github.com/nodemailer/wildduck' + +servers: + - url: 'https://api.wildduck.email' + +tags: + - name: Addresses + - name: ApplicationPasswords + - name: Archive + description: Archive includes all deleted messages. Once messages are old enough then these are permanenetly deleted from the archive as well. Until then you can restore the deleted messages. + - name: Audit + description: 'Auditing allows to monitor an email account. All existing, deleted and new emails are copied to the auditing system. See also https://github.com/nodemailer/wildduck-audit-manager' + - name: Authentication + - name: Autoreplies + - name: Certs + description: WildDuck allows to register TLS certificates to be used with SNI connections. These certificates are used by IMAP, POP3, API and SMTP servers when a SNI capable client establishes a TLS connection. This does not apply for MX servers. + - name: DKIM + description: Whenever an email is sent WildDuck checks if there is a DKIM key registered for the domain name of the sender address and uses it to sign the message. + - name: DomainAccess + description: Add sender domain names to allowlist (messages are all accepted) or blocklist (messages are sent to Spam folder) + - name: DomainAliases + - name: Filters + - name: Mailboxes + - name: Messages + - name: Settings + - name: Storage + description: Storage allows easier attachment handling when composing Draft messages. Instead of uploading the attachmnent with every draft update, you store the attachment to the Storage and then link stored file for the Draft. + - name: Submission + - name: TwoFactorAuth + - name: Users + - name: Webhooks\n`; + const mapPathToMethods = {}; // map -> {path -> {post -> {}, put -> {}, delete -> {}, get -> {}}} + + for (const routePath in routes) { + const route = routes[routePath]; + const { spec } = route; + + if (!spec.include) { + continue; + } + + if (!mapPathToMethods[spec.path]) { + mapPathToMethods[spec.path] = {}; + } + + mapPathToMethods[spec.path][spec.method.toLowerCase()] = {}; + const methodObj = mapPathToMethods[spec.path][spec.method.toLowerCase()]; + // 1) add tags + methodObj.tags = spec.tags; + + // 2) add summary + methodObj.summary = spec.summary || ''; + + // 3) add description + methodObj.description = spec.description || ''; + + // 4) add operationId + methodObj.operationId = spec.name || route.name; + + // 5) add requestBody + const applicationType = spec.applicationType || 'application/json'; + methodObj.requestBody = { + content: { + [applicationType]: { + schema: { + type: 'object', + properties: {} + } + } + }, + required: true + }; + + for (const reqBodyKey in spec.requestBody) { + const reqBodyKeyData = spec.requestBody[reqBodyKey]; + + parseJoiObject(reqBodyKey, reqBodyKeyData, methodObj.requestBody.content[applicationType].schema.properties); + } + + // 6) add parameters (queryParams + pathParams). + methodObj.parameters = {}; + for (const paramKey in spec.pathParams) { + const paramKeyData = spec.pathParams[paramKey]; + + methodObj.parameters[paramKey] = {}; + const obj = methodObj.parameters[paramKey]; + obj.in = 'path'; + obj.description = paramKeyData._flags.description || ''; + obj.required = paramKeyData._flags.presence === 'required'; + obj.schema = { type: paramKeyData.type }; + } + + for (const paramKey in spec.queryParams) { + const paramKeyData = spec.queryParams[paramKey]; + + methodObj.parameters[paramKey] = {}; + const obj = methodObj.parameters[paramKey]; + obj.in = 'query'; + obj.description = paramKeyData._flags.description || ''; + obj.required = paramKeyData._flags.presence === 'required'; + obj.schema = { type: paramKeyData.type }; + } + + // 7) add responses + methodObj.responses = {}; + + for (const resHttpCode in spec.response) { + const restBodyData = spec.response[resHttpCode]; + + parseJoiObject(resHttpCode, restBodyData, methodObj.responses); + } + } + + const components = { components: { schemas: {} } }; + + for (const path in mapPathToMethods) { + // for every path + const pathData = mapPathToMethods[path]; + + for (const httpMethod in pathData) { + // for every http method (post, put, get, delete) + const innerData = pathData[httpMethod]; + + // for every requestBody obj + for (const key in innerData.requestBody.content[Object.keys(innerData.requestBody.content)[0]].schema.properties) { + const reqBodyData = innerData.requestBody.content[Object.keys(innerData.requestBody.content)[0]].schema.properties[key]; + + parseComponetsDecoupled(reqBodyData, components.components.schemas); + replaceWithRefs(reqBodyData); + } + + // for every response object + for (const key in innerData.responses) { + // key here is http method (2xx, 4xx, 5xx) + const obj = innerData.responses[key]; + parseComponetsDecoupled(obj, components.components.schemas); + replaceWithRefs(obj); + } + } + } + + // refify components that use other components + for (const obj of Object.values(components.components.schemas)) { + replaceWithRefs(obj); + } + + const finalObj = { paths: mapPathToMethods }; + + const mapPathToMethodsYaml = yaml.dump(finalObj, { indent: 4, lineWidth: -1, noRefs: true }); + const componentsYaml = yaml.dump(components, { indent: 4, lineWidth: -1, noRefs: true }); + + docs += mapPathToMethodsYaml; + docs += componentsYaml; + + docs += ` +securitySchemes: + AccessTokenAuth: + name: X-Access-Token + type: apiKey + in: header + description: |- + If authentication is enabled in the WildDuck configuration, you will need to supply an access token in the \`X-Access-Token\` header. + + \`\`\`json + { + "X-Access-Token": "59fc66a03e54454869460e45" + } + \`\`\` +security: +- AccessTokenAuth: [] +`; + + await fs.promises.writeFile(__dirname + '/../openapidocs.yml', docs); } }; From dc3e7eef19043f2364dda58bf6addc3702bd7df0 Mon Sep 17 00:00:00 2001 From: Nikolai Ovtsinnikov Date: Fri, 13 Oct 2023 12:58:39 +0300 Subject: [PATCH 11/30] fix array type representation, fix response objects, add necessary data and changes to endpoints --- lib/api/messages.js | 46 +++++++++++++++++-------------- lib/tools.js | 67 ++++++++++++++++++++++++++++++--------------- 2 files changed, 71 insertions(+), 42 deletions(-) diff --git a/lib/api/messages.js b/lib/api/messages.js index e5e81952..e9e219f7 100644 --- a/lib/api/messages.js +++ b/lib/api/messages.js @@ -1590,15 +1590,18 @@ module.exports = (db, server, messageHandler, userHandler, storageHandler, setti queryParams: {}, tags: ['Messages'], response: { - 200: Joi.object({ - success: Joi.boolean(), - message: Joi.object({ - id: Joi.number(), - malbox: Joi.string(), - size: Joi.number() - }), - previousDeleted: Joi.boolean() - }) + 200: { + description: 'Success', + model: Joi.object({ + success: Joi.boolean(), + message: Joi.object({ + id: Joi.number(), + malbox: Joi.string(), + size: Joi.number() + }), + previousDeleted: Joi.boolean() + }) + } }, include: true }, @@ -1975,17 +1978,20 @@ module.exports = (db, server, messageHandler, userHandler, storageHandler, setti queryParams: {}, tags: ['Messages'], response: { - 200: Joi.object({ - success: Joi.boolean(), - queueId: Joi.string(), - forwarded: Joi.array().items( - Joi.object({ - seq: Joi.string(), - type: Joi.string(), - value: Joi.string() - }).$_setFlag('objectName', 'Forwarded') - ) - }) + 200: { + description: 'Success', + model: Joi.object({ + success: Joi.boolean(), + queueId: Joi.string(), + forwarded: Joi.array().items( + Joi.object({ + seq: Joi.string(), + type: Joi.string(), + value: Joi.string() + }).$_setFlag('objectName', 'Forwarded') + ) + }) + } }, include: true }, diff --git a/lib/tools.js b/lib/tools.js index 3d77fc28..840331b6 100644 --- a/lib/tools.js +++ b/lib/tools.js @@ -619,7 +619,7 @@ const joiTypeToOpenApiTypeMap = { function replaceWithRefs(reqBodyData) { if (reqBodyData.type === 'array') { - const obj = reqBodyData.items[0]; + const obj = reqBodyData.items; replaceWithRefs(obj); } else if (reqBodyData.type === 'object') { @@ -645,7 +645,7 @@ function replaceWithRefs(reqBodyData) { function parseComponetsDecoupled(component, components) { if (component.type === 'array') { - const obj = structuredClone(component.items[0]); // copy + const obj = structuredClone(component.items); // copy if (obj.objectName) { for (const key in obj.properties) { @@ -698,8 +698,10 @@ function parseJoiObject(path, joiObject, requestBodyProperties) { if (path) { requestBodyProperties[path] = data; - } else { + } else if (Array.isArray(requestBodyProperties)) { requestBodyProperties.push(data); + } else { + requestBodyProperties.items = data; } for (const [key, value] of fieldsMap) { @@ -718,8 +720,10 @@ function parseJoiObject(path, joiObject, requestBodyProperties) { if (path) { requestBodyProperties[path] = data; - } else { + } else if (Array.isArray(requestBodyProperties)) { requestBodyProperties.push(data); + } else { + requestBodyProperties.items = data; } for (const alternative of matches) { @@ -730,16 +734,18 @@ function parseJoiObject(path, joiObject, requestBodyProperties) { const data = { type: 'array', - items: [], + items: {}, description: joiObject._flags.description || '' }; if (path) { requestBodyProperties[path] = data; - } else { + } else if (Array.isArray(requestBodyProperties)) { requestBodyProperties.push(data); + } else { + requestBodyProperties.items = data; } - parseJoiObject(null, elems[0], data.items); + parseJoiObject(null, elems[0], data); } else { const openApiType = joiTypeToOpenApiTypeMap[joiObject.type]; // even if type is object here then ignore and do not go recursive const isRequired = joiObject._flags.presence === 'required'; @@ -759,10 +765,13 @@ function parseJoiObject(path, joiObject, requestBodyProperties) { if (format) { data.format = format; } + if (path) { requestBodyProperties[path] = data; - } else { + } else if (Array.isArray(requestBodyProperties)) { requestBodyProperties.push(data); + } else { + requestBodyProperties.items = data; } } } @@ -871,24 +880,24 @@ tags: - name: Addresses - name: ApplicationPasswords - name: Archive - description: Archive includes all deleted messages. Once messages are old enough then these are permanenetly deleted from the archive as well. Until then you can restore the deleted messages. + description: Archive includes all deleted messages. Once messages are old enough then these are permanenetly deleted from the archive as well. Until then you can restore the deleted messages. - name: Audit - description: 'Auditing allows to monitor an email account. All existing, deleted and new emails are copied to the auditing system. See also https://github.com/nodemailer/wildduck-audit-manager' + description: 'Auditing allows to monitor an email account. All existing, deleted and new emails are copied to the auditing system. See also https://github.com/nodemailer/wildduck-audit-manager' - name: Authentication - name: Autoreplies - name: Certs - description: WildDuck allows to register TLS certificates to be used with SNI connections. These certificates are used by IMAP, POP3, API and SMTP servers when a SNI capable client establishes a TLS connection. This does not apply for MX servers. + description: WildDuck allows to register TLS certificates to be used with SNI connections. These certificates are used by IMAP, POP3, API and SMTP servers when a SNI capable client establishes a TLS connection. This does not apply for MX servers. - name: DKIM - description: Whenever an email is sent WildDuck checks if there is a DKIM key registered for the domain name of the sender address and uses it to sign the message. + description: Whenever an email is sent WildDuck checks if there is a DKIM key registered for the domain name of the sender address and uses it to sign the message. - name: DomainAccess - description: Add sender domain names to allowlist (messages are all accepted) or blocklist (messages are sent to Spam folder) + description: Add sender domain names to allowlist (messages are all accepted) or blocklist (messages are sent to Spam folder) - name: DomainAliases - name: Filters - name: Mailboxes - name: Messages - name: Settings - name: Storage - description: Storage allows easier attachment handling when composing Draft messages. Instead of uploading the attachmnent with every draft update, you store the attachment to the Storage and then link stored file for the Draft. + description: Storage allows easier attachment handling when composing Draft messages. Instead of uploading the attachmnent with every draft update, you store the attachment to the Storage and then link stored file for the Draft. - name: Submission - name: TwoFactorAuth - name: Users @@ -942,36 +951,50 @@ tags: } // 6) add parameters (queryParams + pathParams). - methodObj.parameters = {}; + methodObj.parameters = []; for (const paramKey in spec.pathParams) { const paramKeyData = spec.pathParams[paramKey]; - methodObj.parameters[paramKey] = {}; - const obj = methodObj.parameters[paramKey]; + const obj = {}; + obj.name = paramKey; obj.in = 'path'; obj.description = paramKeyData._flags.description || ''; obj.required = paramKeyData._flags.presence === 'required'; obj.schema = { type: paramKeyData.type }; + methodObj.parameters.push(obj); } for (const paramKey in spec.queryParams) { const paramKeyData = spec.queryParams[paramKey]; - methodObj.parameters[paramKey] = {}; - const obj = methodObj.parameters[paramKey]; + const obj = {}; + obj.name = paramKey; obj.in = 'query'; obj.description = paramKeyData._flags.description || ''; obj.required = paramKeyData._flags.presence === 'required'; obj.schema = { type: paramKeyData.type }; + methodObj.parameters.push(obj); } // 7) add responses + const responseType = spec.responseType || 'application/json'; methodObj.responses = {}; for (const resHttpCode in spec.response) { - const restBodyData = spec.response[resHttpCode]; + const resBodyData = spec.response[resHttpCode]; + + methodObj.responses[resHttpCode] = { + description: resBodyData.description || '', + content: { + [responseType]: { + schema: {} + } + } + }; + + const obj = methodObj.responses[resHttpCode]; - parseJoiObject(resHttpCode, restBodyData, methodObj.responses); + parseJoiObject('schema', resBodyData.model, obj.content[responseType]); } } @@ -996,7 +1019,7 @@ tags: // for every response object for (const key in innerData.responses) { // key here is http method (2xx, 4xx, 5xx) - const obj = innerData.responses[key]; + const obj = innerData.responses[key].content[Object.keys(innerData.responses[key].content)[0]].schema; parseComponetsDecoupled(obj, components.components.schemas); replaceWithRefs(obj); } From 9ba060c645b920c74bf8ba619dee9f8345ea6a94 Mon Sep 17 00:00:00 2001 From: Nikolai Ovtsinnikov Date: Tue, 17 Oct 2023 10:20:38 +0300 Subject: [PATCH 12/30] redo include logic into exclude login --- lib/api/messages.js | 6 ++---- lib/tools.js | 2 +- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/lib/api/messages.js b/lib/api/messages.js index e9e219f7..59dbffda 100644 --- a/lib/api/messages.js +++ b/lib/api/messages.js @@ -1602,8 +1602,7 @@ module.exports = (db, server, messageHandler, userHandler, storageHandler, setti previousDeleted: Joi.boolean() }) } - }, - include: true + } }, tools.responseWrapper(async (req, res) => { res.charSet('utf-8'); @@ -1992,8 +1991,7 @@ module.exports = (db, server, messageHandler, userHandler, storageHandler, setti ) }) } - }, - include: true + } }, tools.responseWrapper(async (req, res) => { res.charSet('utf-8'); diff --git a/lib/tools.js b/lib/tools.js index 840331b6..f8ce1136 100644 --- a/lib/tools.js +++ b/lib/tools.js @@ -908,7 +908,7 @@ tags: const route = routes[routePath]; const { spec } = route; - if (!spec.include) { + if (spec.exclude) { continue; } From d6aae5fc1d38dcc55e6d45699aeb535a70733e0d Mon Sep 17 00:00:00 2001 From: Nikolai Ovtsinnikov Date: Wed, 18 Oct 2023 10:23:30 +0300 Subject: [PATCH 13/30] fix api generation in tools.js to accomodate new naming of objects --- lib/api/messages.js | 164 +++++++++++++++++++++++--------------------- lib/tools.js | 16 ++--- 2 files changed, 92 insertions(+), 88 deletions(-) diff --git a/lib/api/messages.js b/lib/api/messages.js index d651f5fb..208bde0d 100644 --- a/lib/api/messages.js +++ b/lib/api/messages.js @@ -1532,77 +1532,79 @@ module.exports = (db, server, messageHandler, userHandler, storageHandler, setti server.post( { path: '/users/:user/mailboxes/:mailbox/messages', - pathParams: { - user: Joi.string().hex().lowercase().length(24).required(), - mailbox: Joi.string().hex().lowercase().length(24).required() - }, summary: 'Upload Message', description: 'This method allows to upload either an RFC822 formatted message or a message structure to a mailbox. Raw message is stored unmodified, no headers are added or removed. If you want to generate the uploaded message from structured data fields, then do not use the raw property.', - requestBody: { - date: Joi.date(), - unseen: booleanSchema.default(false), - flagged: booleanSchema.default(false), - draft: booleanSchema.default(false), + validationObjs: { + pathParams: { + user: Joi.string().hex().lowercase().length(24).required(), + mailbox: Joi.string().hex().lowercase().length(24).required() + }, + requestBody: { + date: Joi.date(), + unseen: booleanSchema.default(false), + flagged: booleanSchema.default(false), + draft: booleanSchema.default(false), - raw: Joi.binary().max(consts.MAX_ALLOWED_MESSAGE_SIZE).empty(''), + raw: Joi.binary().max(consts.MAX_ALLOWED_MESSAGE_SIZE).empty(''), - from: Address, + from: Address, - replyTo: Address, + replyTo: Address, - to: AddressOptionalNameArray, + to: AddressOptionalNameArray, - cc: AddressOptionalNameArray, + cc: AddressOptionalNameArray, - bcc: AddressOptionalNameArray, + bcc: AddressOptionalNameArray, - headers: Joi.array().items(Header), + headers: Joi.array().items(Header), - subject: Joi.string() - .empty('') - .max(2 * 1024), - text: Joi.string() - .empty('') - .max(1024 * 1024), - html: Joi.string() - .empty('') - .max(1024 * 1024), + subject: Joi.string() + .empty('') + .max(2 * 1024), + text: Joi.string() + .empty('') + .max(1024 * 1024), + html: Joi.string() + .empty('') + .max(1024 * 1024), - files: Joi.array().items(Joi.string().hex().lowercase().length(24)), + files: Joi.array().items(Joi.string().hex().lowercase().length(24)), - attachments: Joi.array().items(Attachment), + attachments: Joi.array().items(Attachment), - metaData: metaDataSchema.label('metaData'), + metaData: metaDataSchema.label('metaData'), - reference: ReferenceWithAttachments, + reference: ReferenceWithAttachments, - replacePrevious: Joi.object({ - mailbox: Joi.string().hex().lowercase().length(24), - id: Joi.number().required() - }), + replacePrevious: Joi.object({ + mailbox: Joi.string().hex().lowercase().length(24), + id: Joi.number().required() + }), - bimi: Bimi, + bimi: Bimi, - sess: sessSchema, - ip: sessIPSchema - }, - queryParams: {}, - tags: ['Messages'], - response: { - 200: { - description: 'Success', - model: Joi.object({ - success: Joi.boolean(), - message: Joi.object({ - id: Joi.number(), - malbox: Joi.string(), - size: Joi.number() - }), - previousDeleted: Joi.boolean() - }) + sess: sessSchema, + ip: sessIPSchema + }, + queryParams: {}, + response: { + 200: { + description: 'Success', + model: Joi.object({ + success: Joi.boolean(), + message: Joi.object({ + id: Joi.number(), + malbox: Joi.string(), + size: Joi.number() + }), + previousDeleted: Joi.boolean() + }) + } } - } + }, + tags: ['Messages'] }, tools.responseWrapper(async (req, res) => { res.charSet('utf-8'); @@ -1961,38 +1963,40 @@ module.exports = (db, server, messageHandler, userHandler, storageHandler, setti server.post( { path: '/users/:user/mailboxes/:mailbox/messages/:message/forward', - pathParams: { - user: Joi.string().hex().lowercase().length(24).required(), - mailbox: Joi.string().hex().lowercase().length(24).required(), - message: Joi.number().required() + validationObjs: { + pathParams: { + user: Joi.string().hex().lowercase().length(24).required(), + mailbox: Joi.string().hex().lowercase().length(24).required(), + message: Joi.number().required() + }, + queryParams: {}, + requestBody: { + target: Joi.number().min(1).max(1000), + addresses: Joi.array().items(Joi.string().email({ tlds: false })), + sess: sessSchema, + ip: sessIPSchema + }, + response: { + 200: { + description: 'Success', + model: Joi.object({ + success: Joi.boolean(), + queueId: Joi.string(), + forwarded: Joi.array().items( + Joi.object({ + seq: Joi.string(), + type: Joi.string(), + value: Joi.string() + }).$_setFlag('objectName', 'Forwarded') + ) + }) + } + } }, summary: 'Forward stored Message', description: 'This method allows either to re-forward a message to an original forward target or forward it to some other address. This is useful if a user had forwarding turned on but the message was not delivered so you can try again. Forwarding does not modify the original message.', - requestBody: { - target: Joi.number().min(1).max(1000), - addresses: Joi.array().items(Joi.string().email({ tlds: false })), - sess: sessSchema, - ip: sessIPSchema - }, - queryParams: {}, - tags: ['Messages'], - response: { - 200: { - description: 'Success', - model: Joi.object({ - success: Joi.boolean(), - queueId: Joi.string(), - forwarded: Joi.array().items( - Joi.object({ - seq: Joi.string(), - type: Joi.string(), - value: Joi.string() - }).$_setFlag('objectName', 'Forwarded') - ) - }) - } - } + tags: ['Messages'] }, tools.responseWrapper(async (req, res) => { res.charSet('utf-8'); diff --git a/lib/tools.js b/lib/tools.js index 25906994..d496d15b 100644 --- a/lib/tools.js +++ b/lib/tools.js @@ -948,16 +948,16 @@ tags: required: true }; - for (const reqBodyKey in spec.requestBody) { - const reqBodyKeyData = spec.requestBody[reqBodyKey]; + for (const reqBodyKey in spec.validationObjs?.requestBody) { + const reqBodyKeyData = spec.validationObjs.requestBody[reqBodyKey]; parseJoiObject(reqBodyKey, reqBodyKeyData, methodObj.requestBody.content[applicationType].schema.properties); } // 6) add parameters (queryParams + pathParams). methodObj.parameters = []; - for (const paramKey in spec.pathParams) { - const paramKeyData = spec.pathParams[paramKey]; + for (const paramKey in spec.validationObjs?.pathParams) { + const paramKeyData = spec.validationObjs.pathParams[paramKey]; const obj = {}; obj.name = paramKey; @@ -968,8 +968,8 @@ tags: methodObj.parameters.push(obj); } - for (const paramKey in spec.queryParams) { - const paramKeyData = spec.queryParams[paramKey]; + for (const paramKey in spec.validationObjs?.queryParams) { + const paramKeyData = spec.validationObjs.queryParams[paramKey]; const obj = {}; obj.name = paramKey; @@ -984,8 +984,8 @@ tags: const responseType = spec.responseType || 'application/json'; methodObj.responses = {}; - for (const resHttpCode in spec.response) { - const resBodyData = spec.response[resHttpCode]; + for (const resHttpCode in spec.validationObjs?.response) { + const resBodyData = spec.validationObjs.response[resHttpCode]; methodObj.responses[resHttpCode] = { description: resBodyData.description || '', From 06b9a4554568e3faf23d6f6e705984dc9ee9a766 Mon Sep 17 00:00:00 2001 From: Nikolai Ovtsinnikov Date: Wed, 18 Oct 2023 10:43:35 +0300 Subject: [PATCH 14/30] fix messages.js, add structuredClone check in tools.js --- lib/api/messages.js | 4 ++-- lib/tools.js | 5 +++++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/lib/api/messages.js b/lib/api/messages.js index 208bde0d..f35ecc95 100644 --- a/lib/api/messages.js +++ b/lib/api/messages.js @@ -1609,7 +1609,7 @@ module.exports = (db, server, messageHandler, userHandler, storageHandler, setti tools.responseWrapper(async (req, res) => { res.charSet('utf-8'); - const { pathParams, requestBody, queryParams } = req.route.spec; + const { pathParams, requestBody, queryParams } = req.route.spec.validationObjs; const schema = Joi.object({ ...pathParams, @@ -2001,7 +2001,7 @@ module.exports = (db, server, messageHandler, userHandler, storageHandler, setti tools.responseWrapper(async (req, res) => { res.charSet('utf-8'); - const { pathParams, requestBody, queryParams } = req.route.spec; + const { pathParams, requestBody, queryParams } = req.route.spec.validationObjs; const schema = Joi.object({ ...pathParams, diff --git a/lib/tools.js b/lib/tools.js index d496d15b..3e24eafa 100644 --- a/lib/tools.js +++ b/lib/tools.js @@ -20,6 +20,11 @@ const yaml = require('js-yaml'); let templates = false; +if (typeof structuredClone !== 'function') { + // eslint-disable-next-line no-global-assign + structuredClone = obj => JSON.parse(JSON.stringify(obj)); +} + function checkRangeQuery(uids, ne) { // check if uids is a straight continous array and if such then return a range query, // otherwise retrun a $in query From 419e3ec3ce1acd70fe89e1fbec5268884ecbb1c7 Mon Sep 17 00:00:00 2001 From: Nikolai Ovtsinnikov Date: Thu, 19 Oct 2023 10:02:12 +0300 Subject: [PATCH 15/30] fix structured clone definition --- lib/tools.js | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/lib/tools.js b/lib/tools.js index 3e24eafa..6a1184f7 100644 --- a/lib/tools.js +++ b/lib/tools.js @@ -20,10 +20,7 @@ const yaml = require('js-yaml'); let templates = false; -if (typeof structuredClone !== 'function') { - // eslint-disable-next-line no-global-assign - structuredClone = obj => JSON.parse(JSON.stringify(obj)); -} +const structuredCloneWrapper = typeof structuredClone === 'function' ? structuredClone : obj => JSON.parse(JSON.stringify(obj)); function checkRangeQuery(uids, ne) { // check if uids is a straight continous array and if such then return a range query, @@ -650,7 +647,7 @@ function replaceWithRefs(reqBodyData) { function parseComponetsDecoupled(component, components) { if (component.type === 'array') { - const obj = structuredClone(component.items); // copy + const obj = structuredCloneWrapper(component.items); // copy if (obj.objectName) { for (const key in obj.properties) { @@ -664,7 +661,7 @@ function parseComponetsDecoupled(component, components) { // ^ } } else if (component.type === 'object') { - const obj = structuredClone(component); // copy + const obj = structuredCloneWrapper(component); // copy const objectName = obj.objectName; for (const key in obj.properties) { From b5df32a641f0187c04ed37907a2e01802e6612e8 Mon Sep 17 00:00:00 2001 From: Nikolai Ovtsinnikov Date: Thu, 19 Oct 2023 13:38:54 +0300 Subject: [PATCH 16/30] add one endpoint in messages.js to the api generation --- lib/api/messages.js | 214 +++++++++++++++++++++++++++++++++----------- lib/schemas.js | 13 +-- 2 files changed, 171 insertions(+), 56 deletions(-) diff --git a/lib/api/messages.js b/lib/api/messages.js index f35ecc95..14bc4855 100644 --- a/lib/api/messages.js +++ b/lib/api/messages.js @@ -337,23 +337,113 @@ module.exports = (db, server, messageHandler, userHandler, storageHandler, setti }; server.get( - { name: 'messages', path: '/users/:user/mailboxes/:mailbox/messages' }, + { + name: 'messages', + path: '/users/:user/mailboxes/:mailbox/messages', + summary: 'List messages in a Mailbox', + description: 'Lists all messages in a mailbox', + validationObjs: { + requestBody: {}, + pathParams: { + user: Joi.string().hex().lowercase().length(24).required().description('ID of the User'), + mailbox: Joi.string().hex().lowercase().length(24).required().description('ID of the Mailbox') + }, + queryParams: { + unseen: booleanSchema.description('If true, then returns only unseen messages'), + metaData: booleanSchema.default(false).description('If true, then includes metaData in the response'), + threadCounters: booleanSchema + .default(false) + .description('If true, then includes threadMessageCount in the response. Counters come with some overhead'), + limit: Joi.number().empty('').default(20).min(1).max(250).description('How many records to return'), + order: Joi.any().empty('').allow('asc', 'desc').default('desc').description('Ordering of the records by insert date'), + next: nextPageCursorSchema, + previous: previousPageCursorSchema, + page: pageNrSchema, + sess: sessSchema, + ip: sessIPSchema, + includeHeaders: Joi.alternatives() + .try(Joi.string(), booleanSchema) + .description('Comma separated list of header keys to include in the response') + }, + response: { + 200: { + description: 'Success', + model: Joi.object({ + success: booleanSchema.description('Indicates successful response').required(), + total: Joi.number().description('How many results were found').required(), + page: Joi.number().description('Current page number. Derived from page query argument').required(), + previousCursor: Joi.alternatives() + .try(Joi.string(), booleanSchema) + .description('Either a cursor string or false if there are not any previous results') + .required(), + nextCursor: Joi.alternatives() + .try(Joi.string(), booleanSchema) + .description('Either a cursor string or false if there are not any next results') + .required(), + specialUse: Joi.string().description('Special use. If available').required(), + results: Joi.array() + .items( + Joi.object({ + id: Joi.number().required().description('ID of the Message'), + mailbox: Joi.string().required().description('ID of the Mailbox'), + thread: Joi.string().required().description('ID of the Thread'), + threadMessageCount: Joi.number().description( + 'Amount of messages in the Thread. Included if threadCounters query argument was true' + ), + from: Address.description('Sender in From: field'), + to: Joi.array().items(Address).required().description('Recipients in To: field'), + cc: Joi.array().items(Address).required().description('Recipients in Cc: field'), + bcc: Joi.array().items(Address).required().description('Recipients in Bcc: field. Usually only available for drafts'), + messageId: Joi.string().required().description('Message ID'), + subject: Joi.string().required().description('Message subject'), + date: Joi.date().required().description('Date string from header'), + idate: Joi.date().description('Date string of receive time'), + intro: Joi.string().required().description('First 128 bytes of the message'), + attachments: booleanSchema.required().description('Does the message have attachments'), + size: Joi.number().required().description('Message size in bytes'), + seen: booleanSchema.required().description('Is this message alread seen or not'), + deleted: booleanSchema + .required() + .description( + 'Does this message have a Deleted flag (should not have as messages are automatically deleted once this flag is set)' + ), + flagged: booleanSchema.required().description('Does this message have a Flagged flag'), + draft: booleanSchema.required().description('is this message a draft'), + answered: booleanSchema.required().description('Does this message have a Answered flag'), + forwarded: booleanSchema.required().description('Does this message have a $Forwarded flag'), + references: Joi.array().items(ReferenceWithAttachments).required().description('References'), + bimi: Bimi.required().description( + 'Marks BIMI verification as passed for a domain. NB! BIMI record and logo files for the domain must be valid.' + ), + contentType: Joi.object({ + value: Joi.string().required().description('MIME type of the message, eg. "multipart/mixed"'), + params: Joi.object().required().description('An object with Content-Type params as key-value pairs') + }) + .$_setFlag('objectName', 'ContentType') + .required() + .description('Parsed Content-Type header. Usually needed to identify encrypted messages and such'), + encrypted: booleanSchema.description('Specifies whether the message is encrypted'), + metaData: Joi.object().description('Custom metadata value. Included if metaData query argument was true'), + headers: Joi.object().description('Header object keys requested with the includeHeaders argument') + }).$_setFlag('objectName', 'GetMessagesResult') + ) + .required() + .description('Message listing') + }) + } + } + }, + tags: ['Messages'] + }, tools.responseWrapper(async (req, res) => { res.charSet('utf-8'); + const { requestBody, pathParams, queryParams } = req.route.spec.validationObjs; + const schema = Joi.object().keys({ - user: Joi.string().hex().lowercase().length(24).required(), - mailbox: Joi.string().hex().lowercase().length(24).required(), - unseen: booleanSchema, - metaData: booleanSchema.default(false), - threadCounters: booleanSchema.default(false), - limit: Joi.number().empty('').default(20).min(1).max(250), - order: Joi.any().empty('').allow('asc', 'desc').default('desc'), - next: nextPageCursorSchema, - previous: previousPageCursorSchema, - page: pageNrSchema, - sess: sessSchema, - ip: sessIPSchema + ...requestBody, + ...pathParams, + ...queryParams }); const result = schema.validate(req.params, { @@ -1537,53 +1627,71 @@ module.exports = (db, server, messageHandler, userHandler, storageHandler, setti 'This method allows to upload either an RFC822 formatted message or a message structure to a mailbox. Raw message is stored unmodified, no headers are added or removed. If you want to generate the uploaded message from structured data fields, then do not use the raw property.', validationObjs: { pathParams: { - user: Joi.string().hex().lowercase().length(24).required(), - mailbox: Joi.string().hex().lowercase().length(24).required() + user: Joi.string().hex().lowercase().length(24).required().description('ID of the User'), + mailbox: Joi.string().hex().lowercase().length(24).required().description('ID of the Mailbox') }, requestBody: { date: Joi.date(), - unseen: booleanSchema.default(false), - flagged: booleanSchema.default(false), - draft: booleanSchema.default(false), + unseen: booleanSchema.default(false).description('Is the message unseen or not'), + flagged: booleanSchema.default(false).description('Is the message flagged or not'), + draft: booleanSchema.default(false).description('Is the message a draft or not'), - raw: Joi.binary().max(consts.MAX_ALLOWED_MESSAGE_SIZE).empty(''), + raw: Joi.binary() + .max(consts.MAX_ALLOWED_MESSAGE_SIZE) + .empty('') + .description( + 'base64 encoded message source. Alternatively, you can provide this value as POST body by using message/rfc822 MIME type. If raw message is provided then it overrides any other mail configuration' + ), - from: Address, + from: Address.description('Addres for the From: header'), - replyTo: Address, + replyTo: Address.description('Address for the Reply-To: header'), - to: AddressOptionalNameArray, + to: AddressOptionalNameArray.description('Addresses for the To: header'), - cc: AddressOptionalNameArray, + cc: AddressOptionalNameArray.description('Addresses for the Cc: header'), - bcc: AddressOptionalNameArray, + bcc: AddressOptionalNameArray.description('Addresses for the Bcc: header'), - headers: Joi.array().items(Header), + headers: Joi.array() + .items(Header) + .description( + 'Custom headers for the message. If reference message is set then In-Reply-To and References headers are set automatically' + ), subject: Joi.string() .empty('') - .max(2 * 1024), + .max(2 * 1024) + .description('Message subject. If not then resolved from Reference message'), text: Joi.string() .empty('') - .max(1024 * 1024), + .max(1024 * 1024) + .description('Plaintext message'), html: Joi.string() .empty('') - .max(1024 * 1024), + .max(1024 * 1024) + .description('HTML formatted message'), - files: Joi.array().items(Joi.string().hex().lowercase().length(24)), + files: Joi.array() + .items(Joi.string().hex().lowercase().length(24)) + .description( + 'Attachments as storage file IDs. NB! When retrieving message info then an array of objects is returned. When uploading a message then an array of IDs is used.' + ), - attachments: Joi.array().items(Attachment), + attachments: Joi.array().items(Attachment).description('Attachments for the message'), - metaData: metaDataSchema.label('metaData'), + metaData: metaDataSchema.label('metaData').description('Optional metadata, must be an object or JSON formatted string'), - reference: ReferenceWithAttachments, + reference: ReferenceWithAttachments.description( + 'Optional referenced email. If uploaded message is a reply draft and relevant fields are not provided then these are resolved from the message to be replied to' + ), replacePrevious: Joi.object({ mailbox: Joi.string().hex().lowercase().length(24), id: Joi.number().required() - }), + }).description('If set, then deletes a previous message when storing the new one. Useful when uploading a new Draft message.'), - bimi: Bimi, + bimi: Bimi.description('Marks BIMI verification as passed for a domain. NB! BIMI record and logo files for the domain must be valid.'), sess: sessSchema, ip: sessIPSchema @@ -1593,13 +1701,13 @@ module.exports = (db, server, messageHandler, userHandler, storageHandler, setti 200: { description: 'Success', model: Joi.object({ - success: Joi.boolean(), + success: Joi.boolean().description('Indicates successful response'), message: Joi.object({ id: Joi.number(), malbox: Joi.string(), size: Joi.number() - }), - previousDeleted: Joi.boolean() + }).description('Message information'), + previousDeleted: Joi.boolean().description('Set if replacing a previous message was requested') }) } } @@ -1965,14 +2073,16 @@ module.exports = (db, server, messageHandler, userHandler, storageHandler, setti path: '/users/:user/mailboxes/:mailbox/messages/:message/forward', validationObjs: { pathParams: { - user: Joi.string().hex().lowercase().length(24).required(), - mailbox: Joi.string().hex().lowercase().length(24).required(), - message: Joi.number().required() + user: Joi.string().hex().lowercase().length(24).required().description('ID of the User'), + mailbox: Joi.string().hex().lowercase().length(24).required().description('ID of the Mailbox'), + message: Joi.number().required().description('Message ID') }, queryParams: {}, requestBody: { - target: Joi.number().min(1).max(1000), - addresses: Joi.array().items(Joi.string().email({ tlds: false })), + target: Joi.number().min(1).max(1000).description('Number of original forwarding target'), + addresses: Joi.array() + .items(Joi.string().email({ tlds: false })) + .description('An array of additional forward targets'), sess: sessSchema, ip: sessIPSchema }, @@ -1980,15 +2090,17 @@ module.exports = (db, server, messageHandler, userHandler, storageHandler, setti 200: { description: 'Success', model: Joi.object({ - success: Joi.boolean(), - queueId: Joi.string(), - forwarded: Joi.array().items( - Joi.object({ - seq: Joi.string(), - type: Joi.string(), - value: Joi.string() - }).$_setFlag('objectName', 'Forwarded') - ) + success: Joi.boolean().description('Indicates successful response'), + queueId: Joi.string().description('Message ID in outbound queue'), + forwarded: Joi.array() + .items( + Joi.object({ + seq: Joi.string(), + type: Joi.string(), + value: Joi.string() + }).$_setFlag('objectName', 'Forwarded') + ) + .description('Information about forwarding targets') }) } } diff --git a/lib/schemas.js b/lib/schemas.js index 33ec4f09..c5b31f0d 100644 --- a/lib/schemas.js +++ b/lib/schemas.js @@ -3,13 +3,14 @@ const EJSON = require('mongodb-extended-json'); const Joi = require('joi'); -const sessSchema = Joi.string().max(255).label('Session identifier'); +const sessSchema = Joi.string().max(255).label('Session identifier').description('IP address for the logs'); const sessIPSchema = Joi.string() .ip({ version: ['ipv4', 'ipv6'], cidr: 'forbidden' }) - .label('Client IP'); + .label('Client IP') + .description('Session identifier for the logs'); /* const tagSchema = Joi.string().max(); @@ -83,9 +84,11 @@ const metaDataValidator = () => (value, helpers) => { const mongoCursorSchema = Joi.string().trim().empty('').custom(mongoCursorValidator({}), 'Cursor validation').max(1024); const pageLimitSchema = Joi.number().default(20).min(1).max(250).label('Page size'); -const pageNrSchema = Joi.number().default(1).label('Page number'); -const nextPageCursorSchema = mongoCursorSchema.label('Next page cursor'); -const previousPageCursorSchema = mongoCursorSchema.label('Previous page cursor'); +const pageNrSchema = Joi.number().default(1).label('Page number').description('Current page number. Informational only, page numbers start from 1'); +const nextPageCursorSchema = mongoCursorSchema.label('Next page cursor').description('Cursor value for next page, retrieved from nextCursor response value'); +const previousPageCursorSchema = mongoCursorSchema + .label('Previous page cursor') + .description('Cursor value for previous page, retrieved from previousCursor response value'); const booleanSchema = Joi.boolean().empty('').truthy('Y', 'true', 'yes', 'on', '1', 1).falsy('N', 'false', 'no', 'off', '0', 0); const metaDataSchema = Joi.any().custom(metaDataValidator({}), 'metadata validation'); From e8513d4309b7230dd29cc25f812dd1a16e5197fc Mon Sep 17 00:00:00 2001 From: Nikolai Ovtsinnikov Date: Fri, 27 Oct 2023 09:58:18 +0300 Subject: [PATCH 17/30] messages.js add one more endpoint to API generation --- lib/api/messages.js | 41 ++++++++++++++++++++++++++++++++--------- lib/schemas.js | 4 ++-- 2 files changed, 34 insertions(+), 11 deletions(-) diff --git a/lib/api/messages.js b/lib/api/messages.js index 14bc4855..c59f8ea6 100644 --- a/lib/api/messages.js +++ b/lib/api/messages.js @@ -2278,18 +2278,41 @@ module.exports = (db, server, messageHandler, userHandler, storageHandler, setti ); server.post( - '/users/:user/mailboxes/:mailbox/messages/:message/submit', + { + path: '/users/:user/mailboxes/:mailbox/messages/:message/submit', + validationObjs: { + pathParams: { + user: Joi.string().hex().lowercase().length(24).required().description('ID of the User'), + mailbox: Joi.string().hex().lowercase().length(24).required().description('ID of the Mailbox'), + message: Joi.number().required().description('Message ID') + }, + queryParams: {}, + requestBody: { + deleteFiles: booleanSchema.description('If true then deletes attachment files listed in metaData.files array'), + sendTime: Joi.date().description('Datestring for delivery if message should be sent some later time'), + sess: sessSchema, + ip: sessIPSchema + }, + response: { + 200: { + description: 'Success', + model: Joi.object({}) + } + } + }, + summary: 'Submit Draft for delivery', + description: 'This method allows to submit a draft message for delivery. Draft is moved to Sent mail folder.', + tags: ['Messages'] + }, tools.responseWrapper(async (req, res) => { res.charSet('utf-8'); - const schema = Joi.object().keys({ - user: Joi.string().hex().lowercase().length(24).required(), - mailbox: Joi.string().hex().lowercase().length(24).required(), - message: Joi.number().required(), - deleteFiles: booleanSchema, - sendTime: Joi.date(), - sess: sessSchema, - ip: sessIPSchema + const { pathParams, requestBody, queryParams } = req.route.spec.validationObjs; + + const schema = Joi.object({ + ...pathParams, + ...queryParams, + ...requestBody }); const result = schema.validate(req.params, { diff --git a/lib/schemas.js b/lib/schemas.js index c5b31f0d..1aa7d454 100644 --- a/lib/schemas.js +++ b/lib/schemas.js @@ -3,14 +3,14 @@ const EJSON = require('mongodb-extended-json'); const Joi = require('joi'); -const sessSchema = Joi.string().max(255).label('Session identifier').description('IP address for the logs'); +const sessSchema = Joi.string().max(255).label('Session identifier').description('Session identifier for the logs'); const sessIPSchema = Joi.string() .ip({ version: ['ipv4', 'ipv6'], cidr: 'forbidden' }) .label('Client IP') - .description('Session identifier for the logs'); + .description('IP address for the logs '); /* const tagSchema = Joi.string().max(); From 73b96e0df338b7f4f4bb1ba0c6e9218dd5da262d Mon Sep 17 00:00:00 2001 From: Nikolai Ovtsinnikov Date: Fri, 27 Oct 2023 11:21:18 +0300 Subject: [PATCH 18/30] add response to prev commit. Add new endpoint to API generation. Archive message and archive messages --- lib/api/messages.js | 96 ++++++++++++++++++++++++++++++++++++++------- 1 file changed, 81 insertions(+), 15 deletions(-) diff --git a/lib/api/messages.js b/lib/api/messages.js index c59f8ea6..a28436a3 100644 --- a/lib/api/messages.js +++ b/lib/api/messages.js @@ -2296,7 +2296,17 @@ module.exports = (db, server, messageHandler, userHandler, storageHandler, setti response: { 200: { description: 'Success', - model: Joi.object({}) + model: Joi.object({ + success: booleanSchema.description('Indicates successful response').required(), + queueId: Joi.string().description('Message ID in outbound queue').required(), + message: Joi.object({ + id: Joi.number().description('Message ID in mailbox').required(), + mailbox: Joi.string().description('Mailbox ID the message was stored into').required(), + size: Joi.number().description('Size of the RFC822 formatted email') + }) + .description('Message information') + .$_setFlag('objectName', 'Message') + }) } } }, @@ -2818,16 +2828,43 @@ module.exports = (db, server, messageHandler, userHandler, storageHandler, setti ); server.post( - { name: 'create_restore_task', path: '/users/:user/archived/restore' }, + { + name: 'create_restore_task', + path: '/users/:user/archived/restore', + tags: ['Archive'], + summary: 'Restore archived messages', + description: + 'Initiates a restore task to move archived messages of a date range back to the mailboxes the messages were deleted from. If target mailbox does not exist, then the messages are moved to INBOX.', + validationObjs: { + pathParams: { + user: Joi.string().hex().lowercase().length(24).required().description('ID of the User') + }, + requestBody: { + start: Joi.date().label('Start time').required().description('Datestring'), + end: Joi.date().label('End time').required().description('Datestring'), + sess: sessSchema, + ip: sessIPSchema + }, + queryParams: {}, + response: { + 200: { + description: 'Success', + model: Joi.object({ + success: booleanSchema.required().description('Indicates successful response') + }) + } + } + } + }, tools.responseWrapper(async (req, res) => { res.charSet('utf-8'); - const schema = Joi.object().keys({ - user: Joi.string().hex().lowercase().length(24).required(), - start: Joi.date().label('Start time').required(), - end: Joi.date().label('End time').required(), - sess: sessSchema, - ip: sessIPSchema + const { pathParams, requestBody, queryParams } = req.route.spec.validationObjs; + + const schema = Joi.object({ + ...pathParams, + ...queryParams, + ...requestBody }); const result = schema.validate(req.params, { @@ -2902,16 +2939,45 @@ module.exports = (db, server, messageHandler, userHandler, storageHandler, setti ); server.post( - { name: 'archived_restore', path: '/users/:user/archived/messages/:message/restore' }, + { + name: 'archived_restore', + path: '/users/:user/archived/messages/:message/restore', + summary: 'Restore archived messages ', + description: + 'Initiates a restore task to move archived messages of a date range back to the mailboxes the messages were deleted from. If target mailbox does not exist, then the messages are moved to INBOX.', + tags: ['Archive'], + validationObjs: { + requestBody: { + mailbox: Joi.string().hex().lowercase().length(24).description('ID of the target Mailbox. If not set then original mailbox is used.'), + sess: sessSchema, + ip: sessIPSchema + }, + queryParams: {}, + pathParams: { + user: Joi.string().hex().lowercase().length(24).required().description('ID of the User'), + message: Joi.string().hex().lowercase().length(24).required().description('Message ID') + }, + response: { + 200: { + description: 'Success', + model: Joi.object({ + success: booleanSchema.required().description('Indicates successful response'), + mailbox: Joi.string().required().description('Maibox ID the message was moved to'), + id: Joi.number().required().description('New ID for the Message') + }) + } + } + } + }, tools.responseWrapper(async (req, res) => { res.charSet('utf-8'); - const schema = Joi.object().keys({ - user: Joi.string().hex().lowercase().length(24).required(), - message: Joi.string().hex().lowercase().length(24).required(), - mailbox: Joi.string().hex().lowercase().length(24), - sess: sessSchema, - ip: sessIPSchema + const { pathParams, requestBody, queryParams } = req.route.spec.validationObjs; + + const schema = Joi.object({ + ...pathParams, + ...requestBody, + ...queryParams }); const result = schema.validate(req.params, { From 417c09340eec933de5e4af75802b349294bbd11c Mon Sep 17 00:00:00 2001 From: Nikolai Ovtsinnikov Date: Mon, 30 Oct 2023 09:39:38 +0200 Subject: [PATCH 19/30] finish with post endpoints in messages.js --- lib/api/messages.js | 108 +++++++++++++++++++++++++++++--------------- 1 file changed, 72 insertions(+), 36 deletions(-) diff --git a/lib/api/messages.js b/lib/api/messages.js index a28436a3..056d2e84 100644 --- a/lib/api/messages.js +++ b/lib/api/messages.js @@ -612,37 +612,44 @@ module.exports = (db, server, messageHandler, userHandler, storageHandler, setti ); const searchSchema = Joi.object().keys({ - user: Joi.string().hex().lowercase().length(24).required(), - q: Joi.string().trim().empty('').max(1024).optional(), - - mailbox: Joi.string().hex().length(24).empty(''), - thread: Joi.string().hex().length(24).empty(''), - - or: Joi.object().keys({ - query: Joi.string().trim().max(255).empty(''), - from: Joi.string().trim().empty(''), - to: Joi.string().trim().empty(''), - subject: Joi.string().trim().empty('') - }), - - query: Joi.string().trim().max(255).empty(''), - datestart: Joi.date().label('Start time').empty(''), - dateend: Joi.date().label('End time').empty(''), - from: Joi.string().trim().empty(''), - to: Joi.string().trim().empty(''), - subject: Joi.string().trim().empty(''), - minSize: Joi.number().empty(''), - maxSize: Joi.number().empty(''), - attachments: booleanSchema, - flagged: booleanSchema, - unseen: booleanSchema, + q: Joi.string().trim().empty('').max(1024).optional().description('Additional query string'), + + mailbox: Joi.string().hex().length(24).empty('').description('ID of the Mailbox'), + thread: Joi.string().hex().length(24).empty('').description('Thread ID'), + + or: Joi.object({ + query: Joi.string() + .trim() + .max(255) + .empty('') + .description('Search string, uses MongoDB fulltext index. Covers data from mesage body and also common headers like from, to, subject etc.'), + from: Joi.string().trim().empty('').description('Partial match for the From: header line'), + to: Joi.string().trim().empty('').description('Partial match for the To: and Cc: header lines'), + subject: Joi.string().trim().empty('').description('Partial match for the Subject: header line') + }).description('At least onOne of the included terms must match'), + + query: Joi.string() + .trim() + .max(255) + .empty('') + .description('Search string, uses MongoDB fulltext index. Covers data from mesage body and also common headers like from, to, subject etc.'), + datestart: Joi.date().label('Start time').empty('').description('Datestring for the earliest message storing time'), + dateend: Joi.date().label('End time').empty('').description('Datestring for the latest message storing time'), + from: Joi.string().trim().empty('').description('Partial match for the From: header line'), + to: Joi.string().trim().empty('').description('Partial match for the To: and Cc: header lines'), + subject: Joi.string().trim().empty('').description('Partial match for the Subject: header line'), + minSize: Joi.number().empty('').description('Minimal message size in bytes'), + maxSize: Joi.number().empty('').description('Maximal message size in bytes'), + attachments: booleanSchema.description('If true, then matches only messages with attachments'), + flagged: booleanSchema.description('If true, then matches only messages with \\Flagged flags'), + unseen: booleanSchema.description('If true, then matches only messages without \\Seen flags'), includeHeaders: Joi.string() .max(1024) .trim() .empty('') .example('List-ID, MIME-Version') .description('Comma separated list of header keys to include in the response'), - searchable: booleanSchema, + searchable: booleanSchema.description('If true, then matches messages not in Junk or Trash'), sess: sessSchema, ip: sessIPSchema }); @@ -829,20 +836,49 @@ module.exports = (db, server, messageHandler, userHandler, storageHandler, setti ); server.post( - { name: 'searchApply', path: '/users/:user/search' }, + { + name: 'searchApply', + path: '/users/:user/search', + summary: 'Search and update messages', + description: + 'This method allows applying an action to all matching messages. This is an async method so that it will return immediately. Actual modifications are run in the background.', + tags: ['Messages'], + validationObjs: { + requestBody: { + ...searchSchema.keys({ + // actions to take on matching messages + action: Joi.object() + .keys({ + moveTo: Joi.string().hex().lowercase().length(24).description('ID of the target Mailbox if you want to move messages'), + seen: booleanSchema.description('State of the \\Seen flag'), + flagged: booleanSchema.description('State of the \\Flagged flag') + }) + .required() + .description('Define actions to take with matching messages') + }) + }, + queryParams: {}, + pathParams: { + user: Joi.string().hex().lowercase().length(24).required() + }, + response: { + 200: { + description: 'Success', + model: Joi.object({ + success: booleanSchema.required().description('Indicates if the action succeeded or not'), + scheduled: Joi.string().required().description('ID of the scheduled operation'), + existing: booleanSchema.required().description('Indicates if the scheduled operation already exists') + }) + } + } + } + }, tools.responseWrapper(async (req, res) => { res.charSet('utf-8'); - const schema = searchSchema.keys({ - // actions to take on matching messages - action: Joi.object() - .keys({ - moveTo: Joi.string().hex().lowercase().length(24), - seen: booleanSchema, - flagged: booleanSchema - }) - .required() - }); + const { pathParams, requestBody, queryParams } = req.route.spec.validationObjs; + + const schema = Joi.object({ ...pathParams, ...requestBody, ...queryParams }); const result = schema.validate(req.params, { abortEarly: false, From 1db752861c8cf5bc75c82c762b202636268608bf Mon Sep 17 00:00:00 2001 From: Nikolai Ovtsinnikov Date: Thu, 2 Nov 2023 11:45:40 +0200 Subject: [PATCH 20/30] added general request and response schemas. Also added req and res schemas for messages --- lib/schemas/request/general-schemas.js | 12 ++++++++++++ lib/schemas/response/general-schemas.js | 9 +++++++++ lib/schemas/response/messages-schemas.js | 18 ++++++++++++++++++ 3 files changed, 39 insertions(+) create mode 100644 lib/schemas/request/general-schemas.js create mode 100644 lib/schemas/response/general-schemas.js create mode 100644 lib/schemas/response/messages-schemas.js diff --git a/lib/schemas/request/general-schemas.js b/lib/schemas/request/general-schemas.js new file mode 100644 index 00000000..4d98c5c8 --- /dev/null +++ b/lib/schemas/request/general-schemas.js @@ -0,0 +1,12 @@ +'use strict'; + +const Joi = require('joi'); + +const userId = Joi.string().hex().lowercase().length(24).required().description('ID of the User'); +const mailboxId = Joi.string().hex().lowercase().length(24).required().description('ID of the Mailbox'); +const messageId = Joi.number().min(1).required().description('Message ID'); +module.exports = { + userId, + mailboxId, + messageId +}; diff --git a/lib/schemas/response/general-schemas.js b/lib/schemas/response/general-schemas.js new file mode 100644 index 00000000..208eebd7 --- /dev/null +++ b/lib/schemas/response/general-schemas.js @@ -0,0 +1,9 @@ +'use strict'; + +const { booleanSchema } = require('../../schemas'); + +const successRes = booleanSchema.required().description('Indicates successful response'); + +module.exports = { + successRes +}; diff --git a/lib/schemas/response/messages-schemas.js b/lib/schemas/response/messages-schemas.js new file mode 100644 index 00000000..bc0045b2 --- /dev/null +++ b/lib/schemas/response/messages-schemas.js @@ -0,0 +1,18 @@ +'use strict'; +const Joi = require('joi'); + +const Rcpt = Joi.object({ + value: Joi.string().required().description('RCPT TO address as provided by SMTP client'), + formatted: Joi.string().required().description('Normalized RCPT address') +}).$_setFlag('objectName', 'Rcpt'); + +const MsgEnvelope = Joi.object({ + from: Joi.string().required().description('Address from MAIL FROM'), + rcpt: Joi.array().items(Rcpt).description('Array of addresses from RCPT TO (should have just one normally)') +}) + .description('SMTP envelope (if available)') + .$_setFlag('objectName', 'Envelope'); + +module.exports = { + MsgEnvelope +}; From 71c6496def8249e716ea912158d4f2b71bc79309 Mon Sep 17 00:00:00 2001 From: Nikolai Ovtsinnikov Date: Thu, 2 Nov 2023 11:46:16 +0200 Subject: [PATCH 21/30] add multiple GET endpoints to API generation and changed them to new design. Use general schemas made earlier --- lib/api/messages.js | 236 +++++++++++++++++++++++++++++++++++++------- 1 file changed, 198 insertions(+), 38 deletions(-) diff --git a/lib/api/messages.js b/lib/api/messages.js index 056d2e84..12dfdc9a 100644 --- a/lib/api/messages.js +++ b/lib/api/messages.js @@ -26,6 +26,8 @@ const { getMongoDBQuery /*, getElasticSearchQuery*/ } = require('../search-query const BimiHandler = require('../bimi-handler'); const { Address, AddressOptionalNameArray, Header, Attachment, ReferenceWithAttachments, Bimi } = require('../schemas/request/messages-schemas'); +const { userId, mailboxId, messageId, successRes } = require('../schemas/request/general-schemas'); +const { MsgEnvelope } = require('../schemas/response/messages-schemas'); module.exports = (db, server, messageHandler, userHandler, storageHandler, settingsHandler) => { let maildrop = new Maildropper({ @@ -440,7 +442,7 @@ module.exports = (db, server, messageHandler, userHandler, storageHandler, setti const { requestBody, pathParams, queryParams } = req.route.spec.validationObjs; - const schema = Joi.object().keys({ + const schema = Joi.object({ ...requestBody, ...pathParams, ...queryParams @@ -655,24 +657,74 @@ module.exports = (db, server, messageHandler, userHandler, storageHandler, setti }); server.get( - { name: 'search', path: '/users/:user/search' }, + { + name: 'search', + path: '/users/:user/search', + validationObjs: { + queryParams: { + ...searchSchema.keys({ + threadCounters: booleanSchema + .default(false) + .description('If true, then includes threadMessageCount in the response. Counters come with some overhead'), + limit: Joi.number().default(20).min(1).max(250).description('How many records to return'), + order: Joi.any() + .empty('') + .allow('asc', 'desc') + .optional() + .description('Ordering of the records by insert date. If no order is supplied, results are sorted by heir mongoDB ObjectId.'), + includeHeaders: Joi.string() + .max(1024) + .trim() + .empty('') + .example('List-ID, MIME-Version') + .description('Comma separated list of header keys to include in the response'), + next: nextPageCursorSchema, + previous: previousPageCursorSchema, + page: pageNrSchema + }) + }, + pathParams: { user: userId }, + requestBody: {}, + response: { + 200: { + description: 'Success', + model: Joi.object({ + success: booleanSchema.required().description('Indicates successful response'), + query: Joi.string().required('Query'), + total: Joi.number().required('How many results were found'), + page: Joi.number().required('Current page number. Derived from page query argument'), + previousCursor: Joi.alternatives() + .try(booleanSchema, Joi.string()) + .required() + .description('Either a cursor string or false if there are not any previous results'), + nextCursor: Joi.alternatives() + .try(booleanSchema, Joi.string()) + .required() + .description('Either a cursor string or false if there are not any next results'), + results: Joi.array() + .items( + Joi.object({ + id: Joi.string().required().description('ID of the Domain Alias'), + alias: Joi.string().required().description('Domain Alias'), + domain: Joi.string().required().description('The domain this alias applies to') + }).$_setFlag('objectName', 'GetDomainAliasesResult') + ) + .required() + .description('Aliases listing') + }) + } + } + }, + summary: 'Search for messages ', + description: 'This method allows searching for matching messages.', + tags: ['Messages'] + }, tools.responseWrapper(async (req, res) => { res.charSet('utf-8'); - const schema = searchSchema.keys({ - threadCounters: booleanSchema.default(false), - limit: Joi.number().default(20).min(1).max(250), - order: Joi.any().empty('').allow('asc', 'desc').optional(), - includeHeaders: Joi.string() - .max(1024) - .trim() - .empty('') - .example('List-ID, MIME-Version') - .description('Comma separated list of header keys to include in the response'), - next: nextPageCursorSchema, - previous: previousPageCursorSchema, - page: pageNrSchema - }); + const { requestBody, queryParams, pathParams } = req.route.spec.validationObjs; + + const schema = Joi.object({ ...requestBody, ...queryParams, ...pathParams }); const result = schema.validate(req.params, { abortEarly: false, @@ -932,19 +984,126 @@ module.exports = (db, server, messageHandler, userHandler, storageHandler, setti ); server.get( - { name: 'message', path: '/users/:user/mailboxes/:mailbox/messages/:message' }, + { + name: 'message', + path: '/users/:user/mailboxes/:mailbox/messages/:message', + summary: 'Request Message information', + validationObjs: { + queryParams: { + replaceCidLinks: booleanSchema.default(false).description('If true then replaces cid links'), + markAsSeen: booleanSchema.default(false).description('If true then marks message as seen'), + sess: sessSchema, + ip: sessIPSchema + }, + pathParams: { + user: userId, + mailbox: mailboxId, + message: messageId + }, + requestBody: {}, + response: { + 200: { + description: 'Success', + model: Joi.object({ + success: successRes, + id: messageId, + mailbox: mailboxId, + user: userId, + envelope: MsgEnvelope.required(), + thread: Joi.string().required().description('ID of the Thread'), + from: Address.required(), + to: Address, + cc: Address, + bcc: Address, + subject: Joi.string().required().description('Message subject'), + messageId: Joi.string().required().description('Message-ID header'), + date: Joi.date().required().description('Date string from header'), + idate: Joi.date().description('Date string of receive time'), + list: Joi.object({ + id: Joi.string().required().description('Value from List-ID header'), + unsubscribe: Joi.string().required().description('Value from List-Unsubscribe header') + }) + .description('If set then this message is from a mailing list') + .$_setFlag('objectName', 'List'), + expires: Joi.string().description('Datestring, if set then indicates the time after this message is automatically deleted'), + seen: booleanSchema.required().description('Does this message have a \\Seen flag'), + deleted: booleanSchema.required().description('Does this message have a \\Deleted flag'), + flagged: booleanSchema.required().description('Does this message have a \\Flagged flag'), + draft: booleanSchema.required().description('Does this message have a \\Draft flag'), + html: Joi.array() + .items(Joi.string()) + .description( + 'An array of HTML string. Every array element is from a separate mime node, usually you would just join these to a single string' + ), + text: Joi.string().description('Plaintext content of the message'), + attachments: Joi.array() + .items( + Joi.object({ + id: Joi.string().required().description('Attachment ID'), + hash: Joi.string().description('SHA-256 hash of the contents of the attachment'), + filename: Joi.string().required().description('Filename of the attachment'), + contentType: Joi.string().required().description('MIME type'), + disposition: Joi.string().required().description('Attachment disposition'), + transferEncoding: Joi.string() + .required() + .description('Which transfer encoding was used (actual content when fetching attachments is not encoded)'), + related: booleanSchema + .required() + .description( + 'Was this attachment found from a multipart/related node. This usually means that this is an embedded image' + ), + sizeKb: Joi.number().required().description('Approximate size of the attachment in kilobytes') + }) + ) + .description('Attachments for the message'), + verificationResults: Joi.object({ + tls: Joi.object({ + name: Joi.object().required().description('Cipher name, eg "ECDHE-RSA-AES128-GCM-SHA256"'), + version: Joi.object().required().description('TLS version, eg "TLSv1/SSLv3"') + }) + .$_setFlag('objectName', 'Tls') + .required() + .description('TLS information. Value is false if TLS was not used'), + spf: Joi.object({}) + .required() + .description('Domain name (either MFROM or HELO) of verified SPF or false if no SPF match was found'), + dkim: Joi.object({}).required().description('Domain name of verified DKIM signature or false if no valid signature was found') + }).description( + 'Security verification info if message was received from MX. If this property is missing then do not automatically assume invalid TLS, SPF or DKIM.' + ), + bimi: Joi.object({ + certified: booleanSchema.description('If true, then this logo is from a VMC file'), + url: Joi.string().description('URL of the resource the logo was retrieved from'), + image: Joi.string().description('Data URL for the SVG image') + }).description('BIMI logo info. If logo validation failed in any way, then this property is not set'), + contentType: Joi.object({ + value: Joi.string().required().description('MIME type of the message, eg. "multipart/mixed'), + params: Joi.object({}).required().description('An object with Content-Type params as key-value pairs') + }) + .required() + .description('Parsed Content-Type header. Usually needed to identify encrypted messages and such'), + metaData: Joi.object({}).description('Custom metadata object set for this message'), + references: Joi.object({}), + files: Joi.object({}).description( + 'List of files added to this message as attachments. Applies to Drafts, normal messages do not have this property. Needed to prevent uploading the same attachment every time a draft is updated' + ), + outbound: Joi.array().items(Joi.object({})).description('Outbound queue entries'), + forwardTargets: Joi.object({}), + reference: Joi.object({}).description('Referenced message info'), + answered: booleanSchema.required(), + forwarded: booleanSchema.required() + }) + } + } + }, + tags: ['Messages'] + }, tools.responseWrapper(async (req, res) => { res.charSet('utf-8'); - const schema = Joi.object().keys({ - user: Joi.string().hex().lowercase().length(24).required(), - mailbox: Joi.string().hex().lowercase().length(24).required(), - message: Joi.number().min(1).required(), - replaceCidLinks: booleanSchema.default(false), - markAsSeen: booleanSchema.default(false), - sess: sessSchema, - ip: sessIPSchema - }); + const { requestBody, queryParams, pathParams } = req.route.spec.validationObjs; + + const schema = Joi.object({ ...requestBody, ...queryParams, ...pathParams }); const result = schema.validate(req.params, { abortEarly: false, @@ -1663,8 +1822,8 @@ module.exports = (db, server, messageHandler, userHandler, storageHandler, setti 'This method allows to upload either an RFC822 formatted message or a message structure to a mailbox. Raw message is stored unmodified, no headers are added or removed. If you want to generate the uploaded message from structured data fields, then do not use the raw property.', validationObjs: { pathParams: { - user: Joi.string().hex().lowercase().length(24).required().description('ID of the User'), - mailbox: Joi.string().hex().lowercase().length(24).required().description('ID of the Mailbox') + user: userId, + mailbox: mailboxId }, requestBody: { date: Joi.date(), @@ -2109,9 +2268,9 @@ module.exports = (db, server, messageHandler, userHandler, storageHandler, setti path: '/users/:user/mailboxes/:mailbox/messages/:message/forward', validationObjs: { pathParams: { - user: Joi.string().hex().lowercase().length(24).required().description('ID of the User'), - mailbox: Joi.string().hex().lowercase().length(24).required().description('ID of the Mailbox'), - message: Joi.number().required().description('Message ID') + user: userId, + mailbox: mailboxId, + message: messageId }, queryParams: {}, requestBody: { @@ -2318,9 +2477,9 @@ module.exports = (db, server, messageHandler, userHandler, storageHandler, setti path: '/users/:user/mailboxes/:mailbox/messages/:message/submit', validationObjs: { pathParams: { - user: Joi.string().hex().lowercase().length(24).required().description('ID of the User'), - mailbox: Joi.string().hex().lowercase().length(24).required().description('ID of the Mailbox'), - message: Joi.number().required().description('Message ID') + user: userId, + mailbox: mailboxId, + message: messageId }, queryParams: {}, requestBody: { @@ -2873,7 +3032,7 @@ module.exports = (db, server, messageHandler, userHandler, storageHandler, setti 'Initiates a restore task to move archived messages of a date range back to the mailboxes the messages were deleted from. If target mailbox does not exist, then the messages are moved to INBOX.', validationObjs: { pathParams: { - user: Joi.string().hex().lowercase().length(24).required().description('ID of the User') + user: userId }, requestBody: { start: Joi.date().label('Start time').required().description('Datestring'), @@ -2886,7 +3045,8 @@ module.exports = (db, server, messageHandler, userHandler, storageHandler, setti 200: { description: 'Success', model: Joi.object({ - success: booleanSchema.required().description('Indicates successful response') + success: booleanSchema.required().description('Indicates successful response'), + task: Joi.string().required().description('Task ID') }) } } @@ -2990,8 +3150,8 @@ module.exports = (db, server, messageHandler, userHandler, storageHandler, setti }, queryParams: {}, pathParams: { - user: Joi.string().hex().lowercase().length(24).required().description('ID of the User'), - message: Joi.string().hex().lowercase().length(24).required().description('Message ID') + user: userId, + message: messageId }, response: { 200: { From 00a5afb21016caee0fde32c6a454868d1c32562a Mon Sep 17 00:00:00 2001 From: Nikolai Ovtsinnikov Date: Thu, 2 Nov 2023 12:03:13 +0200 Subject: [PATCH 22/30] fix incorrect import of successRes --- lib/api/messages.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/api/messages.js b/lib/api/messages.js index 12dfdc9a..dcc79701 100644 --- a/lib/api/messages.js +++ b/lib/api/messages.js @@ -26,8 +26,9 @@ const { getMongoDBQuery /*, getElasticSearchQuery*/ } = require('../search-query const BimiHandler = require('../bimi-handler'); const { Address, AddressOptionalNameArray, Header, Attachment, ReferenceWithAttachments, Bimi } = require('../schemas/request/messages-schemas'); -const { userId, mailboxId, messageId, successRes } = require('../schemas/request/general-schemas'); +const { userId, mailboxId, messageId } = require('../schemas/request/general-schemas'); const { MsgEnvelope } = require('../schemas/response/messages-schemas'); +const { successRes } = require('../schemas/response/general-schemas'); module.exports = (db, server, messageHandler, userHandler, storageHandler, settingsHandler) => { let maildrop = new Maildropper({ From fd310ae2576e1b6e42b330efce0ebde1b279625f Mon Sep 17 00:00:00 2001 From: Nikolai Ovtsinnikov Date: Thu, 2 Nov 2023 12:22:12 +0200 Subject: [PATCH 23/30] fix mailboxes.js --- lib/api/mailboxes.js | 38 ++++++++++++++++++++++++++++---------- 1 file changed, 28 insertions(+), 10 deletions(-) diff --git a/lib/api/mailboxes.js b/lib/api/mailboxes.js index f2933f21..e6fe667c 100644 --- a/lib/api/mailboxes.js +++ b/lib/api/mailboxes.js @@ -7,6 +7,8 @@ const tools = require('../tools'); const roles = require('../roles'); const util = require('util'); const { sessSchema, sessIPSchema, booleanSchema } = require('../schemas'); +const { userId, mailboxId } = require('../schemas/request/general-schemas'); +const { successRes } = require('../schemas/response/general-schemas'); module.exports = (db, server, mailboxHandler) => { const getMailboxCounter = util.promisify(tools.getMailboxCounter); @@ -248,22 +250,38 @@ module.exports = (db, server, mailboxHandler) => { server.post( { path: '/users/:user/mailboxes', - pathParams: { user: Joi.string().hex().lowercase().length(24).required().description('This is the user path param') }, - description: 'Some description for the API path', - requestBody: { - path: Joi.string() - .regex(/\/{2,}|\/$/, { invert: true }) - .required(), - hidden: booleanSchema.default(false), - retention: Joi.number().min(0) + summary: 'Create new Mailbox', + validationObjs: { + pathParams: { user: userId }, + requestBody: { + path: Joi.string() + .regex(/\/{2,}|\/$/, { invert: true }) + .required() + .description('Full path of the mailbox, folders are separated by slashes, ends with the mailbox name (unicode string)'), + hidden: booleanSchema.default(false).description('Is the folder hidden or not. Hidden folders can not be opened in IMAP.'), + retention: Joi.number() + .min(0) + .description('Retention policy for the created Mailbox. Milliseconds after a message added to mailbox expires. Set to 0 to disable.'), + sess: sessSchema, + ip: sessIPSchema + }, + queryParams: {}, + response: { + 200: { + description: 'Success', + model: { + success: successRes, + id: mailboxId + } + } + } }, - queryParams: { sess: sessSchema, ip: sessIPSchema }, tags: ['Mailboxes'] }, tools.responseWrapper(async (req, res) => { res.charSet('utf-8'); - const { pathParams, requestBody, queryParams } = req.route.spec; + const { pathParams, requestBody, queryParams } = req.route.spec.validationObjs; const schema = Joi.object({ ...pathParams, From 2d31d2b53833c94470122534e82aef4b869ea2d9 Mon Sep 17 00:00:00 2001 From: Nikolai Ovtsinnikov Date: Thu, 2 Nov 2023 12:40:58 +0200 Subject: [PATCH 24/30] refactor general-schemas.js. Fix searchSchema in messages.js. Mailboxes.js fix response --- lib/api/mailboxes.js | 4 ++-- lib/api/messages.js | 16 +++++++++------- lib/schemas/request/general-schemas.js | 1 + 3 files changed, 12 insertions(+), 9 deletions(-) diff --git a/lib/api/mailboxes.js b/lib/api/mailboxes.js index e6fe667c..6fc45810 100644 --- a/lib/api/mailboxes.js +++ b/lib/api/mailboxes.js @@ -269,10 +269,10 @@ module.exports = (db, server, mailboxHandler) => { response: { 200: { description: 'Success', - model: { + model: Joi.object({ success: successRes, id: mailboxId - } + }) } } }, diff --git a/lib/api/messages.js b/lib/api/messages.js index dcc79701..67beecc7 100644 --- a/lib/api/messages.js +++ b/lib/api/messages.js @@ -614,7 +614,7 @@ module.exports = (db, server, messageHandler, userHandler, storageHandler, setti }) ); - const searchSchema = Joi.object().keys({ + const searchSchema = { q: Joi.string().trim().empty('').max(1024).optional().description('Additional query string'), mailbox: Joi.string().hex().length(24).empty('').description('ID of the Mailbox'), @@ -655,7 +655,7 @@ module.exports = (db, server, messageHandler, userHandler, storageHandler, setti searchable: booleanSchema.description('If true, then matches messages not in Junk or Trash'), sess: sessSchema, ip: sessIPSchema - }); + }; server.get( { @@ -663,7 +663,8 @@ module.exports = (db, server, messageHandler, userHandler, storageHandler, setti path: '/users/:user/search', validationObjs: { queryParams: { - ...searchSchema.keys({ + ...searchSchema, + ...{ threadCounters: booleanSchema .default(false) .description('If true, then includes threadMessageCount in the response. Counters come with some overhead'), @@ -682,7 +683,7 @@ module.exports = (db, server, messageHandler, userHandler, storageHandler, setti next: nextPageCursorSchema, previous: previousPageCursorSchema, page: pageNrSchema - }) + } }, pathParams: { user: userId }, requestBody: {}, @@ -716,7 +717,7 @@ module.exports = (db, server, messageHandler, userHandler, storageHandler, setti } } }, - summary: 'Search for messages ', + summary: 'Search for messages', description: 'This method allows searching for matching messages.', tags: ['Messages'] }, @@ -898,7 +899,8 @@ module.exports = (db, server, messageHandler, userHandler, storageHandler, setti tags: ['Messages'], validationObjs: { requestBody: { - ...searchSchema.keys({ + ...searchSchema, + ...{ // actions to take on matching messages action: Joi.object() .keys({ @@ -908,7 +910,7 @@ module.exports = (db, server, messageHandler, userHandler, storageHandler, setti }) .required() .description('Define actions to take with matching messages') - }) + } }, queryParams: {}, pathParams: { diff --git a/lib/schemas/request/general-schemas.js b/lib/schemas/request/general-schemas.js index 4d98c5c8..2eef368e 100644 --- a/lib/schemas/request/general-schemas.js +++ b/lib/schemas/request/general-schemas.js @@ -5,6 +5,7 @@ const Joi = require('joi'); const userId = Joi.string().hex().lowercase().length(24).required().description('ID of the User'); const mailboxId = Joi.string().hex().lowercase().length(24).required().description('ID of the Mailbox'); const messageId = Joi.number().min(1).required().description('Message ID'); + module.exports = { userId, mailboxId, From 5e4a20a644bf405acd30fc1613449a2a4213881e Mon Sep 17 00:00:00 2001 From: Nikolai Ovtsinnikov Date: Thu, 2 Nov 2023 12:42:13 +0200 Subject: [PATCH 25/30] tools.js rename methodObj in API generation to operationObj --- lib/tools.js | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/lib/tools.js b/lib/tools.js index 6a1184f7..b28ca7e7 100644 --- a/lib/tools.js +++ b/lib/tools.js @@ -923,22 +923,22 @@ tags: } mapPathToMethods[spec.path][spec.method.toLowerCase()] = {}; - const methodObj = mapPathToMethods[spec.path][spec.method.toLowerCase()]; + const operationObj = mapPathToMethods[spec.path][spec.method.toLowerCase()]; // 1) add tags - methodObj.tags = spec.tags; + operationObj.tags = spec.tags; // 2) add summary - methodObj.summary = spec.summary || ''; + operationObj.summary = spec.summary || ''; // 3) add description - methodObj.description = spec.description || ''; + operationObj.description = spec.description || ''; // 4) add operationId - methodObj.operationId = spec.name || route.name; + operationObj.operationId = spec.name || route.name; // 5) add requestBody const applicationType = spec.applicationType || 'application/json'; - methodObj.requestBody = { + operationObj.requestBody = { content: { [applicationType]: { schema: { @@ -953,11 +953,11 @@ tags: for (const reqBodyKey in spec.validationObjs?.requestBody) { const reqBodyKeyData = spec.validationObjs.requestBody[reqBodyKey]; - parseJoiObject(reqBodyKey, reqBodyKeyData, methodObj.requestBody.content[applicationType].schema.properties); + parseJoiObject(reqBodyKey, reqBodyKeyData, operationObj.requestBody.content[applicationType].schema.properties); } // 6) add parameters (queryParams + pathParams). - methodObj.parameters = []; + operationObj.parameters = []; for (const paramKey in spec.validationObjs?.pathParams) { const paramKeyData = spec.validationObjs.pathParams[paramKey]; @@ -967,7 +967,7 @@ tags: obj.description = paramKeyData._flags.description || ''; obj.required = paramKeyData._flags.presence === 'required'; obj.schema = { type: paramKeyData.type }; - methodObj.parameters.push(obj); + operationObj.parameters.push(obj); } for (const paramKey in spec.validationObjs?.queryParams) { @@ -979,17 +979,17 @@ tags: obj.description = paramKeyData._flags.description || ''; obj.required = paramKeyData._flags.presence === 'required'; obj.schema = { type: paramKeyData.type }; - methodObj.parameters.push(obj); + operationObj.parameters.push(obj); } // 7) add responses const responseType = spec.responseType || 'application/json'; - methodObj.responses = {}; + operationObj.responses = {}; for (const resHttpCode in spec.validationObjs?.response) { const resBodyData = spec.validationObjs.response[resHttpCode]; - methodObj.responses[resHttpCode] = { + operationObj.responses[resHttpCode] = { description: resBodyData.description || '', content: { [responseType]: { @@ -998,7 +998,7 @@ tags: } }; - const obj = methodObj.responses[resHttpCode]; + const obj = operationObj.responses[resHttpCode]; parseJoiObject('schema', resBodyData.model, obj.content[responseType]); } From 9234b0d4d410eda6e1444cd69d3194fd53f60398 Mon Sep 17 00:00:00 2001 From: Nikolai Ovtsinnikov Date: Thu, 2 Nov 2023 12:44:39 +0200 Subject: [PATCH 26/30] tools.js api generation remove string fallbacks --- lib/tools.js | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/lib/tools.js b/lib/tools.js index b28ca7e7..0e9e59d1 100644 --- a/lib/tools.js +++ b/lib/tools.js @@ -689,7 +689,7 @@ function parseJoiObject(path, joiObject, requestBodyProperties) { const data = { type: joiObject.type, - description: joiObject._flags.description || '', + description: joiObject._flags.description, properties: {}, required: [] }; @@ -717,7 +717,7 @@ function parseJoiObject(path, joiObject, requestBodyProperties) { const data = { oneOf: [], - description: joiObject._flags.description || '' + description: joiObject._flags.description }; if (path) { @@ -737,7 +737,7 @@ function parseJoiObject(path, joiObject, requestBodyProperties) { const data = { type: 'array', items: {}, - description: joiObject._flags.description || '' + description: joiObject._flags.description }; if (path) { @@ -751,7 +751,7 @@ function parseJoiObject(path, joiObject, requestBodyProperties) { } else { const openApiType = joiTypeToOpenApiTypeMap[joiObject.type]; // even if type is object here then ignore and do not go recursive const isRequired = joiObject._flags.presence === 'required'; - const description = joiObject._flags.description || ''; + const description = joiObject._flags.description; let format = undefined; if (!openApiType) { @@ -928,10 +928,10 @@ tags: operationObj.tags = spec.tags; // 2) add summary - operationObj.summary = spec.summary || ''; + operationObj.summary = spec.summary; // 3) add description - operationObj.description = spec.description || ''; + operationObj.description = spec.description; // 4) add operationId operationObj.operationId = spec.name || route.name; @@ -964,7 +964,7 @@ tags: const obj = {}; obj.name = paramKey; obj.in = 'path'; - obj.description = paramKeyData._flags.description || ''; + obj.description = paramKeyData._flags.description; obj.required = paramKeyData._flags.presence === 'required'; obj.schema = { type: paramKeyData.type }; operationObj.parameters.push(obj); @@ -976,7 +976,7 @@ tags: const obj = {}; obj.name = paramKey; obj.in = 'query'; - obj.description = paramKeyData._flags.description || ''; + obj.description = paramKeyData._flags.description; obj.required = paramKeyData._flags.presence === 'required'; obj.schema = { type: paramKeyData.type }; operationObj.parameters.push(obj); @@ -990,7 +990,7 @@ tags: const resBodyData = spec.validationObjs.response[resHttpCode]; operationObj.responses[resHttpCode] = { - description: resBodyData.description || '', + description: resBodyData.description, content: { [responseType]: { schema: {} From 70ebdbc3712ee9319e1b441d09d095fa832e563d Mon Sep 17 00:00:00 2001 From: Nikolai Ovtsinnikov Date: Fri, 3 Nov 2023 10:43:24 +0200 Subject: [PATCH 27/30] messages.js finish with GET endpoints, addition to API doc generation --- lib/api/messages.js | 203 +++++++++++++++++++++++++++++++++++++------- 1 file changed, 171 insertions(+), 32 deletions(-) diff --git a/lib/api/messages.js b/lib/api/messages.js index 67beecc7..08e6cea5 100644 --- a/lib/api/messages.js +++ b/lib/api/messages.js @@ -1393,14 +1393,41 @@ module.exports = (db, server, messageHandler, userHandler, storageHandler, setti ); server.get( - { name: 'raw', path: '/users/:user/mailboxes/:mailbox/messages/:message/message.eml' }, + { + name: 'raw', + path: '/users/:user/mailboxes/:mailbox/messages/:message/message.eml', + summary: 'Get Message source', + description: 'This method returns the full RFC822 formatted source of the stored message', + validationObjs: { + pathParams: { + user: userId, + mailbox: mailboxId, + message: messageId + }, + queryParams: { + sess: sessSchema, + ip: sessIPSchema + }, + requestBody: {}, + response: { + 200: { + description: 'Success', + model: Joi.object({ + success: Joi.object({}).description('Success') + }) + } + } + }, + responseType: 'message/rfc822', + tags: ['Messages'] + }, tools.responseWrapper(async (req, res) => { - const schema = Joi.object().keys({ - user: Joi.string().hex().lowercase().length(24).required(), - mailbox: Joi.string().hex().lowercase().length(24).required(), - message: Joi.number().min(1).required(), - sess: sessSchema, - ip: sessIPSchema + const { pathParams, queryParams, requestBody } = req.route.spec.validationObjs; + + const schema = Joi.object({ + ...pathParams, + ...queryParams, + ...requestBody }); const result = schema.validate(req.params, { @@ -1489,16 +1516,43 @@ module.exports = (db, server, messageHandler, userHandler, storageHandler, setti ); server.get( - { name: 'attachment', path: '/users/:user/mailboxes/:mailbox/messages/:message/attachments/:attachment' }, + { + name: 'attachment', + path: '/users/:user/mailboxes/:mailbox/messages/:message/attachments/:attachment', + summary: 'Download Attachment', + description: 'This method returns attachment file contents in binary form', + validationObjs: { + queryParams: {}, + pathParams: { + user: userId, + mailbox: mailboxId, + message: messageId, + attachment: Joi.string() + .regex(/^ATT\d+$/i) + .uppercase() + .required() + .description('ID of the Attachment') + }, + requestBody: {}, + response: { + 200: { + description: 'Success', + model: Joi.object({ + success: Joi.binary() + }) + } + } + }, + responseType: 'application/octet-stream', + tags: ['Messages'] + }, tools.responseWrapper(async (req, res) => { - const schema = Joi.object().keys({ - user: Joi.string().hex().lowercase().length(24).required(), - mailbox: Joi.string().hex().lowercase().length(24).required(), - message: Joi.number().min(1).required(), - attachment: Joi.string() - .regex(/^ATT\d+$/i) - .uppercase() - .required() + const { requestBody, queryParams, pathParams } = req.route.spec.validationObjs; + + const schema = Joi.object({ + ...requestBody, + ...queryParams, + ...pathParams }); const result = schema.validate(req.params, { @@ -2882,25 +2936,110 @@ module.exports = (db, server, messageHandler, userHandler, storageHandler, setti ); server.get( - { name: 'archived', path: '/users/:user/archived/messages' }, + { + name: 'archived', + path: '/users/:user/archived/messages', + summary: 'List archived messages', + description: 'Archive contains all recently deleted messages besides Drafts etc.', + validationObjs: { + pathParams: { + user: userId + }, + queryParams: { + limit: Joi.number().empty('').default(20).min(1).max(250).description('How many records to return'), + next: nextPageCursorSchema, + previous: previousPageCursorSchema, + order: Joi.any().empty('').allow('asc', 'desc').default('desc').description('Ordering of the records by insert date'), + includeHeaders: Joi.string() + .max(1024) + .trim() + .empty('') + .example('List-ID, MIME-Version') + .description('Comma separated list of header keys to include in the response'), + page: pageNrSchema, + sess: sessSchema, + ip: sessIPSchema + }, + requestBody: {}, + response: { + 200: { + description: 'Success', + model: Joi.object({ + success: booleanSchema.description('Indicates successful response').required(), + total: Joi.number().description('How many results were found').required(), + page: Joi.number().description('Current page number. Derived from page query argument').required(), + previousCursor: Joi.alternatives() + .try(Joi.string(), booleanSchema) + .description('Either a cursor string or false if there are not any previous results') + .required(), + nextCursor: Joi.alternatives() + .try(Joi.string(), booleanSchema) + .description('Either a cursor string or false if there are not any next results') + .required(), + specialUse: Joi.string().description('Special use. If available').required(), + results: Joi.array() + .items( + Joi.object({ + id: Joi.number().required().description('ID of the Message'), + mailbox: Joi.string().required().description('ID of the Mailbox'), + thread: Joi.string().required().description('ID of the Thread'), + threadMessageCount: Joi.number().description( + 'Amount of messages in the Thread. Included if threadCounters query argument was true' + ), + from: Address.description('Sender in From: field'), + to: Joi.array().items(Address).required().description('Recipients in To: field'), + cc: Joi.array().items(Address).required().description('Recipients in Cc: field'), + bcc: Joi.array().items(Address).required().description('Recipients in Bcc: field. Usually only available for drafts'), + messageId: Joi.string().required().description('Message ID'), + subject: Joi.string().required().description('Message subject'), + date: Joi.date().required().description('Date string from header'), + idate: Joi.date().description('Date string of receive time'), + intro: Joi.string().required().description('First 128 bytes of the message'), + attachments: booleanSchema.required().description('Does the message have attachments'), + size: Joi.number().required().description('Message size in bytes'), + seen: booleanSchema.required().description('Is this message alread seen or not'), + deleted: booleanSchema + .required() + .description( + 'Does this message have a Deleted flag (should not have as messages are automatically deleted once this flag is set)' + ), + flagged: booleanSchema.required().description('Does this message have a Flagged flag'), + draft: booleanSchema.required().description('is this message a draft'), + answered: booleanSchema.required().description('Does this message have a Answered flag'), + forwarded: booleanSchema.required().description('Does this message have a $Forwarded flag'), + references: Joi.array().items(ReferenceWithAttachments).required().description('References'), + bimi: Bimi.required().description( + 'Marks BIMI verification as passed for a domain. NB! BIMI record and logo files for the domain must be valid.' + ), + contentType: Joi.object({ + value: Joi.string().required().description('MIME type of the message, eg. "multipart/mixed"'), + params: Joi.object().required().description('An object with Content-Type params as key-value pairs') + }) + .$_setFlag('objectName', 'ContentType') + .required() + .description('Parsed Content-Type header. Usually needed to identify encrypted messages and such'), + encrypted: booleanSchema.description('Specifies whether the message is encrypted'), + metaData: Joi.object().description('Custom metadata value. Included if metaData query argument was true'), + headers: Joi.object().description('Header object keys requested with the includeHeaders argument') + }).$_setFlag('objectName', 'GetMessagesResult') + ) + .required() + .description('Message listing') + }) + } + } + }, + tags: ['Archive'] + }, tools.responseWrapper(async (req, res) => { res.charSet('utf-8'); - const schema = Joi.object().keys({ - user: Joi.string().hex().lowercase().length(24).required(), - limit: Joi.number().empty('').default(20).min(1).max(250), - next: nextPageCursorSchema, - previous: previousPageCursorSchema, - order: Joi.any().empty('').allow('asc', 'desc').default('desc'), - includeHeaders: Joi.string() - .max(1024) - .trim() - .empty('') - .example('List-ID, MIME-Version') - .description('Comma separated list of header keys to include in the response'), - page: pageNrSchema, - sess: sessSchema, - ip: sessIPSchema + const { pathParams, queryParams, requestBody } = req.route.spec.validationObjs; + + const schema = Joi.object({ + ...pathParams, + ...queryParams, + ...requestBody }); const result = schema.validate(req.params, { From 2935961f38de7011daf272ba30de70eaafca848c Mon Sep 17 00:00:00 2001 From: Nikolai Ovtsinnikov Date: Fri, 10 Nov 2023 10:20:19 +0200 Subject: [PATCH 28/30] fix description on the object issue --- lib/tools.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/tools.js b/lib/tools.js index 0e9e59d1..536599c7 100644 --- a/lib/tools.js +++ b/lib/tools.js @@ -628,7 +628,7 @@ function replaceWithRefs(reqBodyData) { if (reqBodyData.objectName) { const objectName = reqBodyData.objectName; Object.keys(reqBodyData).forEach(key => { - if (key !== '$ref') { + if (key !== '$ref' || key !== 'description') { delete reqBodyData[key]; } }); From c7404bbf97660f2e00fdcec4dcd97c357d63ad8b Mon Sep 17 00:00:00 2001 From: Nikolai Ovtsinnikov Date: Fri, 10 Nov 2023 10:46:25 +0200 Subject: [PATCH 29/30] add descriptions to fields for the schemas in the messages-schemas.js file --- lib/schemas/request/messages-schemas.js | 54 ++++++++++++++----------- 1 file changed, 31 insertions(+), 23 deletions(-) diff --git a/lib/schemas/request/messages-schemas.js b/lib/schemas/request/messages-schemas.js index d4226e60..f0844283 100644 --- a/lib/schemas/request/messages-schemas.js +++ b/lib/schemas/request/messages-schemas.js @@ -2,52 +2,60 @@ const Joi = require('joi'); const { booleanSchema } = require('../../schemas'); +const { mailboxId, messageId } = require('./general-schemas'); const Address = Joi.object({ - name: Joi.string().empty('').max(255), - address: Joi.string().email({ tlds: false }).required() + name: Joi.string().empty('').max(255).required().description('Name of the sender/recipient'), + address: Joi.string().email({ tlds: false }).required().description('Address of the sender/recipient') }).$_setFlag('objectName', 'Address'); const AddressOptionalName = Joi.object({ - name: Joi.string().empty('').max(255), - address: Joi.string().email({ tlds: false }).required() + name: Joi.string().empty('').max(255).description('Name of the sender'), + address: Joi.string().email({ tlds: false }).required().description('Address of the sender') }).$_setFlag('objectName', 'AddressOptionalName'); const AddressOptionalNameArray = Joi.array().items(AddressOptionalName); const Header = Joi.object({ - key: Joi.string().empty('').max(255), + key: Joi.string().empty('').max(255).required().description("Header key ('X-Mailer')"), value: Joi.string() .empty('') .max(100 * 1024) + .required() + .description("Header value ('My Awesome Mailing Service')") }).$_setFlag('objectName', 'Header'); const Attachment = Joi.object({ - filename: Joi.string().empty('').max(255), - contentType: Joi.string().empty('').max(255), - encoding: Joi.string().empty('').default('base64'), - contentTransferEncoding: Joi.string().empty(''), - content: Joi.string().required(), - cid: Joi.string().empty('').max(255) + filename: Joi.string().empty('').max(255).description('Attachment filename'), + contentType: Joi.string().empty('').max(255).description('MIME type for the attachment file'), + encoding: Joi.string().empty('').default('base64').description('Encoding to use to store the attachments'), + contentTransferEncoding: Joi.string().empty('').description('Transfer encoding'), + content: Joi.string().required().description('Base64 encoded attachment content'), + cid: Joi.string().empty('').max(255).description('Content-ID value if you want to reference to this attachment from HTML formatted message') }).$_setFlag('objectName', 'Attachment'); const ReferenceWithAttachments = Joi.object({ - mailbox: Joi.string().hex().lowercase().length(24).required(), - id: Joi.number().required(), - action: Joi.string().valid('reply', 'replyAll', 'forward').required(), - attachments: Joi.alternatives().try( - booleanSchema, - Joi.array().items( - Joi.string() - .regex(/^ATT\d+$/i) - .uppercase() + mailbox: mailboxId, + id: messageId, + action: Joi.string().valid('reply', 'replyAll', 'forward').required().description('Either reply, replyAll or forward'), + attachments: Joi.alternatives() + .try( + booleanSchema, + Joi.array().items( + Joi.string() + .regex(/^ATT\d+$/i) + .uppercase() + ) + ) + .required() + .description( + "If true, then includes all attachments from the original message. If it is an array of attachment ID's includes attachments from the list" ) - ) }).$_setFlag('objectName', 'ReferenceWithAttachments'); const Bimi = Joi.object({ - domain: Joi.string().domain().required(), - selector: Joi.string().empty('').max(255) + domain: Joi.string().domain().required().description('Domain name for the BIMI record. It does not have to be the same as the From address.'), + selector: Joi.string().empty('').max(255).description('Optional BIMI selector') }).$_setFlag('objectName', 'Bimi'); module.exports = { From 4687fb9dc56bcb5edfc3fc267d3852ea3ebac5fa Mon Sep 17 00:00:00 2001 From: Nikolai Ovtsinnikov Date: Mon, 20 Nov 2023 11:14:53 +0200 Subject: [PATCH 30/30] remove yarml import in tools.js --- lib/tools.js | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/tools.js b/lib/tools.js index 19bf75e1..fb96ce57 100644 --- a/lib/tools.js +++ b/lib/tools.js @@ -16,7 +16,6 @@ const ipaddr = require('ipaddr.js'); const ObjectId = require('mongodb').ObjectId; const log = require('npmlog'); const addressparser = require('nodemailer/lib/addressparser'); -const yaml = require('js-yaml'); let templates = false;