diff --git a/README.md b/README.md index 4ceef896..2920b9cf 100644 --- a/README.md +++ b/README.md @@ -319,6 +319,129 @@ This error will occur when the request had a `Content-Encoding` header that contained an unsupported encoding. The encoding is contained in the message as well as in the `encoding` property. The `status` property is set to `415`. +### bodyParser.generic(options) + +Returns middleware that only parses `generic` bodies. This parser accepts +any encoding of the body and supports automatic inflation of `gzip` +and `deflate` encodings. + +A new `body` object containing the parsed data is populated on the `request` +object after the middleware (i.e. `req.body`). This object will contain the result +from the parser you specify. + +#### Options + +The `generic` function takes an option `options` object that may contain +any of the following key: + +##### parser (required) + +This must be a function which takes the `body` optionally followed by `options`. + +For example: + +```js +function (body, options) { + return JSON.parse(body, options.reviver); +} +``` + +##### parserOptions + +This should be an object containing any options you wish to pass through to the parser. + +##### inflate + +When set to `true`, then deflated (compressed) bodies will be inflated; when +`false`, deflated bodies are rejected. Defaults to `true`. + +##### limit + +Controls the maximum request body size. If this is a number, then the value +specifies the number of bytes; if it is a string, the value is passed to the +[bytes](https://www.npmjs.com/package/bytes) library for parsing. Defaults +to `'100kb'`. + +##### parameterLimit + +The `parameterLimit` option controls the maximum number of parameters that +are allowed in the URL-encoded data. If a request contains more parameters +than this value, a 413 will be returned to the client. Defaults to `1000`. + +##### type + +The `type` option is used to determine what media type the middleware will +parse. This option can be a function or a string. If a string, `type` option +is passed directly to the [type-is](https://www.npmjs.org/package/type-is#readme) +library and this can be an extension name (like `urlencoded`), a mime type (like +`application/x-www-form-urlencoded`), or a mime type with a wildcard (like +`*/x-www-form-urlencoded`). If a function, the `type` option is called as +`fn(req)` and the request is parsed if it returns a truthy value. Defaults +to `application/x-www-form-urlencoded`. + +##### verify + +The `verify` option, if supplied, is called as `verify(req, res, buf, encoding)`, +where `buf` is a `Buffer` of the raw request body and `encoding` is the +encoding of the request. The parsing can be aborted by throwing an error. + +## Errors + +The middlewares provided by this module create errors depending on the error +condition during parsing. The errors will typically have a `status` property +that contains the suggested HTTP response code and a `body` property containing +the read body, if available. + +The following are the common errors emitted, though any error can come through +for various reasons. + +### content encoding unsupported + +This error will occur when the request had a `Content-Encoding` header that +contained an encoding but the "inflation" option was set to `false`. The +`status` property is set to `415`. + +### request aborted + +This error will occur when the request is aborted by the client before reading +the body has finished. The `received` property will be set to the number of +bytes received before the request was aborted and the `expected` property is +set to the number of expected bytes. The `status` property is set to `400`. + +### request entity too large + +This error will occur when the request body's size is larger than the "limit" +option. The `limit` property will be set to the byte limit and the `length` +property will be set to the request body's length. The `status` property is +set to `413`. + +### request size did not match content length + +This error will occur when the request's length did not match the length from +the `Content-Length` header. This typically occurs when the request is malformed, +typically when the `Content-Length` header was calculated based on characters +instead of bytes. The `status` property is set to `400`. + +### stream encoding should not be set + +This error will occur when something called the `req.setEncoding` method prior +to this middleware. This module operates directly on bytes only and you cannot +call `req.setEncoding` when using this module. The `status` property is set to +`500`. + +### unsupported charset "BOGUS" + +This error will occur when the request had a charset parameter in the +`Content-Type` header, but the `iconv-lite` module does not support it OR the +parser does not support it. The charset is contained in the message as well +as in the `charset` property. The `status` property is set to `415`. + +### unsupported content encoding "bogus" + +This error will occur when the request had a `Content-Encoding` header that +contained an unsupported encoding. The encoding is contained in the message +as well as in the `encoding` property. The `status` property is set to `415`. + ## Examples ### Express/Connect top-level generic diff --git a/index.js b/index.js index 93c3a1ff..905502a7 100644 --- a/index.js +++ b/index.js @@ -27,6 +27,7 @@ var parsers = Object.create(null) * @property {function} raw * @property {function} text * @property {function} urlencoded + * @property {function} generic */ /** @@ -48,6 +49,17 @@ Object.defineProperty(exports, 'json', { get: createParserGetter('json') }) +/** + * Generic parser. + * @public + */ + +Object.defineProperty(exports, 'generic', { + configurable: true, + enumerable: true, + get: createParserGetter('generic') +}) + /** * Raw parser. * @public @@ -150,6 +162,9 @@ function loadParser (parserName) { case 'urlencoded': parser = require('./lib/types/urlencoded') break + case 'generic': + parser = require('./lib/types/generic') + break } // store to prevent invoking require() diff --git a/lib/types/generic.js b/lib/types/generic.js new file mode 100644 index 00000000..6f3a7db7 --- /dev/null +++ b/lib/types/generic.js @@ -0,0 +1,120 @@ +/*! + * body-parser + * Copyright(c) 2014 Jonathan Ong + * Copyright(c) 2014-2015 Douglas Christopher Wilson + * MIT Licensed + */ + +'use strict' + +/** + * Module dependencies. + * @private + */ + +var bytes = require('bytes') +var contentType = require('content-type') +var debug = require('debug')('body-parser:generic') +var read = require('../read') +var typeis = require('type-is') + +/** + * Module exports. + */ + +module.exports = generic + +/** + * Create a middleware to parse generic bodies. + * + * @param {object} [options] + * @return {function} + * @public + */ + +function generic (opts) { + var inflate = opts.inflate !== false + var limit = typeof opts.limit !== 'number' + ? bytes.parse(opts.limit || '100kb') + : opts.limit + var type = opts.type || 'application/x-www-form-urlencoded' + var verify = opts.verify || false + + if (verify !== false && typeof verify !== 'function') { + throw new TypeError('option verify must be function') + } + + // create the appropriate type checking function + var shouldParse = typeof type !== 'function' + ? typeChecker(type) + : type + + function parse (body) { + return body.length + ? opts.parser(body, opts.parserOptions || {}) + : {} + } + + return function genericParser (req, res, next) { + if (req._body) { + debug('body already parsed') + next() + return + } + + req.body = req.body || {} + + // skip requests without bodies + if (!typeis.hasBody(req)) { + debug('skip empty body') + next() + return + } + + debug('content-type %j', req.headers['content-type']) + + // determine if request should be parsed + if (!shouldParse(req)) { + debug('skip parsing') + next() + return + } + + // read + read(req, res, next, parse, debug, { + debug: debug, + encoding: getCharset(req) || 'utf-8', + inflate: inflate, + limit: limit, + verify: verify + }) + } +} + +/** + * Get the charset of a request. + * + * @param {object} req + * @api private + */ + +function getCharset (req) { + try { + return contentType.parse(req).parameters.charset.toLowerCase() + } catch (e) { + return undefined + } +} + +/** + * Get the simple type checker. + * + * @param {string} type + * @return {function} + */ + +function typeChecker (type) { + return function checkType (req) { + return Boolean(typeis(req, type)) + } +} diff --git a/package.json b/package.json index c4598256..42991268 100644 --- a/package.json +++ b/package.json @@ -28,6 +28,7 @@ "istanbul": "0.4.4", "methods": "1.1.2", "mocha": "2.5.3", + "object-assign": "4.1.0", "supertest": "1.1.0" }, "files": [ diff --git a/test/generic.js b/test/generic.js new file mode 100644 index 00000000..c83fcbe3 --- /dev/null +++ b/test/generic.js @@ -0,0 +1,372 @@ + +var assert = require('assert') +var http = require('http') +var request = require('supertest') +var objectAssign = require('object-assign') + +var bodyParser = require('..') + +describe('bodyParser.generic()', function () { + var server + before(function () { + server = createServer() + }) + + it('should parse x-www-form-urlencoded', function (done) { + request(server) + .post('/') + .set('Content-Type', 'application/x-www-form-urlencoded') + .send('user=tobi') + .expect(200, '{"user":"tobi"}', done) + }) + + it('should 400 when invalid content-length', function (done) { + var genericParser = bodyParser.generic({ + parser: require('qs').parse + }) + var server = createServer(function (req, res, next) { + req.headers['content-length'] = '20' // bad length + genericParser(req, res, next) + }) + + request(server) + .post('/') + .set('Content-Type', 'application/x-www-form-urlencoded') + .send('str=') + .expect(400, /content length/, done) + }) + + it('should handle Content-Length: 0', function (done) { + request(server) + .post('/') + .set('Content-Type', 'application/x-www-form-urlencoded') + .set('Content-Length', '0') + .send('') + .expect(200, '{}', done) + }) + + it('should handle empty message-body', function (done) { + request(createServer({ limit: '1kb' })) + .post('/') + .set('Content-Type', 'application/x-www-form-urlencoded') + .set('Transfer-Encoding', 'chunked') + .send('') + .expect(200, '{}', done) + }) + + it('should handle duplicated middleware', function (done) { + var genericParser = bodyParser.generic({ + parser: require('qs').parse + }) + var server = createServer(function (req, res, next) { + genericParser(req, res, function (err) { + if (err) return next(err) + genericParser(req, res, next) + }) + }) + + request(server) + .post('/') + .set('Content-Type', 'application/x-www-form-urlencoded') + .send('user=tobi') + .expect(200, '{"user":"tobi"}', done) + }) + + describe('with parserOptions', function () { + var server + before(function () { + server = createServer({ + parserOptions: { + allowDots: true + } + }) + }) + + it('should pass on parserOptions', function (done) { + request(server) + .post('/') + .set('Content-Type', 'application/x-www-form-urlencoded') + .send('users.0.name=tobi') + .expect(200, '{"users":[{"name":"tobi"}]}', done) + }) + }) + + describe('with inflate option', function () { + describe('when false', function () { + var server + before(function () { + server = createServer({ inflate: false }) + }) + + it('should not accept content-encoding', function (done) { + var test = request(server).post('/') + test.set('Content-Encoding', 'gzip') + test.set('Content-Type', 'application/x-www-form-urlencoded') + test.write(new Buffer('1f8b080000000000000bcb4bcc4db57db16e170099a4bad608000000', 'hex')) + test.expect(415, 'content encoding unsupported', done) + }) + }) + + describe('when true', function () { + var server + before(function () { + server = createServer({ inflate: true }) + }) + + it('should accept content-encoding', function (done) { + var test = request(server).post('/') + test.set('Content-Encoding', 'gzip') + test.set('Content-Type', 'application/x-www-form-urlencoded') + test.write(new Buffer('1f8b080000000000000bcb4bcc4db57db16e170099a4bad608000000', 'hex')) + test.expect(200, '{"name":"论"}', done) + }) + }) + }) + + describe('with limit option', function () { + it('should 413 when over limit with Content-Length', function (done) { + var buf = allocBuffer(1024, '.') + request(createServer({ limit: '1kb' })) + .post('/') + .set('Content-Type', 'application/x-www-form-urlencoded') + .set('Content-Length', '1028') + .send('str=' + buf.toString()) + .expect(413, done) + }) + + it('should 413 when over limit with chunked encoding', function (done) { + var buf = allocBuffer(1024, '.') + var server = createServer({ limit: '1kb' }) + var test = request(server).post('/') + test.set('Content-Type', 'application/x-www-form-urlencoded') + test.set('Transfer-Encoding', 'chunked') + test.write('str=') + test.write(buf.toString()) + test.expect(413, done) + }) + + it('should accept number of bytes', function (done) { + var buf = allocBuffer(1024, '.') + request(createServer({ limit: 1024 })) + .post('/') + .set('Content-Type', 'application/x-www-form-urlencoded') + .send('str=' + buf.toString()) + .expect(413, done) + }) + + it('should not change when options altered', function (done) { + var buf = allocBuffer(1024, '.') + var options = { limit: '1kb' } + var server = createServer(options) + + options.limit = '100kb' + + request(server) + .post('/') + .set('Content-Type', 'application/x-www-form-urlencoded') + .send('str=' + buf.toString()) + .expect(413, done) + }) + + it('should not hang response', function (done) { + var buf = allocBuffer(10240, '.') + var server = createServer({ limit: '8kb' }) + var test = request(server).post('/') + test.set('Content-Type', 'application/x-www-form-urlencoded') + test.write(buf) + test.write(buf) + test.write(buf) + test.expect(413, done) + }) + }) + + describe('with type option', function () { + describe('when "application/vnd.x-www-form-urlencoded"', function () { + var server + before(function () { + server = createServer({ type: 'application/vnd.x-www-form-urlencoded' }) + }) + + it('should parse for custom type', function (done) { + request(server) + .post('/') + .set('Content-Type', 'application/vnd.x-www-form-urlencoded') + .send('user=tobi') + .expect(200, '{"user":"tobi"}', done) + }) + + it('should ignore standard type', function (done) { + request(server) + .post('/') + .set('Content-Type', 'application/x-www-form-urlencoded') + .send('user=tobi') + .expect(200, '{}', done) + }) + }) + + describe('when a function', function () { + it('should parse when truthy value returned', function (done) { + var server = createServer({ type: accept }) + + function accept (req) { + return req.headers['content-type'] === 'application/vnd.something' + } + + request(server) + .post('/') + .set('Content-Type', 'application/vnd.something') + .send('user=tobi') + .expect(200, '{"user":"tobi"}', done) + }) + + it('should work without content-type', function (done) { + var server = createServer({ type: accept }) + + function accept (req) { + return true + } + + var test = request(server).post('/') + test.write('user=tobi') + test.expect(200, '{"user":"tobi"}', done) + }) + + it('should not invoke without a body', function (done) { + var server = createServer({ type: accept }) + + function accept (req) { + throw new Error('oops!') + } + + request(server) + .get('/') + .expect(200, done) + }) + }) + }) + + describe('with verify option', function () { + it('should assert value if function', function () { + assert.throws(createServer.bind(null, { verify: 'lol' }), + /TypeError: option verify must be function/) + }) + + it('should error from verify', function (done) { + var server = createServer({verify: function (req, res, buf) { + if (buf[0] === 0x20) throw new Error('no leading space') + }}) + + request(server) + .post('/') + .set('Content-Type', 'application/x-www-form-urlencoded') + .send(' user=tobi') + .expect(403, 'no leading space', done) + }) + + it('should allow custom codes', function (done) { + var server = createServer({verify: function (req, res, buf) { + if (buf[0] !== 0x20) return + var err = new Error('no leading space') + err.status = 400 + throw err + }}) + + request(server) + .post('/') + .set('Content-Type', 'application/x-www-form-urlencoded') + .send(' user=tobi') + .expect(400, 'no leading space', done) + }) + + it('should allow pass-through', function (done) { + var server = createServer({verify: function (req, res, buf) { + if (buf[0] === 0x5b) throw new Error('no arrays') + }}) + + request(server) + .post('/') + .set('Content-Type', 'application/x-www-form-urlencoded') + .send('user=tobi') + .expect(200, '{"user":"tobi"}', done) + }) + }) + + describe('encoding', function () { + var server + before(function () { + server = createServer({ limit: '10kb' }) + }) + + it('should parse without encoding', function (done) { + var test = request(server).post('/') + test.set('Content-Type', 'application/x-www-form-urlencoded') + test.write(new Buffer('6e616d653de8aeba', 'hex')) + test.expect(200, '{"name":"论"}', done) + }) + + it('should support identity encoding', function (done) { + var test = request(server).post('/') + test.set('Content-Encoding', 'identity') + test.set('Content-Type', 'application/x-www-form-urlencoded') + test.write(new Buffer('6e616d653de8aeba', 'hex')) + test.expect(200, '{"name":"论"}', done) + }) + + it('should support gzip encoding', function (done) { + var test = request(server).post('/') + test.set('Content-Encoding', 'gzip') + test.set('Content-Type', 'application/x-www-form-urlencoded') + test.write(new Buffer('1f8b080000000000000bcb4bcc4db57db16e170099a4bad608000000', 'hex')) + test.expect(200, '{"name":"论"}', done) + }) + + it('should support deflate encoding', function (done) { + var test = request(server).post('/') + test.set('Content-Encoding', 'deflate') + test.set('Content-Type', 'application/x-www-form-urlencoded') + test.write(new Buffer('789ccb4bcc4db57db16e17001068042f', 'hex')) + test.expect(200, '{"name":"论"}', done) + }) + + it('should be case-insensitive', function (done) { + var test = request(server).post('/') + test.set('Content-Encoding', 'GZIP') + test.set('Content-Type', 'application/x-www-form-urlencoded') + test.write(new Buffer('1f8b080000000000000bcb4bcc4db57db16e170099a4bad608000000', 'hex')) + test.expect(200, '{"name":"论"}', done) + }) + + it('should fail on unknown encoding', function (done) { + var test = request(server).post('/') + test.set('Content-Encoding', 'nulls') + test.set('Content-Type', 'application/x-www-form-urlencoded') + test.write(new Buffer('000000000000', 'hex')) + test.expect(415, 'unsupported content encoding "nulls"', done) + }) + }) +}) + +function allocBuffer (size, fill) { + if (Buffer.alloc) { + return Buffer.alloc(size, fill) + } + + var buf = new Buffer(size) + buf.fill(fill) + return buf +} + +function createServer (opts) { + var _bodyParser = typeof opts !== 'function' + ? bodyParser.generic(objectAssign({}, { + parser: require('qs').parse + }, opts)) + : opts + + return http.createServer(function (req, res) { + _bodyParser(req, res, function (err) { + res.statusCode = err ? (err.status || 500) : 200 + res.end(err ? err.message : JSON.stringify(req.body)) + }) + }) +}