Drop support of key & value types other than string and Buffer (#179)

Author: vweevers

README.md

@@ -1,6 +1,6 @@
 # level-js
 
-> An [`abstract-leveldown`][abstract-leveldown] compliant store on top of [IndexedDB][indexeddb], which is in turn implemented on top of [LevelDB][leveldb] which brings this whole shebang full circle.
+> An [`abstract-leveldown`][abstract-leveldown] compliant store on top of [IndexedDB][indexeddb].
 
 [![level badge][level-badge]][awesome]
 [![npm](https://img.shields.io/npm/v/level-js.svg?label=&logo=npm)](https://www.npmjs.com/package/level-js)
@@ -35,13 +35,11 @@ Here are the goals of `level-js`:
 
 - Store large amounts of data in modern browsers
 - Pass the full [`abstract-leveldown`][abstract-leveldown] test suite
-- Support [`Buffer`][buffer] keys and values
-- Support all key types of IndexedDB Second Edition
-- Support all value types of the [structured clone algorithm][structured-clone-algorithm] except for `null` and `undefined`
+- Support string and [`Buffer`][buffer] keys and values
 - Be as fast as possible
-- Sync with [multilevel](https://github.com/juliangruber/multilevel) over ASCII or binary transports.
+- ~~Sync with [multilevel](https://github.com/juliangruber/multilevel) over ASCII or binary transports.~~
 
-Being `abstract-leveldown` compliant means you can use many of the [Level modules][awesome] on top of this library. For some demos of it working, see [**@brycebaril**](https://github.com/brycebaril)'s presentation [Path of the NodeBases Jedi](http://brycebaril.github.io/nodebase_jedi/#/vanilla).
+Being `abstract-leveldown` compliant means you can use many of the [Level modules][awesome] on top of this library.
 
 ## Example
 
@@ -80,120 +78,17 @@ const value = await db.get('hello')
 
 ## Type Support
 
-Unlike [`leveldown`][leveldown], `level-js` does not stringify keys or values. This means that in addition to strings and Buffers you can store almost any JavaScript type without the need for [`encoding-down`][encoding-down].
+Keys and values can be a string or [`Buffer`][buffer]. Any other type will be irreversibly stringified. The only exceptions are `null` and `undefined`. Keys and values of that type are rejected.
 
-### Values
+In order to sort string and Buffer keys the same way, for compatibility with `leveldown` and the larger ecosystem, `level-js` internally converts keys and values to binary before passing them to IndexedDB. If binary keys are not supported by the environment (like IE11) `level-js` falls back to `String(key)`.
 
-All value types of the [structured clone algorithm][structured-clone-algorithm] are supported except for `null` and `undefined`. Depending on the environment, this includes:
+If you desire non-destructive encoding (e.g. to store and retrieve numbers as-is), wrap `level-js` with [`encoding-down`][encoding-down]. Alternatively install [`level`][level] which conveniently bundles [`levelup`][levelup], `level-js` and `encoding-down`. Such an approach is also recommended if you want to achieve universal (isomorphic) behavior. For example, you could have [`leveldown`][leveldown] in a backend and `level-js` in the frontend. The `level` package does exactly that.
 
-- Number, including `NaN`, `Infinity` and `-Infinity`
-- String, Boolean, Date, RegExp, Array, Object
-- ArrayBuffer or a view thereof (typed arrays);
-- Map, Set, Blob, File, FileList, ImageData (limited support).
-
-In addition `level-js` stores [`Buffer`][buffer] values without transformation. This works in all target environments because `Buffer` is a subclass of `Uint8Array`, meaning such values can be passed to `IndexedDB` as-is.
-
-When getting or iterating binary values, regardless of whether they were stored as a `Buffer`, `ArrayBuffer` or a view thereof, values will return as a `Buffer`. This behavior can be disabled, in which case `ArrayBuffer` returns as `ArrayBuffer`, typed arrays return as typed arrays and `Buffer` returns as `Uint8Array`:
+When getting or iterating keys and values, regardless of the type with which they were stored, keys and values will return as a Buffer unless the `asBuffer`, `keyAsBuffer` or `valueAsBuffer` options are set, in which case strings are returned. Setting these options is not needed when `level-js` is wrapped with `encoding-down`, which determines the optimal return type by the chosen encoding.
 
 ```js
 db.get('key', { asBuffer: false })
-db.iterator({ valueAsBuffer: false })
-```
-
-If the environment does not support a type, it will throw an error which `level-js` catches and passes to the callbacks of `put` or `batch`. For example, IE does not support typed array values. At the time of writing, Chrome is the only browser that supports all types listed above.
-
-### Keys
-
-All key types of IndexedDB Second Edition are supported. Depending on the environment, this includes:
-
-- Number, including `Infinity` and `-Infinity`, but not `NaN`
-- Date, except invalid (`NaN`)
-- String
-- ArrayBuffer or a view thereof (typed arrays);
-- Array, except cyclical, empty and sparse arrays. Elements must be valid types themselves.
-
-In addition you can use [`Buffer`][buffer] keys, giving `level-js` the same power as implementations like `leveldown` and `memdown`. When iterating binary keys, regardless of whether they were stored as `Buffer`, `ArrayBuffer` or a view thereof, keys will return as a `Buffer`. This behavior can be disabled, in which case binary keys will always return as `ArrayBuffer`:
-
-```js
-db.iterator({ keyAsBuffer: false })
-```
-
-Note that this behavior is slightly different from values due to the way that IndexedDB works. IndexedDB stores binary _values_ using the structured clone algorithm, which preserves views, but it stores binary _keys_ as an array of octets, so that it is able to compare and sort differently typed keys.
-
-If the environment does not support a type, it will throw an error which `level-js` catches and passes to the callbacks of `get`, `put`, `del`, `batch` or an iterator. Exceptions are:
-
-- `null` and `undefined`: rejected early by `abstract-leveldown`
-- Binary and array keys: if not supported by the environment, `level-js` falls back to `String(key)`.
-
-### Normalization
-
-If you desire normalization for keys and values (e.g. to stringify numbers), wrap `level-js` with [`encoding-down`][encoding-down]. Alternatively install [`level`][level] which conveniently bundles [`levelup`][levelup], `level-js` and `encoding-down`. Such an approach is also recommended if you want to achieve universal (isomorphic) behavior or to smooth over type differences between browsers. For example, you could have [`leveldown`][leveldown] in a backend and `level-js` in the frontend. The `level` package does exactly that.
-
-Another reason you might want to use `encoding-down` is that the structured clone algorithm, while rich in types, can be slower than `JSON.stringify`.
-
-### Sort Order
-
-Unless `level-js` is wrapped with [`encoding-down`][encoding-down], IndexedDB will sort your keys in the following order:
-
-1. number (numeric)
-2. date (numeric, by epoch offset)
-3. binary (bitwise)
-4. string (lexicographic)
-5. array (componentwise).
-
-You can take advantage of this fact with `levelup` streams. For example, if your keys are dates, you can select everything greater than a specific date (let's be happy and ignore timezones for a moment):
-
-```js
-const db = levelup(leveljs('time-db'))
-
-db.createReadStream({ gt: new Date('2019-01-01') })
-  .pipe(..)
-```
-
-Or if your keys are arrays, you can do things like:
-
-```js
-const db = levelup(leveljs('books-db'))
-
-await db.put(['Roald Dahl', 'Charlie and the Chocolate Factory'], {})
-await db.put(['Roald Dahl', 'Fantastic Mr Fox'], {})
-
-// Select all books by Roald Dahl
-db.createReadStream({ gt: ['Roald Dahl'], lt: ['Roald Dahl', '\xff'] })
-  .pipe(..)
-```
-
-To achieve this on other `abstract-leveldown` implementations, wrap them with [`encoding-down`][encoding-down] and [`charwise`][charwise] (or similar).
-
-#### Known Browser Issues
-
-IE11 and Edge yield incorrect results for `{ gte: '' }` if the database contains any key types other than strings.
-
-### Buffer vs ArrayBuffer
-
-For interoperability it is recommended to use `Buffer` as your binary type. While we recognize that Node.js core modules are moving towards supporting `ArrayBuffer` and views thereof, `Buffer` remains the primary binary type in the Level ecosystem.
-
-That said: if you want to `put()` an `ArrayBuffer` you can! Just know that it will come back as a `Buffer` by default. If you want to `get()` or iterate stored `ArrayBuffer` data as an `ArrayBuffer`, you have a few options. Without `encoding-down`:
-
-```js
-const db = levelup(leveljs('mydb'))
-
-// Yields an ArrayBuffer, Buffer and ArrayBuffer
-const value1 = await db.get('key', { asBuffer: false })
-const value2 = await db.get('key')
-const value3 = value2.buffer
-```
-
-With `encoding-down` (or `level`) you can use the `id` encoding to selectively bypass encodings:
-
-```js
-const encode = require('encoding-down')
-const db = levelup(encode(leveljs('mydb'), { valueEncoding: 'binary' }))
-
-// Yields an ArrayBuffer, Buffer and ArrayBuffer
-const value1 = await db.get('key', { valueEncoding: 'id' })
-const value2 = await db.get('key')
-const value3 = value2.buffer
+db.iterator({ keyAsBuffer: false, valueAsBuffer: false })
 ```
 
 ## Install
@@ -268,22 +163,16 @@ To sustain [`Level`](https://github.com/Level) and its activities, become a back
 
 [indexeddb]: https://developer.mozilla.org/en-US/docs/Web/API/IndexedDB_API
 
-[leveldb]: https://github.com/google/leveldb
-
 [buffer]: https://nodejs.org/api/buffer.html
 
 [awesome]: https://github.com/Level/awesome
 
 [abstract-leveldown]: https://github.com/Level/abstract-leveldown
 
-[charwise]: https://github.com/dominictarr/charwise
-
 [levelup]: https://github.com/Level/levelup
 
 [leveldown]: https://github.com/Level/leveldown
 
 [level]: https://github.com/Level/level
 
 [encoding-down]: https://github.com/Level/encoding-down
-
-[structured-clone-algorithm]: https://developer.mozilla.org/en-US/docs/Web/API/Web_Workers_API/Structured_clone_algorithm

UPGRADING.md

@@ -2,6 +2,41 @@
 
 This document describes breaking changes and how to upgrade. For a complete list of changes including minor and patch releases, please refer to the [changelog][changelog].
 
+## 5.0.0 (unreleased)
+
+Support of keys & values other than strings and Buffers has been dropped. Internally `level-js` now stores keys & values as binary which solves a number of compatibility issues ([Level/memdown#186](https://github.com/Level/memdown/issues/186)). If you pass in a key or value that isn't a string or Buffer, it will be irreversibly stringified.
+
+Existing IndexedDB databases created with `level-js@4` can be read only if they used binary keys and string or binary values. Other types will come out stringified, and string keys will sort incorrectly. Use the included `upgrade()` utility to convert stored data to binary (in so far the environment supports it):
+
+```js
+var leveljs = require('level-js')
+var db = leveljs('my-db')
+
+db.open(function (err) {
+  if (err) throw err
+
+  db.upgrade(function (err) {
+    if (err) throw err
+  })
+})
+```
+
+Or with (the upcoming release of) `level`:
+
+```js
+var level = require('level')
+var reachdown = require('reachdown')
+var db = level('my-db')
+
+db.open(function (err) {
+  if (err) throw err
+
+  reachdown(db, 'level-js').upgrade(function (err) {
+    if (err) throw err
+  })
+})
+```
+
 ## 4.0.0
 
 This is an upgrade to `abstract-leveldown@6` which solves long-standing issues around serialization and type support.

index.js

@@ -7,7 +7,8 @@ module.exports = Level
 var AbstractLevelDOWN = require('abstract-leveldown').AbstractLevelDOWN
 var inherits = require('inherits')
 var Iterator = require('./iterator')
-var mixedToBuffer = require('./util/mixed-to-buffer')
+var serialize = require('./util/serialize')
+var deserialize = require('./util/deserialize')
 var setImmediate = require('./util/immediate')
 var support = require('./util/support')
 
@@ -25,13 +26,17 @@ function Level (location, opts) {
   this.location = location
   this.prefix = opts.prefix || DEFAULT_PREFIX
   this.version = parseInt(opts.version || 1, 10)
+
+  // Experimental, do not externally rely on this object yet.
+  // See Level/community#42.
+  this.supports = {
+    bufferKeys: support.bufferKeys(indexedDB)
+  }
 }
 
 inherits(Level, AbstractLevelDOWN)
 
-// Detect binary and array key support (IndexedDB Second Edition)
-Level.binaryKeys = support.binaryKeys(indexedDB)
-Level.arrayKeys = support.arrayKeys(indexedDB)
+Level.prototype.type = 'level-js'
 
 Level.prototype._open = function (options, callback) {
   var req = indexedDB.open(this.prefix + this.location, this.version)
@@ -93,11 +98,7 @@ Level.prototype._get = function (key, options, callback) {
       return callback(new Error('NotFound'))
     }
 
-    if (options.asBuffer) {
-      value = mixedToBuffer(value)
-    }
-
-    callback(null, value)
+    callback(null, deserialize(value, options.asBuffer))
   })
 }
 
@@ -131,27 +132,12 @@ Level.prototype._put = function (key, value, options, callback) {
   this.await(req, callback)
 }
 
-// Valid key types in IndexedDB Second Edition:
-//
-// - Number, except NaN. Includes Infinity and -Infinity
-// - Date, except invalid (NaN)
-// - String
-// - ArrayBuffer or a view thereof (typed arrays). In level-js we also support
-//   Buffer (which is an Uint8Array) (and the primary binary type of Level).
-// - Array, except cyclical and empty (e.g. Array(10)). Elements must be valid
-//   types themselves.
 Level.prototype._serializeKey = function (key) {
-  if (Buffer.isBuffer(key)) {
-    return Level.binaryKeys ? key : key.toString()
-  } else if (Array.isArray(key)) {
-    return Level.arrayKeys ? key.map(this._serializeKey, this) : String(key)
-  } else {
-    return key
-  }
+  return serialize(key, this.supports.bufferKeys)
 }
 
 Level.prototype._serializeValue = function (value) {
-  return value
+  return serialize(value, true)
 }
 
 Level.prototype._iterator = function (options) {
@@ -200,6 +186,55 @@ Level.prototype._close = function (callback) {
   setImmediate(callback)
 }
 
+// NOTE: remove in a next major release
+Level.prototype.upgrade = function (callback) {
+  if (this.status !== 'open') {
+    return setImmediate(function () {
+      callback(new Error('cannot upgrade() before open()'))
+    })
+  }
+
+  var it = this.iterator()
+  var batchOptions = {}
+  var self = this
+
+  it._deserializeKey = it._deserializeValue = identity
+  next()
+
+  function next (err) {
+    if (err) return finish(err)
+    it.next(each)
+  }
+
+  function each (err, key, value) {
+    if (err || key === undefined) {
+      return finish(err)
+    }
+
+    var newKey = self._serializeKey(deserialize(key, true))
+    var newValue = self._serializeValue(deserialize(value, true))
+
+    // To bypass serialization on the old key, use _batch() instead of batch().
+    // NOTE: if we disable snapshotting (#86) this could lead to a loop of
+    // inserting and then iterating those same entries, because the new keys
+    // possibly sort after the old keys.
+    self._batch([
+      { type: 'del', key: key },
+      { type: 'put', key: newKey, value: newValue }
+    ], batchOptions, next)
+  }
+
+  function finish (err) {
+    it.end(function (err2) {
+      callback(err || err2)
+    })
+  }
+
+  function identity (data) {
+    return data
+  }
+}
+
 Level.destroy = function (location, prefix, callback) {
   if (typeof prefix === 'function') {
     callback = prefix

iterator.js

@@ -5,7 +5,7 @@
 var inherits = require('inherits')
 var AbstractIterator = require('abstract-leveldown').AbstractIterator
 var ltgt = require('ltgt')
-var mixedToBuffer = require('./util/mixed-to-buffer')
+var deserialize = require('./util/deserialize')
 var setImmediate = require('./util/immediate')
 var noop = function () {}
 
@@ -23,6 +23,8 @@ function Iterator (db, location, options) {
   this._error = null
   this._transaction = null
 
+  this._keys = options.keys
+  this._values = options.values
   this._keyAsBuffer = options.keyAsBuffer
   this._valueAsBuffer = options.valueAsBuffer
 
@@ -126,8 +128,17 @@ Iterator.prototype._next = function (callback) {
     var key = this._cache.shift()
     var value = this._cache.shift()
 
-    if (this._keyAsBuffer) key = mixedToBuffer(key)
-    if (this._valueAsBuffer) value = mixedToBuffer(value)
+    if (this._keys && key !== undefined) {
+      key = this._deserializeKey(key, this._keyAsBuffer)
+    } else {
+      key = undefined
+    }
+
+    if (this._values && value !== undefined) {
+      value = this._deserializeValue(value, this._valueAsBuffer)
+    } else {
+      value = undefined
+    }
 
     setImmediate(function () {
       callback(null, key, value)
@@ -139,6 +150,10 @@ Iterator.prototype._next = function (callback) {
   }
 }
 
+// Exposed for the v4 to v5 upgrade utility
+Iterator.prototype._deserializeKey = deserialize
+Iterator.prototype._deserializeValue = deserialize
+
 Iterator.prototype._end = function (callback) {
   if (this._aborted || this._completed) {
     var err = this._error