msgpack: support error extension type

Tarantool supports error extension type since version 2.4.1 [1],
encoding was introduced in Tarantool 2.10.0 [2]. This patch introduced
the support of Tarantool error extension type in msgpack decoders and
encoders.

Tarantool error extension type objects are decoded to
`tarantool.BoxError` type. `tarantool.BoxError` may be encoded to
Tarantool error extension type objects.

Error extension type internals are the same as errors extra information:
the only difference is that extra information is encoded as a separate
error dictionary field and error extension type objects is encoded as
MessagePack extension type objects.

The only way to receive an error extension type object from Tarantool is
to receive an explicitly built `box.error` object: either from
`return box.error.new(...)` or a tuple with it. All errors raised within
Tarantool (including those raised with `box.error(...)`) are encoded
based on the same rules as simple errors due to backward compatibility.

It is possible to create error extension type objects with Python code,
but it not likely to be really useful since most of their fields is
computed on error initialization on server side (even for custom error
types):

```
tarantool.BoxError([
    tarantool.BoxErrorStackUnit(
        type='ClientError',
        file='[string "                     local err = box.error.ne..."]',
        line=1,
        message='Unknown error',
        errno=0,
        errcode=0,
    )
])
```

1. tarantool/tarantool#4398
2. tarantool/tarantool#6433

Part of #232

Author: DifferentialOrange

Diff for: doc/dev-guide.rst

@@ -83,6 +83,8 @@ they are represented with in-built and custom types:
     +-----------------------------+----+-------------+----+-----------------------------+
     | :obj:`uuid.UUID`            | -> | `UUID`_     | -> | :obj:`uuid.UUID`            |
     +-----------------------------+----+-------------+----+-----------------------------+
+    | :class:`tarantool.BoxError` | -> | `ERROR`_    | -> | :class:`tarantool.BoxError` |
+    +-----------------------------+----+-------------+----+-----------------------------+
     | :class:`tarantool.Datetime` | -> | `DATETIME`_ | -> | :class:`tarantool.Datetime` |
     +-----------------------------+----+-------------+----+-----------------------------+
     | :class:`tarantool.Interval` | -> | `INTERVAL`_ | -> | :class:`tarantool.Interval` |
@@ -109,5 +111,6 @@ and iterate through it as with any other serializable object.
 .. _extension types: https://www.tarantool.io/en/doc/latest/dev_guide/internals/msgpack_extensions/
 .. _DECIMAL: https://www.tarantool.io/en/doc/latest/dev_guide/internals/msgpack_extensions/#the-decimal-type
 .. _UUID: https://www.tarantool.io/en/doc/latest/dev_guide/internals/msgpack_extensions/#the-uuid-type
+.. _ERROR: https://www.tarantool.io/en/doc/latest/dev_guide/internals/msgpack_extensions/#the-error-type
 .. _DATETIME: https://www.tarantool.io/en/doc/latest/dev_guide/internals/msgpack_extensions/#the-datetime-type
 .. _INTERVAL: https://www.tarantool.io/en/doc/latest/dev_guide/internals/msgpack_extensions/#the-interval-type

Diff for: tarantool/__init__.py

@@ -42,6 +42,8 @@
 
 from tarantool.connection_pool import ConnectionPool, Mode
 
+from tarantool.types import BoxError
+
 __version__ = "0.9.0"
 
 
@@ -136,4 +138,4 @@ def connectmesh(addrs=({'host': 'localhost', 'port': 3301},), user=None,
 __all__ = ['connect', 'Connection', 'connectmesh', 'MeshConnection', 'Schema',
            'Error', 'DatabaseError', 'NetworkError', 'NetworkWarning',
            'SchemaError', 'dbapi', 'Datetime', 'Interval', 'IntervalAdjust',
-           'ConnectionPool', 'Mode']
+           'ConnectionPool', 'Mode', 'BoxError',]

Diff for: tarantool/const.py

@@ -129,4 +129,4 @@
 # Tarantool 2.10 protocol version is 3
 CONNECTOR_IPROTO_VERSION = 3
 # List of connector-supported features
-CONNECTOR_FEATURES = []
+CONNECTOR_FEATURES = [IPROTO_FEATURE_ERROR_EXTENSION,]

Diff for: tarantool/msgpack_ext/error.py

@@ -0,0 +1,52 @@
+"""
+Tarantool `error`_ extension type support module.
+
+Refer to :mod:`~tarantool.msgpack_ext.types.error`.
+
+.. _error: https://www.tarantool.io/en/doc/latest/dev_guide/internals/msgpack_extensions/#the-error-type
+"""
+
+from tarantool.types import (
+    encode_box_error,
+    decode_box_error,
+)
+
+EXT_ID = 3
+"""
+`error`_ type id.
+"""
+
+def encode(obj, packer):
+    """
+    Encode an error object.
+
+    :param obj: Error to encode.
+    :type obj: :class:`tarantool.BoxError`
+
+    :param packer: msgpack packer to encode error dictionary.
+    :type packer: :class:`msgpack.Packer`
+
+    :return: Encoded error.
+    :rtype: :obj:`bytes`
+    """
+
+    err_map = encode_box_error(obj)
+    return packer.pack(err_map)
+
+def decode(data, unpacker):
+    """
+    Decode an error object.
+
+    :param obj: Error to decode.
+    :type obj: :obj:`bytes`
+
+    :param unpacker: msgpack unpacker to decode error dictionary.
+    :type unpacker: :class:`msgpack.Unpacker`
+
+    :return: Decoded error.
+    :rtype: :class:`tarantool.BoxError`
+    """
+
+    unpacker.feed(data)
+    err_map = unpacker.unpack()
+    return decode_box_error(err_map)

Diff for: tarantool/msgpack_ext/packer.py

@@ -8,28 +8,36 @@
 from uuid import UUID
 from msgpack import ExtType
 
+from tarantool.types import BoxError
 from tarantool.msgpack_ext.types.datetime import Datetime
 from tarantool.msgpack_ext.types.interval import Interval
 
 import tarantool.msgpack_ext.decimal as ext_decimal
 import tarantool.msgpack_ext.uuid as ext_uuid
+import tarantool.msgpack_ext.error as ext_error
 import tarantool.msgpack_ext.datetime as ext_datetime
 import tarantool.msgpack_ext.interval as ext_interval
 
 encoders = [
     {'type': Decimal,  'ext': ext_decimal },
     {'type': UUID,     'ext': ext_uuid    },
+    {'type': BoxError, 'ext': ext_error   },
     {'type': Datetime, 'ext': ext_datetime},
     {'type': Interval, 'ext': ext_interval},
 ]
 
-def default(obj):
+def default(obj, packer=None):
     """
     :class:`msgpack.Packer` encoder.
 
     :param obj: Object to encode.
     :type obj: :class:`decimal.Decimal` or :class:`uuid.UUID` or
-        :class:`tarantool.Datetime` or :class:`tarantool.Interval`
+         or :class:`tarantool.BoxError` or :class:`tarantool.Datetime`
+         or :class:`tarantool.Interval`
+
+    :param packer: msgpack packer to work with common types
+        (like dictionary in extended error payload)
+    :type packer: :class:`msgpack.Packer`, optional
 
     :return: Encoded value.
     :rtype: :class:`msgpack.ExtType`
@@ -39,5 +47,5 @@ def default(obj):
 
     for encoder in encoders:
         if isinstance(obj, encoder['type']):
-            return ExtType(encoder['ext'].EXT_ID, encoder['ext'].encode(obj))
+            return ExtType(encoder['ext'].EXT_ID, encoder['ext'].encode(obj, packer))
     raise TypeError("Unknown type: %r" % (obj,))

Diff for: tarantool/msgpack_ext/unpacker.py

@@ -6,17 +6,19 @@
 
 import tarantool.msgpack_ext.decimal as ext_decimal
 import tarantool.msgpack_ext.uuid as ext_uuid
+import tarantool.msgpack_ext.error as ext_error
 import tarantool.msgpack_ext.datetime as ext_datetime
 import tarantool.msgpack_ext.interval as ext_interval
 
 decoders = {
     ext_decimal.EXT_ID : ext_decimal.decode ,
     ext_uuid.EXT_ID    : ext_uuid.decode    ,
+    ext_error.EXT_ID   : ext_error.decode   ,
     ext_datetime.EXT_ID: ext_datetime.decode,
     ext_interval.EXT_ID: ext_interval.decode,
 }
 
-def ext_hook(code, data):
+def ext_hook(code, data, unpacker=None):
     """
     :class:`msgpack.Unpacker` decoder.
 
@@ -26,13 +28,18 @@ def ext_hook(code, data):
     :param data: MessagePack extension type data.
     :type data: :obj:`bytes`
 
+    :param unpacker: msgpack unpacker to work with common types
+        (like dictionary in extended error payload)
+    :type unpacker: :class:`msgpack.Unpacker`, optional
+
     :return: Decoded value.
     :rtype: :class:`decimal.Decimal` or :class:`uuid.UUID` or
-        :class:`tarantool.Datetime` or :class:`tarantool.Interval`
+         or :class:`tarantool.BoxError` or :class:`tarantool.Datetime`
+         or :class:`tarantool.Interval`
 
     :raise: :exc:`NotImplementedError`
     """
 
     if code in decoders:
-        return decoders[code](data)
+        return decoders[code](data, unpacker)
     raise NotImplementedError("Unknown msgpack extension type code %d" % (code,))

Diff for: tarantool/request.py

@@ -63,6 +63,67 @@
 
 from tarantool.msgpack_ext.packer import default as packer_default
 
+def build_packer(conn):
+    """
+    Build packer to pack request.
+
+    :param conn: Request sender.
+    :type conn: :class:`~tarantool.Connection`
+
+    :rtype: :class:`msgpack.Packer`
+    """
+
+    packer_kwargs = dict()
+
+    # use_bin_type=True is default since msgpack-1.0.0.
+    #
+    # The option controls whether to pack binary (non-unicode)
+    # string values as mp_bin or as mp_str.
+    #
+    # The default behaviour of the Python 3 connector (since
+    # default encoding is "utf-8") is to pack bytes as mp_bin
+    # and Unicode strings as mp_str. encoding=None mode must
+    # be used to work with non-utf strings.
+    #
+    # encoding = 'utf-8'
+    #
+    # Python 3 -> Tarantool          -> Python 3
+    # str      -> mp_str (string)    -> str
+    # bytes    -> mp_bin (varbinary) -> bytes
+    #
+    # encoding = None
+    #
+    # Python 3 -> Tarantool          -> Python 3
+    # bytes    -> mp_str (string)    -> bytes
+    # str      -> mp_str (string)    -> bytes
+    #             mp_bin (varbinary) -> bytes
+    #
+    # msgpack-0.5.0 (and only this version) warns when the
+    # option is unset:
+    #
+    #  | FutureWarning: use_bin_type option is not specified.
+    #  | Default value of the option will be changed in future
+    #  | version.
+    #
+    # The option is supported since msgpack-0.4.0, so we can
+    # just always set it for all msgpack versions to get rid
+    # of the warning on msgpack-0.5.0 and to keep our
+    # behaviour on msgpack-1.0.0.
+    if conn.encoding is None:
+        packer_kwargs['use_bin_type'] = False
+    else:
+        packer_kwargs['use_bin_type'] = True
+
+    # We need configured packer to work with error extention
+    # type payload, but module do not provide access to self
+    # inside extension type packers.
+    packer_no_ext = msgpack.Packer(**packer_kwargs)
+    default = lambda obj: packer_default(obj, packer_no_ext)
+    packer_kwargs['default'] = default
+
+    return msgpack.Packer(**packer_kwargs)
+
+
 class Request(object):
     """
     Represents a single request to the server in compliance with the
@@ -87,50 +148,7 @@ def __init__(self, conn):
         self._body = ''
         self.response_class = Response
 
-        packer_kwargs = dict()
-
-        # use_bin_type=True is default since msgpack-1.0.0.
-        #
-        # The option controls whether to pack binary (non-unicode)
-        # string values as mp_bin or as mp_str.
-        #
-        # The default behaviour of the Python 3 connector (since
-        # default encoding is "utf-8") is to pack bytes as mp_bin
-        # and Unicode strings as mp_str. encoding=None mode must
-        # be used to work with non-utf strings.
-        #
-        # encoding = 'utf-8'
-        #
-        # Python 3 -> Tarantool          -> Python 3
-        # str      -> mp_str (string)    -> str
-        # bytes    -> mp_bin (varbinary) -> bytes
-        #
-        # encoding = None
-        #
-        # Python 3 -> Tarantool          -> Python 3
-        # bytes    -> mp_str (string)    -> bytes
-        # str      -> mp_str (string)    -> bytes
-        #             mp_bin (varbinary) -> bytes
-        #
-        # msgpack-0.5.0 (and only this version) warns when the
-        # option is unset:
-        #
-        #  | FutureWarning: use_bin_type option is not specified.
-        #  | Default value of the option will be changed in future
-        #  | version.
-        #
-        # The option is supported since msgpack-0.4.0, so we can
-        # just always set it for all msgpack versions to get rid
-        # of the warning on msgpack-0.5.0 and to keep our
-        # behaviour on msgpack-1.0.0.
-        if conn.encoding is None:
-            packer_kwargs['use_bin_type'] = False
-        else:
-            packer_kwargs['use_bin_type'] = True
-
-        packer_kwargs['default'] = packer_default
-
-        self.packer = msgpack.Packer(**packer_kwargs)
+        self.packer = build_packer(conn)
 
     def _dumps(self, src):
         """

Diff for: tarantool/response.py

@@ -32,6 +32,58 @@
 
 from tarantool.msgpack_ext.unpacker import ext_hook as unpacker_ext_hook
 
+def build_unpacker(conn):
+    """
+    Build unpacker to unpack request response.
+
+    :param conn: Request sender.
+    :type conn: :class:`~tarantool.Connection`
+
+    :rtype: :class:`msgpack.Unpacker`
+    """
+
+    unpacker_kwargs = dict()
+
+    # Decode MsgPack arrays into Python lists by default (not tuples).
+    # Can be configured in the Connection init
+    unpacker_kwargs['use_list'] = conn.use_list
+
+    # Use raw=False instead of encoding='utf-8'.
+    if msgpack.version >= (0, 5, 2) and conn.encoding == 'utf-8':
+        # Get rid of the following warning.
+        # > PendingDeprecationWarning: encoding is deprecated,
+        # > Use raw=False instead.
+        unpacker_kwargs['raw'] = False
+    elif conn.encoding is not None:
+        unpacker_kwargs['encoding'] = conn.encoding
+
+    # raw=False is default since msgpack-1.0.0.
+    #
+    # The option decodes mp_str to bytes, not a Unicode
+    # string (when True).
+    if msgpack.version >= (1, 0, 0) and conn.encoding is None:
+        unpacker_kwargs['raw'] = True
+
+    # encoding option is not supported since msgpack-1.0.0,
+    # but it is handled in the Connection constructor.
+    assert(msgpack.version < (1, 0, 0) or conn.encoding in (None, 'utf-8'))
+
+    # strict_map_key=True is default since msgpack-1.0.0.
+    #
+    # The option forbids non-string keys in a map (when True).
+    if msgpack.version >= (1, 0, 0):
+        unpacker_kwargs['strict_map_key'] = False
+
+    # We need configured unpacker to work with error extention
+    # type payload, but module do not provide access to self
+    # inside extension type unpackers.
+    unpacker_no_ext = msgpack.Unpacker(**unpacker_kwargs)
+    ext_hook = lambda code, data: unpacker_ext_hook(code, data, unpacker_no_ext)
+    unpacker_kwargs['ext_hook'] = ext_hook
+
+    return msgpack.Unpacker(**unpacker_kwargs)
+
+
 class Response(Sequence):
     """
     Represents a single response from the server in compliance with the
@@ -56,41 +108,7 @@ def __init__(self, conn, response):
         # created in the __new__().
         # super(Response, self).__init__()
 
-        unpacker_kwargs = dict()
-
-        # Decode MsgPack arrays into Python lists by default (not tuples).
-        # Can be configured in the Connection init
-        unpacker_kwargs['use_list'] = conn.use_list
-
-        # Use raw=False instead of encoding='utf-8'.
-        if msgpack.version >= (0, 5, 2) and conn.encoding == 'utf-8':
-            # Get rid of the following warning.
-            # > PendingDeprecationWarning: encoding is deprecated,
-            # > Use raw=False instead.
-            unpacker_kwargs['raw'] = False
-        elif conn.encoding is not None:
-            unpacker_kwargs['encoding'] = conn.encoding
-
-        # raw=False is default since msgpack-1.0.0.
-        #
-        # The option decodes mp_str to bytes, not a Unicode
-        # string (when True).
-        if msgpack.version >= (1, 0, 0) and conn.encoding is None:
-            unpacker_kwargs['raw'] = True
-
-        # encoding option is not supported since msgpack-1.0.0,
-        # but it is handled in the Connection constructor.
-        assert(msgpack.version < (1, 0, 0) or conn.encoding in (None, 'utf-8'))
-
-        # strict_map_key=True is default since msgpack-1.0.0.
-        #
-        # The option forbids non-string keys in a map (when True).
-        if msgpack.version >= (1, 0, 0):
-            unpacker_kwargs['strict_map_key'] = False
-
-        unpacker_kwargs['ext_hook'] = unpacker_ext_hook
-
-        unpacker = msgpack.Unpacker(**unpacker_kwargs)
+        unpacker = build_unpacker(conn)
 
         unpacker.feed(response)
         header = unpacker.unpack()