From 4a7ab5a7252352fa1e1620ae0ccd42543eff7dc1 Mon Sep 17 00:00:00 2001 From: Georgy Moiseev Date: Tue, 30 Aug 2022 17:44:10 +0300 Subject: [PATCH 1/2] git: ignore venv folder --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index 98b355be..85d8c07b 100644 --- a/.gitignore +++ b/.gitignore @@ -17,3 +17,5 @@ sophia # Vim Swap files *.sw[a-z] .idea + +venv/* From 45c5b34617a1b83bf73278a9e8d3ea91fb380078 Mon Sep 17 00:00:00 2001 From: Georgy Moiseev Date: Tue, 30 Aug 2022 19:01:55 +0300 Subject: [PATCH 2/2] msgpack: support decimal extended type Tarantool supports decimal type since version 2.2.1 [1]. This patch introduced the support of Tarantool decimal type in msgpack decoders and encoders. The Tarantool decimal type is mapped to the native Python decimal.Decimal type. Tarantool decimal numbers have 38 digits of precision, that is, the total number of digits before and after the decimal point can be 38 [2]. If there are more digits arter the decimal point, the precision is lost. If there are more digits before the decimal point, error is thrown. In fact, there is also an exceptional case: if decimal starts with `0.`, 38 digits after the decimal point are supported without the loss of precision. msgpack encoder checks if everything is alright. If number is not a valid Tarantool decimal, the error is raised. If precision will be lost on conversion, warning is issued. Any Tarantool decimal could be converted to a Python decimal without the loss of precision. Python decimals have its own user alterable precision (defaulting to 28 places), but it's related only to arithmetic operations: we can allocate 38-placed decimal disregarding of what decimal module configuration is used [3]. 1. https://github.com/tarantool/tarantool/issues/692 2. https://www.tarantool.io/ru/doc/latest/reference/reference_lua/decimal/ 3. https://docs.python.org/3/library/decimal.html Closed #203 --- CHANGELOG.md | 1 + tarantool/error.py | 12 +- tarantool/msgpack_ext/decimal.py | 228 ++++++++++++++++ tarantool/msgpack_ext/packer.py | 9 + tarantool/msgpack_ext/unpacker.py | 6 + tarantool/request.py | 4 + tarantool/response.py | 3 + test/suites/__init__.py | 4 +- test/suites/lib/skip.py | 42 +++ test/suites/test_msgpack_ext.py | 431 ++++++++++++++++++++++++++++++ 10 files changed, 738 insertions(+), 2 deletions(-) create mode 100644 tarantool/msgpack_ext/decimal.py create mode 100644 tarantool/msgpack_ext/packer.py create mode 100644 tarantool/msgpack_ext/unpacker.py create mode 100644 test/suites/test_msgpack_ext.py diff --git a/CHANGELOG.md b/CHANGELOG.md index a6db6533..1c314cb1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## Unreleased ### Added +- Decimal type support (#203). ### Changed - Bump msgpack requirement to 1.0.4 (PR #223). diff --git a/tarantool/error.py b/tarantool/error.py index ba71ac8a..9ea49d71 100644 --- a/tarantool/error.py +++ b/tarantool/error.py @@ -109,10 +109,20 @@ class ConfigurationError(Error): Error of initialization with a user-provided configuration. ''' +class MsgpackError(Error): + ''' + Error with encoding or decoding of MP_EXT types + ''' + +class MsgpackWarning(UserWarning): + ''' + Warning with encoding or decoding of MP_EXT types + ''' __all__ = ("Warning", "Error", "InterfaceError", "DatabaseError", "DataError", "OperationalError", "IntegrityError", "InternalError", - "ProgrammingError", "NotSupportedError") + "ProgrammingError", "NotSupportedError", "MsgpackError", + "MsgpackWarning") # Monkey patch os.strerror for win32 if sys.platform == "win32": diff --git a/tarantool/msgpack_ext/decimal.py b/tarantool/msgpack_ext/decimal.py new file mode 100644 index 00000000..616024b1 --- /dev/null +++ b/tarantool/msgpack_ext/decimal.py @@ -0,0 +1,228 @@ +from decimal import Decimal + +from tarantool.error import MsgpackError, MsgpackWarning, warn + +# https://www.tarantool.io/en/doc/latest/dev_guide/internals/msgpack_extensions/#the-decimal-type +# +# The decimal MessagePack representation looks like this: +# +--------+-------------------+------------+===============+ +# | MP_EXT | length (optional) | MP_DECIMAL | PackedDecimal | +# +--------+-------------------+------------+===============+ +# +# PackedDecimal has the following structure: +# +# <--- length bytes --> +# +-------+=============+ +# | scale | BCD | +# +-------+=============+ +# +# Here scale is either MP_INT or MP_UINT. +# scale = number of digits after the decimal point +# +# BCD is a sequence of bytes representing decimal digits of the encoded number +# (each byte has two decimal digits each encoded using 4-bit nibbles), so +# byte >> 4 is the first digit and byte & 0x0f is the second digit. The +# leftmost digit in the array is the most significant. The rightmost digit in +# the array is the least significant. +# +# The first byte of the BCD array contains the first digit of the number, +# represented as follows: +# +# | 4 bits | 4 bits | +# = 0x = the 1st digit +# +# (The first nibble contains 0 if the decimal number has an even number of +# digits.) The last byte of the BCD array contains the last digit of the number +# and the final nibble, represented as follows: +# +# | 4 bits | 4 bits | +# = the last digit = nibble +# +# The final nibble represents the number’s sign: +# +# 0x0a, 0x0c, 0x0e, 0x0f stand for plus, +# 0x0b and 0x0d stand for minus. + +EXT_ID = 1 + +TARANTOOL_DECIMAL_MAX_DIGITS = 38 + +def get_mp_sign(sign): + if sign == '+': + return 0x0c + + if sign == '-': + return 0x0d + + raise RuntimeError + +def add_mp_digit(digit, bytes_reverted, digit_count): + if digit_count % 2 == 0: + bytes_reverted[-1] = bytes_reverted[-1] | (digit << 4) + else: + bytes_reverted.append(digit) + +def check_valid_tarantool_decimal(str_repr, scale, first_digit_ind): +# Decimal numbers have 38 digits of precision, that is, the total number of +# digits before and after the decimal point can be 38. If there are more +# digits arter the decimal point, the precision is lost. If there are more +# digits before the decimal point, error is thrown. +# +# Tarantool 2.10.1-0-g482d91c66 +# +# tarantool> decimal.new('10000000000000000000000000000000000000') +# --- +# - 10000000000000000000000000000000000000 +# ... +# +# tarantool> decimal.new('100000000000000000000000000000000000000') +# --- +# - error: '[string "return VERSION"]:1: variable ''VERSION'' is not declared' +# ... +# +# tarantool> decimal.new('1.0000000000000000000000000000000000001') +# --- +# - 1.0000000000000000000000000000000000001 +# ... +# +# tarantool> decimal.new('1.00000000000000000000000000000000000001') +# --- +# - 1.0000000000000000000000000000000000000 +# ... +# +# In fact, there is also an exceptional case: if decimal starts with `0.`, +# 38 digits after the decimal point are supported without the loss of precision. +# +# tarantool> decimal.new('0.00000000000000000000000000000000000001') +# --- +# - 0.00000000000000000000000000000000000001 +# ... +# +# tarantool> decimal.new('0.000000000000000000000000000000000000001') +# --- +# - 0.00000000000000000000000000000000000000 +# ... + if scale > 0: + digit_count = len(str_repr) - 1 - first_digit_ind + else: + digit_count = len(str_repr) - first_digit_ind + + if digit_count <= TARANTOOL_DECIMAL_MAX_DIGITS: + return True + + if (digit_count - scale) > TARANTOOL_DECIMAL_MAX_DIGITS: + raise MsgpackError('Decimal cannot be encoded: Tarantool decimal ' + \ + 'supports a maximum of 38 digits.') + + starts_with_zero = str_repr[first_digit_ind] == '0' + + if ( (digit_count > TARANTOOL_DECIMAL_MAX_DIGITS + 1) or \ + (digit_count == TARANTOOL_DECIMAL_MAX_DIGITS + 1 \ + and not starts_with_zero)): + warn('Decimal encoded with loss of precision: ' + \ + 'Tarantool decimal supports a maximum of 38 digits.', + MsgpackWarning) + return False + + return True + +def strip_decimal_str(str_repr, scale, first_digit_ind): + assert scale > 0 + # Strip extra bytes + str_repr = str_repr[:TARANTOOL_DECIMAL_MAX_DIGITS + 1 + first_digit_ind] + + str_repr = str_repr.rstrip('0') + str_repr = str_repr.rstrip('.') + # Do not strips zeroes before the decimal point + return str_repr + +def encode(obj): + # Non-scientific string with trailing zeroes removed + str_repr = format(obj, 'f') + + bytes_reverted = bytearray() + + scale = 0 + for i in range(len(str_repr)): + str_digit = str_repr[i] + if str_digit == '.': + scale = len(str_repr) - i - 1 + break + + if str_repr[0] == '-': + sign = '-' + first_digit_ind = 1 + else: + sign = '+' + first_digit_ind = 0 + + if not check_valid_tarantool_decimal(str_repr, scale, first_digit_ind): + str_repr = strip_decimal_str(str_repr, scale, first_digit_ind) + + bytes_reverted.append(get_mp_sign(sign)) + + digit_count = 0 + # We need to update the scale after possible strip_decimal_str() + scale = 0 + + for i in range(len(str_repr) - 1, first_digit_ind - 1, -1): + str_digit = str_repr[i] + if str_digit == '.': + scale = len(str_repr) - i - 1 + continue + + add_mp_digit(int(str_digit), bytes_reverted, digit_count) + digit_count = digit_count + 1 + + # Remove leading zeroes since they already covered by scale + for i in range(len(bytes_reverted) - 1, 0, -1): + if bytes_reverted[i] != 0: + break + bytes_reverted.pop() + + bytes_reverted.append(scale) + + return bytes(bytes_reverted[::-1]) + + +def get_str_sign(nibble): + if nibble == 0x0a or nibble == 0x0c or nibble == 0x0e or nibble == 0x0f: + return '+' + + if nibble == 0x0b or nibble == 0x0d: + return '-' + + raise MsgpackError('Unexpected MP_DECIMAL sign nibble') + +def add_str_digit(digit, digits_reverted, scale): + if not (0 <= digit <= 9): + raise MsgpackError('Unexpected MP_DECIMAL digit nibble') + + if len(digits_reverted) == scale: + digits_reverted.append('.') + + digits_reverted.append(str(digit)) + +def decode(data): + scale = data[0] + + sign = get_str_sign(data[-1] & 0x0f) + + # Parse from tail since scale is counted from the tail. + digits_reverted = [] + + add_str_digit(data[-1] >> 4, digits_reverted, scale) + + for i in range(len(data) - 2, 0, -1): + add_str_digit(data[i] & 0x0f, digits_reverted, scale) + add_str_digit(data[i] >> 4, digits_reverted, scale) + + # Add leading zeroes in case of 0.000... number + for i in range(len(digits_reverted), scale + 1): + add_str_digit(0, digits_reverted, scale) + + digits_reverted.append(sign) + + str_repr = ''.join(digits_reverted[::-1]) + + return Decimal(str_repr) diff --git a/tarantool/msgpack_ext/packer.py b/tarantool/msgpack_ext/packer.py new file mode 100644 index 00000000..db4aa710 --- /dev/null +++ b/tarantool/msgpack_ext/packer.py @@ -0,0 +1,9 @@ +from decimal import Decimal +from msgpack import ExtType + +import tarantool.msgpack_ext.decimal as ext_decimal + +def default(obj): + if isinstance(obj, Decimal): + return ExtType(ext_decimal.EXT_ID, ext_decimal.encode(obj)) + raise TypeError("Unknown type: %r" % (obj,)) diff --git a/tarantool/msgpack_ext/unpacker.py b/tarantool/msgpack_ext/unpacker.py new file mode 100644 index 00000000..dd6c0112 --- /dev/null +++ b/tarantool/msgpack_ext/unpacker.py @@ -0,0 +1,6 @@ +import tarantool.msgpack_ext.decimal as ext_decimal + +def ext_hook(code, data): + if code == ext_decimal.EXT_ID: + return ext_decimal.decode(data) + raise NotImplementedError("Unknown msgpack type: %d" % (code,)) diff --git a/tarantool/request.py b/tarantool/request.py index d58960cc..44fb5a3c 100644 --- a/tarantool/request.py +++ b/tarantool/request.py @@ -59,6 +59,8 @@ binary_types ) +from tarantool.msgpack_ext.packer import default as packer_default + class Request(object): ''' Represents a single request to the server in compliance with the @@ -122,6 +124,8 @@ def __init__(self, conn): else: packer_kwargs['use_bin_type'] = True + packer_kwargs['default'] = packer_default + self.packer = msgpack.Packer(**packer_kwargs) def _dumps(self, src): diff --git a/tarantool/response.py b/tarantool/response.py index 177fd146..3e367787 100644 --- a/tarantool/response.py +++ b/tarantool/response.py @@ -29,6 +29,7 @@ tnt_strerror ) +from tarantool.msgpack_ext.unpacker import ext_hook as unpacker_ext_hook class Response(Sequence): ''' @@ -86,6 +87,8 @@ def __init__(self, conn, response): if msgpack.version >= (1, 0, 0): unpacker_kwargs['strict_map_key'] = False + unpacker_kwargs['ext_hook'] = unpacker_ext_hook + unpacker = msgpack.Unpacker(**unpacker_kwargs) unpacker.feed(response) diff --git a/test/suites/__init__.py b/test/suites/__init__.py index 406ca140..984665b6 100644 --- a/test/suites/__init__.py +++ b/test/suites/__init__.py @@ -15,12 +15,14 @@ from .test_dbapi import TestSuite_DBAPI from .test_encoding import TestSuite_Encoding from .test_ssl import TestSuite_Ssl +from .test_msgpack_ext import TestSuite_MsgpackExt test_cases = (TestSuite_Schema_UnicodeConnection, TestSuite_Schema_BinaryConnection, TestSuite_Request, TestSuite_Protocol, TestSuite_Reconnect, TestSuite_Mesh, TestSuite_Execute, TestSuite_DBAPI, - TestSuite_Encoding, TestSuite_Pool, TestSuite_Ssl) + TestSuite_Encoding, TestSuite_Pool, TestSuite_Ssl, + TestSuite_MsgpackExt) def load_tests(loader, tests, pattern): suite = unittest.TestSuite() diff --git a/test/suites/lib/skip.py b/test/suites/lib/skip.py index 284b70b6..9ac5fe9e 100644 --- a/test/suites/lib/skip.py +++ b/test/suites/lib/skip.py @@ -43,6 +43,38 @@ def wrapper(self, *args, **kwargs): return wrapper +def skip_or_run_test_pcall_require(func, REQUIRED_TNT_MODULE, msg): + """Decorator to skip or run tests depending on tarantool + module requre success or fail. + + Also, it can be used with the 'setUp' method for skipping + the whole test suite. + """ + + @functools.wraps(func) + def wrapper(self, *args, **kwargs): + if func.__name__ == 'setUp': + func(self, *args, **kwargs) + + srv = None + + if hasattr(self, 'servers'): + srv = self.servers[0] + + if hasattr(self, 'srv'): + srv = self.srv + + assert srv is not None + + resp = srv.admin("pcall(require, '%s')" % REQUIRED_TNT_MODULE) + if not resp[0]: + self.skipTest('Tarantool %s' % (msg, )) + + if func.__name__ != 'setUp': + func(self, *args, **kwargs) + + return wrapper + def skip_or_run_test_python(func, REQUIRED_PYTHON_VERSION, msg): """Decorator to skip or run tests depending on the Python version. @@ -101,3 +133,13 @@ def skip_or_run_conn_pool_test(func): return skip_or_run_test_python(func, '3.7', 'does not support ConnectionPool') +def skip_or_run_decimal_test(func): + """Decorator to skip or run decimal-related tests depending on + the tarantool version. + + Tarantool supports decimal type only since 2.2.1 version. + See https://github.com/tarantool/tarantool/issues/692 + """ + + return skip_or_run_test_pcall_require(func, 'decimal', + 'does not support decimal type') diff --git a/test/suites/test_msgpack_ext.py b/test/suites/test_msgpack_ext.py new file mode 100644 index 00000000..42b937fe --- /dev/null +++ b/test/suites/test_msgpack_ext.py @@ -0,0 +1,431 @@ +# -*- coding: utf-8 -*- + +from __future__ import print_function + +import sys +import unittest +import decimal +import msgpack +import warnings +import tarantool + +from tarantool.msgpack_ext.packer import default as packer_default +from tarantool.msgpack_ext.unpacker import ext_hook as unpacker_ext_hook + +from .lib.tarantool_server import TarantoolServer +from .lib.skip import skip_or_run_decimal_test +from tarantool.error import MsgpackError, MsgpackWarning + +class TestSuite_MsgpackExt(unittest.TestCase): + @classmethod + def setUpClass(self): + print(' MSGPACK EXT TYPES '.center(70, '='), file=sys.stderr) + print('-' * 70, file=sys.stderr) + self.srv = TarantoolServer() + self.srv.script = 'test/suites/box.lua' + self.srv.start() + + self.adm = self.srv.admin + self.adm(r""" + _, decimal = pcall(require, 'decimal') + + box.schema.space.create('test') + box.space['test']:create_index('primary', { + type = 'tree', + parts = {1, 'string'}, + unique = true}) + + box.schema.user.create('test', {password = 'test', if_not_exists = true}) + box.schema.user.grant('test', 'read,write,execute', 'universe') + """) + + self.con = tarantool.Connection(self.srv.host, self.srv.args['primary'], + user='test', password='test') + + def setUp(self): + # prevent a remote tarantool from clean our session + if self.srv.is_started(): + self.srv.touch_lock() + + self.adm("box.space['test']:truncate()") + + + valid_decimal_cases = { + 'simple_decimal_1': { + 'python': decimal.Decimal('0.7'), + 'msgpack': (b'\x01\x7c'), + 'tarantool': "decimal.new('0.7')", + }, + 'simple_decimal_2': { + 'python': decimal.Decimal('0.3'), + 'msgpack': (b'\x01\x3c'), + 'tarantool': "decimal.new('0.3')", + }, + 'simple_decimal_3': { + 'python': decimal.Decimal('-18.34'), + 'msgpack': (b'\x02\x01\x83\x4d'), + 'tarantool': "decimal.new('-18.34')", + }, + 'simple_decimal_4': { + 'python': decimal.Decimal('-108.123456789'), + 'msgpack': (b'\x09\x01\x08\x12\x34\x56\x78\x9d'), + 'tarantool': "decimal.new('-108.123456789')", + }, + 'simple_decimal_5': { + 'python': decimal.Decimal('100'), + 'msgpack': (b'\x00\x10\x0c'), + 'tarantool': "decimal.new('100')", + }, + 'simple_decimal_6': { + 'python': decimal.Decimal('0.1'), + 'msgpack': (b'\x01\x1c'), + 'tarantool': "decimal.new('0.1')", + }, + 'simple_decimal_7': { + 'python': decimal.Decimal('-0.1'), + 'msgpack': (b'\x01\x1d'), + 'tarantool': "decimal.new('-0.1')", + }, + 'simple_decimal_8': { + 'python': decimal.Decimal('-12.34'), + 'msgpack': (b'\x02\x01\x23\x4d'), + 'tarantool': "decimal.new('-12.34')", + }, + 'simple_decimal_9': { + 'python': decimal.Decimal('12.34'), + 'msgpack': (b'\x02\x01\x23\x4c'), + 'tarantool': "decimal.new('12.34')", + }, + 'simple_decimal_10': { + 'python': decimal.Decimal('1.4'), + 'msgpack': (b'\x01\x01\x4c'), + 'tarantool': "decimal.new('1.4')", + }, + 'simple_decimal_11': { + 'python': decimal.Decimal('2.718281828459045'), + 'msgpack': (b'\x0f\x02\x71\x82\x81\x82\x84\x59\x04\x5c'), + 'tarantool': "decimal.new('2.718281828459045')", + }, + 'simple_decimal_12': { + 'python': decimal.Decimal('-2.718281828459045'), + 'msgpack': (b'\x0f\x02\x71\x82\x81\x82\x84\x59\x04\x5d'), + 'tarantool': "decimal.new('-2.718281828459045')", + }, + 'simple_decimal_13': { + 'python': decimal.Decimal('3.141592653589793'), + 'msgpack': (b'\x0f\x03\x14\x15\x92\x65\x35\x89\x79\x3c'), + 'tarantool': "decimal.new('3.141592653589793')", + }, + 'simple_decimal_14': { + 'python': decimal.Decimal('-3.141592653589793'), + 'msgpack': (b'\x0f\x03\x14\x15\x92\x65\x35\x89\x79\x3d'), + 'tarantool': "decimal.new('-3.141592653589793')", + }, + 'simple_decimal_15': { + 'python': decimal.Decimal('1'), + 'msgpack': (b'\x00\x1c'), + 'tarantool': "decimal.new('1')", + }, + 'simple_decimal_16': { + 'python': decimal.Decimal('-1'), + 'msgpack': (b'\x00\x1d'), + 'tarantool': "decimal.new('-1')", + }, + 'simple_decimal_17': { + 'python': decimal.Decimal('0'), + 'msgpack': (b'\x00\x0c'), + 'tarantool': "decimal.new('0')", + }, + 'simple_decimal_18': { + 'python': decimal.Decimal('-0'), + 'msgpack': (b'\x00\x0d'), + 'tarantool': "decimal.new('-0')", + }, + 'simple_decimal_19': { + 'python': decimal.Decimal('0.01'), + 'msgpack': (b'\x02\x1c'), + 'tarantool': "decimal.new('0.01')", + }, + 'simple_decimal_20': { + 'python': decimal.Decimal('0.001'), + 'msgpack': (b'\x03\x1c'), + 'tarantool': "decimal.new('0.001')", + }, + 'decimal_limits_1': { + 'python': decimal.Decimal('11111111111111111111111111111111111111'), + 'msgpack': (b'\x00\x01\x11\x11\x11\x11\x11\x11\x11\x11\x11' + + b'\x11\x11\x11\x11\x11\x11\x11\x11\x11\x1c'), + 'tarantool': "decimal.new('11111111111111111111111111111111111111')", + }, + 'decimal_limits_2': { + 'python': decimal.Decimal('-11111111111111111111111111111111111111'), + 'msgpack': (b'\x00\x01\x11\x11\x11\x11\x11\x11\x11\x11\x11' + + b'\x11\x11\x11\x11\x11\x11\x11\x11\x11\x1d'), + 'tarantool': "decimal.new('-11111111111111111111111111111111111111')", + }, + 'decimal_limits_3': { + 'python': decimal.Decimal('0.0000000000000000000000000000000000001'), + 'msgpack': (b'\x25\x1c'), + 'tarantool': "decimal.new('0.0000000000000000000000000000000000001')", + }, + 'decimal_limits_4': { + 'python': decimal.Decimal('-0.0000000000000000000000000000000000001'), + 'msgpack': (b'\x25\x1d'), + 'tarantool': "decimal.new('-0.0000000000000000000000000000000000001')", + }, + 'decimal_limits_5': { + 'python': decimal.Decimal('0.00000000000000000000000000000000000001'), + 'msgpack': (b'\x26\x1c'), + 'tarantool': "decimal.new('0.00000000000000000000000000000000000001')", + }, + 'decimal_limits_6': { + 'python': decimal.Decimal('-0.00000000000000000000000000000000000001'), + 'msgpack': (b'\x26\x1d'), + 'tarantool': "decimal.new('-0.00000000000000000000000000000000000001')", + }, + 'decimal_limits_7': { + 'python': decimal.Decimal('0.00000000000000000000000000000000000009'), + 'msgpack': (b'\x26\x9c'), + 'tarantool': "decimal.new('0.00000000000000000000000000000000000009')", + }, + 'decimal_limits_8': { + 'python': decimal.Decimal('0.00000000000000000000000000000000000009'), + 'msgpack': (b'\x26\x9c'), + 'tarantool': "decimal.new('0.00000000000000000000000000000000000009')", + }, + 'decimal_limits_9': { + 'python': decimal.Decimal('99999999999999999999999999999999999999'), + 'msgpack': (b'\x00\x09\x99\x99\x99\x99\x99\x99\x99\x99\x99' + + b'\x99\x99\x99\x99\x99\x99\x99\x99\x99\x9c'), + 'tarantool': "decimal.new('99999999999999999999999999999999999999')", + }, + 'decimal_limits_10': { + 'python': decimal.Decimal('-99999999999999999999999999999999999999'), + 'msgpack': (b'\x00\x09\x99\x99\x99\x99\x99\x99\x99\x99\x99' + + b'\x99\x99\x99\x99\x99\x99\x99\x99\x99\x9d'), + 'tarantool': "decimal.new('-99999999999999999999999999999999999999')", + }, + 'decimal_limits_11': { + 'python': decimal.Decimal('1234567891234567890.0987654321987654321'), + 'msgpack': (b'\x13\x01\x23\x45\x67\x89\x12\x34\x56\x78\x90' + + b'\x09\x87\x65\x43\x21\x98\x76\x54\x32\x1c'), + 'tarantool': "decimal.new('1234567891234567890.0987654321987654321')", + }, + 'decimal_limits_12': { + 'python': decimal.Decimal('-1234567891234567890.0987654321987654321'), + 'msgpack': (b'\x13\x01\x23\x45\x67\x89\x12\x34\x56\x78\x90' + + b'\x09\x87\x65\x43\x21\x98\x76\x54\x32\x1d'), + 'tarantool': "decimal.new('-1234567891234567890.0987654321987654321')", + }, + } + + def test_decimal_msgpack_decode(self): + for name in self.valid_decimal_cases.keys(): + with self.subTest(msg=name): + decimal_case = self.valid_decimal_cases[name] + + self.assertEqual(unpacker_ext_hook(1, decimal_case['msgpack']), + decimal_case['python']) + + @skip_or_run_decimal_test + def test_decimal_tarantool_decode(self): + for name in self.valid_decimal_cases.keys(): + with self.subTest(msg=name): + decimal_case = self.valid_decimal_cases[name] + + self.adm(f"box.space['test']:replace{{'{name}', {decimal_case['tarantool']}}}") + + self.assertSequenceEqual( + self.con.select('test', name), + [[name, decimal_case['python']]]) + + def test_decimal_msgpack_encode(self): + for name in self.valid_decimal_cases.keys(): + with self.subTest(msg=name): + decimal_case = self.valid_decimal_cases[name] + + self.assertEqual(packer_default(decimal_case['python']), + msgpack.ExtType(code=1, data=decimal_case['msgpack'])) + + @skip_or_run_decimal_test + def test_decimal_tarantool_encode(self): + for name in self.valid_decimal_cases.keys(): + with self.subTest(msg=name): + decimal_case = self.valid_decimal_cases[name] + + self.con.insert('test', [name, decimal_case['python']]) + + lua_eval = f""" + local tuple = box.space['test']:get('{name}') + assert(tuple ~= nil) + + local dec = {decimal_case['tarantool']} + if tuple[2] == dec then + return true + else + return nil, ('%s is not equal to expected %s'):format( + tostring(tuple[2]), tostring(dec)) + end + """ + + self.assertSequenceEqual(self.con.eval(lua_eval), [True]) + + + error_decimal_cases = { + 'decimal_limit_break_head_1': { + 'python': decimal.Decimal('999999999999999999999999999999999999999'), + }, + 'decimal_limit_break_head_2': { + 'python': decimal.Decimal('-999999999999999999999999999999999999999'), + }, + 'decimal_limit_break_head_3': { + 'python': decimal.Decimal('999999999999999999900000099999999999999999999'), + }, + 'decimal_limit_break_head_4': { + 'python': decimal.Decimal('-999999999999999999900000099999999999999999999'), + }, + 'decimal_limit_break_head_5': { + 'python': decimal.Decimal('100000000000000000000000000000000000000.1'), + }, + 'decimal_limit_break_head_6': { + 'python': decimal.Decimal('-100000000000000000000000000000000000000.1'), + }, + 'decimal_limit_break_head_7': { + 'python': decimal.Decimal('1000000000000000000011110000000000000000000.1'), + }, + 'decimal_limit_break_head_8': { + 'python': decimal.Decimal('-1000000000000000000011110000000000000000000.1'), + }, + } + + def test_decimal_msgpack_encode_error(self): + for name in self.error_decimal_cases.keys(): + with self.subTest(msg=name): + decimal_case = self.error_decimal_cases[name] + + msg = 'Decimal cannot be encoded: Tarantool decimal ' + \ + 'supports a maximum of 38 digits.' + self.assertRaisesRegex( + MsgpackError, msg, + lambda: packer_default(decimal_case['python'])) + + @skip_or_run_decimal_test + def test_decimal_tarantool_encode_error(self): + for name in self.error_decimal_cases.keys(): + with self.subTest(msg=name): + decimal_case = self.error_decimal_cases[name] + + msg = 'Decimal cannot be encoded: Tarantool decimal ' + \ + 'supports a maximum of 38 digits.' + self.assertRaisesRegex( + MsgpackError, msg, + lambda: self.con.insert('test', [name, decimal_case['python']])) + + + precision_loss_decimal_cases = { + 'decimal_limit_break_tail_1': { + 'python': decimal.Decimal('1.00000000000000000000000000000000000001'), + 'msgpack': (b'\x00\x1c'), + 'tarantool': "decimal.new('1')", + }, + 'decimal_limit_break_tail_2': { + 'python': decimal.Decimal('-1.00000000000000000000000000000000000001'), + 'msgpack': (b'\x00\x1d'), + 'tarantool': "decimal.new('-1')", + }, + 'decimal_limit_break_tail_3': { + 'python': decimal.Decimal('0.000000000000000000000000000000000000001'), + 'msgpack': (b'\x00\x0c'), + 'tarantool': "decimal.new('0.000000000000000000000000000000000000001')", + }, + 'decimal_limit_break_tail_4': { + 'python': decimal.Decimal('-0.000000000000000000000000000000000000001'), + 'msgpack': (b'\x00\x0d'), + 'tarantool': "decimal.new('-0.000000000000000000000000000000000000001')", + }, + 'decimal_limit_break_tail_5': { + 'python': decimal.Decimal('9999999.99999900000000000000000000000000000000000001'), + 'msgpack': (b'\x06\x99\x99\x99\x99\x99\x99\x9c'), + 'tarantool': "decimal.new('9999999.999999')", + }, + 'decimal_limit_break_tail_6': { + 'python': decimal.Decimal('-9999999.99999900000000000000000000000000000000000001'), + 'msgpack': (b'\x06\x99\x99\x99\x99\x99\x99\x9d'), + 'tarantool': "decimal.new('-9999999.999999')", + }, + 'decimal_limit_break_tail_7': { + 'python': decimal.Decimal('99999999999999999999999999999999999999.1'), + 'msgpack': (b'\x00\x09\x99\x99\x99\x99\x99\x99\x99\x99\x99' + + b'\x99\x99\x99\x99\x99\x99\x99\x99\x99\x9c'), + 'tarantool': "decimal.new('99999999999999999999999999999999999999')", + }, + 'decimal_limit_break_tail_8': { + 'python': decimal.Decimal('-99999999999999999999999999999999999999.1'), + 'msgpack': (b'\x00\x09\x99\x99\x99\x99\x99\x99\x99\x99\x99' + + b'\x99\x99\x99\x99\x99\x99\x99\x99\x99\x9d'), + 'tarantool': "decimal.new('-99999999999999999999999999999999999999')", + }, + 'decimal_limit_break_tail_9': { + 'python': decimal.Decimal('99999999999999999999999999999999999999.1111111111111111111111111'), + 'msgpack': (b'\x00\x09\x99\x99\x99\x99\x99\x99\x99\x99\x99' + + b'\x99\x99\x99\x99\x99\x99\x99\x99\x99\x9c'), + 'tarantool': "decimal.new('99999999999999999999999999999999999999')", + }, + 'decimal_limit_break_tail_10': { + 'python': decimal.Decimal('-99999999999999999999999999999999999999.1111111111111111111111111'), + 'msgpack': (b'\x00\x09\x99\x99\x99\x99\x99\x99\x99\x99\x99' + + b'\x99\x99\x99\x99\x99\x99\x99\x99\x99\x9d'), + 'tarantool': "decimal.new('-99999999999999999999999999999999999999')", + }, + } + + def test_decimal_msgpack_encode_with_precision_loss(self): + for name in self.precision_loss_decimal_cases.keys(): + with self.subTest(msg=name): + decimal_case = self.precision_loss_decimal_cases[name] + + msg = 'Decimal encoded with loss of precision: ' + \ + 'Tarantool decimal supports a maximum of 38 digits.' + + self.assertWarnsRegex( + MsgpackWarning, msg, + lambda: self.assertEqual( + packer_default(decimal_case['python']), + msgpack.ExtType(code=1, data=decimal_case['msgpack']) + ) + ) + + + @skip_or_run_decimal_test + def test_decimal_tarantool_encode_with_precision_loss(self): + for name in self.precision_loss_decimal_cases.keys(): + with self.subTest(msg=name): + decimal_case = self.precision_loss_decimal_cases[name] + + msg = 'Decimal encoded with loss of precision: ' + \ + 'Tarantool decimal supports a maximum of 38 digits.' + + self.assertWarnsRegex( + MsgpackWarning, msg, + lambda: self.con.insert('test', [name, decimal_case['python']])) + + lua_eval = f""" + local tuple = box.space['test']:get('{name}') + assert(tuple ~= nil) + + local dec = {decimal_case['tarantool']} + if tuple[2] == dec then + return true + else + return nil, ('%s is not equal to expected %s'):format( + tostring(tuple[2]), tostring(dec)) + end + """ + + self.assertSequenceEqual(self.con.eval(lua_eval), [True]) + + @classmethod + def tearDownClass(self): + self.con.close() + self.srv.stop() + self.srv.clean()