msgpack: support datetime extended type

Tarantool supports datetime type since version 2.10.0 [1]. This patch
introduced the support of Tarantool datetime type in msgpack decoders
and encoders. The Tarantool datetime type is mapped to the
pandas.Timestamp type. pandas.Timestamp was chosen because it could be
used to store both nanoseconds and timezone information. In-build Python
datetime.datetime supports microseconds at most, numpy.datetime64 do
not support timezones.

If you want to use numpy.datetime64 or datetime.datetime in your logic,
you can use pandas converters in your code:
- pandas.to_datetime(): numpy.datetime64 -> pandas.Timestamp
- pandas.to_datetime(): datetime.datetime -> pandas.Timestamp
- pandas.Timestamp.to_datetime64(): pandas.Timestamp -> numpy.datetime64
- pandas.Timestamp.to_pydatetime(): pandas.Timestamp -> datetime.datetime

This patch does not yet introduce the support of timezones in datetime.

1. tarantool/tarantool#5941

Part of #204

Author: DifferentialOrange

requirements.txt

@@ -1 +1,2 @@
 msgpack>=1.0.4
+pandas>=1.0.0

tarantool/msgpack_ext_types/datetime.py

@@ -0,0 +1,91 @@
+import time
+import math
+import pandas
+
+from tarantool.error import MsgpackWarning, warn
+
+# https://www.tarantool.io/ru/doc/latest/dev_guide/internals/msgpack_extensions/#the-datetime-type
+#
+# The datetime MessagePack representation looks like this:
+# +---------+----------------+==========+-----------------+
+# | MP_EXT  | MP_DATETIME    | seconds  | nsec; tzoffset; |
+# | = d7/d8 | = 4            |          | tzindex;        |
+# +---------+----------------+==========+-----------------+
+# MessagePack data contains:
+#
+# * Seconds (8 bytes) as an unencoded 64-bit signed integer stored in the
+#   little-endian order.
+# * The optional fields (8 bytes), if any of them have a non-zero value.
+#   The fields include nsec (4 bytes), tzoffset (2 bytes), and
+#   tzindex (2 bytes) packed in the little-endian order.
+#
+# seconds is seconds since Epoch, where the epoch is the point where the time
+# starts, and is platform dependent. For Unix, the epoch is January 1,
+# 1970, 00:00:00 (UTC). Tarantool uses a double type, see a structure
+# definition in src/lib/core/datetime.h and reasons in
+# https://github.com/tarantool/tarantool/wiki/Datetime-internals#intervals-in-c
+#
+# nsec is nanoseconds, fractional part of seconds. Tarantool uses int32_t, see
+# a definition in src/lib/core/datetime.h.
+#
+# tzoffset is timezone offset in minutes from UTC. Tarantool uses a int16_t type,
+# see a structure definition in src/lib/core/datetime.h.
+#
+# tzindex is Olson timezone id. Tarantool uses a int16_t type, see a structure
+# definition in src/lib/core/datetime.h. If both tzoffset and tzindex are
+# specified, tzindex has the preference and the tzoffset value is ignored.
+
+EXT_ID = 4
+
+SECONDS_SIZE_BYTES  = 8
+NSEC_SIZE_BYTES     = 4
+TZOFFSET_SIZE_BYTES = 2
+TZINDEX_SIZE_BYTES  = 2
+
+BYTEORDER = 'little'
+
+NSEC_IN_SEC = 1000000000
+assert isinstance(NSEC_IN_SEC, int)
+
+def encode(obj):
+    seconds = obj.value // NSEC_IN_SEC
+    nsec = obj.value % NSEC_IN_SEC
+    tzoffset = 0
+    tzindex = 0
+
+    bytes_buffer = bytes(seconds.to_bytes(SECONDS_SIZE_BYTES,
+                                         byteorder=BYTEORDER))
+
+    if (nsec != 0) or (tzoffset != 0) or (tzindex != 0):
+        bytes_buffer = bytes_buffer + nsec.to_bytes(NSEC_SIZE_BYTES,
+                                                    byteorder=BYTEORDER)
+        bytes_buffer = bytes_buffer + tzoffset.to_bytes(TZOFFSET_SIZE_BYTES,
+                                                        byteorder=BYTEORDER)
+        bytes_buffer = bytes_buffer + tzindex.to_bytes(TZINDEX_SIZE_BYTES,
+                                                       byteorder=BYTEORDER)
+
+    return bytes_buffer
+
+def get_bytes_as_int(data, cursor, size):
+    part = data[cursor:cursor + size]
+    return int.from_bytes(part, BYTEORDER), cursor + size
+
+def decode(data):
+    cursor = 0
+    seconds, cursor = get_bytes_as_int(data, cursor, SECONDS_SIZE_BYTES)
+
+    if len(data) > SECONDS_SIZE_BYTES:
+        nsec, cursor     = get_bytes_as_int(data, cursor, NSEC_SIZE_BYTES)
+        tzoffset, cursor = get_bytes_as_int(data, cursor, TZOFFSET_SIZE_BYTES)
+        tzindex, cursor  = get_bytes_as_int(data, cursor, TZINDEX_SIZE_BYTES)
+    else:
+        nsec = 0
+        tzoffset = 0
+        tzindex = 0
+
+    if (tzoffset != 0) or (tzindex != 0):
+        raise NotImplementedError
+
+    total_nsec = seconds * NSEC_IN_SEC + nsec
+
+    return pandas.to_datetime(total_nsec, unit='ns')

tarantool/msgpack_ext_types/packer.py

@@ -1,13 +1,16 @@
+import pandas
 from decimal import Decimal
 from uuid import UUID
 from msgpack import ExtType
 
 import tarantool.msgpack_ext_types.decimal as ext_decimal
 import tarantool.msgpack_ext_types.uuid as ext_uuid
+import tarantool.msgpack_ext_types.datetime as ext_datetime
 
 encoders = [
-    {'type': Decimal, 'ext': ext_decimal},
-    {'type': UUID,    'ext': ext_uuid   },
+    {'type': Decimal,          'ext': ext_decimal },
+    {'type': UUID,             'ext': ext_uuid    },
+    {'type': pandas.Timestamp, 'ext': ext_datetime},
 ]
 
 def default(obj):

tarantool/msgpack_ext_types/unpacker.py

@@ -1,9 +1,11 @@
 import tarantool.msgpack_ext_types.decimal as ext_decimal
 import tarantool.msgpack_ext_types.uuid as ext_uuid
+import tarantool.msgpack_ext_types.datetime as ext_datetime
 
 decoders = {
-    ext_decimal.EXT_ID: ext_decimal.decode,
-    ext_uuid.EXT_ID   : ext_uuid.decode   ,
+    ext_decimal.EXT_ID  : ext_decimal.decode ,
+    ext_uuid.EXT_ID     : ext_uuid.decode    ,
+    ext_datetime.EXT_ID : ext_datetime.decode,
 }
 
 def ext_hook(code, data):

test/suites/lib/skip.py

@@ -154,3 +154,14 @@ def skip_or_run_UUID_test(func):
 
     return skip_or_run_test_tarantool(func, '2.4.1',
                                       'does not support UUID type')
+
+def skip_or_run_datetime_test(func):
+    """Decorator to skip or run datetime-related tests depending on
+    the tarantool version.
+
+    Tarantool supports datetime type only since 2.10.0 version.
+    See https://github.com/tarantool/tarantool/issues/5941
+    """
+
+    return skip_or_run_test_pcall_require(func, 'datetime',
+                                      'does not support datetime type')

test/suites/test_msgpack_ext_types.py

@@ -9,12 +9,15 @@
 import msgpack
 import warnings
 import tarantool
+import pandas
 
 from tarantool.msgpack_ext_types.packer import default as packer_default
 from tarantool.msgpack_ext_types.unpacker import ext_hook as unpacker_ext_hook
 
 from .lib.tarantool_server import TarantoolServer
-from .lib.skip import skip_or_run_decimal_test, skip_or_run_UUID_test
+from .lib.skip import (
+    skip_or_run_datetime_test, skip_or_run_decimal_test,
+    skip_or_run_UUID_test,)
 from tarantool.error import MsgpackError, MsgpackWarning
 
 class TestSuite_MsgpackExtTypes(unittest.TestCase):
@@ -30,6 +33,7 @@ def setUpClass(self):
         self.adm(r"""
             _, decimal = pcall(require, 'decimal')
             _, uuid = pcall(require, 'uuid')
+            _, datetime = pcall(require, 'datetime')
 
             box.schema.space.create('test')
             box.space['test']:create_index('primary', {
@@ -499,6 +503,96 @@ def test_UUID_tarantool_encode(self):
 
                 self.assertSequenceEqual(self.con.eval(lua_eval), [True])
 
+
+    datetime_cases = [
+        {
+            'python': pandas.Timestamp(year=1970, month=1, day=1),
+            'msgpack': (b'\x00\x00\x00\x00\x00\x00\x00\x00'),
+            'tarantool': r"datetime.new({year=1970, month=1, day=1})",
+        },
+        {
+            'python': pandas.Timestamp(year=2022, month=8, day=31),
+            'msgpack': (b'\x80\xa4\x0e\x63\x00\x00\x00\x00'),
+            'tarantool': r"datetime.new({year=2022, month=8, day=31})",
+        },
+        {
+            'python': pandas.Timestamp(year=2022, month=8, day=31, hour=18, minute=7),
+            'msgpack': (b'\x44\xa3\x0f\x63\x00\x00\x00\x00'),
+            'tarantool': r"datetime.new({year=2022, month=8, day=31, hour=18, min=7})",
+        },
+        {
+            'python': pandas.Timestamp(year=2022, month=8, day=31, hour=18, minute=7, second=54),
+            'msgpack': (b'\x7a\xa3\x0f\x63\x00\x00\x00\x00'),
+            'tarantool': r"datetime.new({year=2022, month=8, day=31, hour=18, min=7, sec=54})",
+        },
+        {
+            'python': pandas.Timestamp(year=2022, month=8, day=31, hour=18, minute=7, second=54,
+                                       microsecond=308543),
+            'msgpack': (b'\x7a\xa3\x0f\x63\x00\x00\x00\x00\x18\xfe\x63\x12\x00\x00\x00\x00'),
+            'tarantool': r"datetime.new({year=2022, month=8, day=31, hour=18, min=7, sec=54, " +
+                         r"nsec=308543000})",
+        },
+        {
+            'python': pandas.Timestamp(year=2022, month=8, day=31, hour=18, minute=7, second=54,
+                                       microsecond=308543, nanosecond=321),
+            'msgpack': (b'\x7a\xa3\x0f\x63\x00\x00\x00\x00\x59\xff\x63\x12\x00\x00\x00\x00'),
+            'tarantool': r"datetime.new({year=2022, month=8, day=31, hour=18, min=7, sec=54, " +
+                         r"nsec=308543321})",
+        },
+    ]
+
+    def test_datetime_msgpack_decode(self):
+        for i in range(len(self.datetime_cases)):
+            with self.subTest(msg=str(i)):
+                datetime_case = self.datetime_cases[i]
+
+                self.assertEqual(unpacker_ext_hook(4, datetime_case['msgpack']),
+                                 datetime_case['python'])
+
+    @skip_or_run_datetime_test
+    def test_datetime_tarantool_decode(self):
+        for i in range(len(self.datetime_cases)):
+            with self.subTest(msg=str(i)):
+                datetime_case = self.datetime_cases[i]
+
+                self.adm(f"box.space['test']:replace{{{i}, {datetime_case['tarantool']}}}")
+
+                self.assertSequenceEqual(self.con.select('test', i),
+                                         [[i, datetime_case['python']]])
+
+    def test_datetime_msgpack_encode(self):
+        for i in range(len(self.datetime_cases)):
+            with self.subTest(msg=str(i)):
+                datetime_case = self.datetime_cases[i]
+
+                self.assertEqual(packer_default(datetime_case['python']),
+                                 msgpack.ExtType(code=4, data=datetime_case['msgpack']))
+
+    @skip_or_run_datetime_test
+    def test_datetime_msgpack_encode(self):
+        for i in range(len(self.datetime_cases)):
+            with self.subTest(msg=str(i)):
+                datetime_case = self.datetime_cases[i]
+
+                self.con.insert('test', [i, datetime_case['python']])
+
+                lua_eval = f"""
+                    local dt = {datetime_case['tarantool']}
+                    
+                    local tuple = box.space['test']:get({i})
+                    assert(tuple ~= nil)
+
+                    if tuple[2] == dt then
+                        return true
+                    else
+                        return nil, ('%s is not equal to expected %s'):format(
+                            tostring(tuple[2]), tostring(dt))
+                    end
+                """
+
+                self.assertSequenceEqual(self.adm(lua_eval), [True])
+
+
     @classmethod
     def tearDownClass(self):
         self.con.close()