Merge pull request #2257 from CosmWasm/test-messagepack-encodings

Add messagepack encoding tests and docs for integer types

Author: webmaster128

packages/std/src/math/mod.rs

@@ -33,7 +33,39 @@ pub use uint64::Uint64;
 macro_rules! impl_int_serde {
     ($ty:ty) => {
         impl ::serde::Serialize for $ty {
-            /// Serializes as an integer string using base 10
+            /// Serializes as an integer string using base 10.
+            ///
+            /// We consistently serialize all `UintXXX` and `IntYYY` types as strings in JSON
+            /// to ensure the best possible compatibility with clients. E.g. JavaScript and jq
+            /// only support up to ~53bit numbers without losing precision, making it hard to use
+            /// serialized `u64`s on other systems than Rust or Go. `Uint64`/`Int64` ensure the full
+            /// 64 bit range is supported. For larger integers, the use of strings is pretty much the
+            /// only reasonable way to store them in JSON.
+            ///
+            /// For binary encodings (notably MessagePack) strings are used too. The reason is that
+            /// in MessagePack integers are limited to 64 bit and we strive for consistent encoding
+            /// within the `UintXXX`/`IntYYY` family. Also for small to mid sized values, decimal strings
+            /// are often more compact than a fixed-length binary encoding.
+            ///
+            /// ## Examples
+            ///
+            /// Serialize to JSON:
+            ///
+            /// ```
+            /// # use cosmwasm_std::{to_json_vec, Uint64};
+            /// let value = Uint64::new(17);
+            /// let serialized = to_json_vec(&value).unwrap();
+            /// assert_eq!(serialized, b"\"17\"");
+            /// ```
+            ///
+            /// Serialize to MessagePack:
+            ///
+            /// ```
+            /// # use cosmwasm_std::{to_msgpack_vec, Uint64};
+            /// let value = Uint64::new(17);
+            /// let serialized = to_msgpack_vec(&value).unwrap();
+            /// assert_eq!(serialized, [0b10100000 ^ 2, b'1', b'7']); // string of lengths 2 with value "17"
+            /// ```
             fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
             where
                 S: ::serde::ser::Serializer,
@@ -43,7 +75,10 @@ macro_rules! impl_int_serde {
         }
 
         impl<'de> ::serde::Deserialize<'de> for $ty {
-            /// Deserialized from an integer string using base 10
+            /// Deserializes from an integer string using base 10.
+            ///
+            /// See the [`Serialize` documentation](#method.serialize) for a few more words
+            /// on the encoding of the `UintXXX`/`IntYYY` family.
             fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
             where
                 D: ::serde::de::Deserializer<'de>,

packages/std/src/msgpack.rs

@@ -35,6 +35,7 @@ where
 #[cfg(test)]
 mod tests {
     use super::*;
+    use crate::{Int128, Int256, Int512, Int64, Uint128, Uint256, Uint512, Uint64};
     use serde::Deserialize;
 
     #[derive(Serialize, Deserialize, Debug, PartialEq)]
@@ -224,4 +225,236 @@ mod tests {
             }
         );
     }
+
+    #[test]
+    fn msgpack_serialization_for_boolean_types() {
+        // "Bool format family stores false or true in 1 byte."
+        let serialized = to_msgpack_vec(&false).unwrap();
+        assert_eq!(serialized, [0xc2]);
+        let serialized = to_msgpack_vec(&true).unwrap();
+        assert_eq!(serialized, [0xc3]);
+    }
+
+    #[test]
+    fn msgpack_serialization_for_integer_types() {
+        // primitive integers up to 64bit
+        // similar to VARINT in protobuf or number in JSON, the encoding does not contain integer size
+        {
+            // "positive fixint stores 7-bit positive integer"
+            let serialized = to_msgpack_vec(&0u8).unwrap();
+            assert_eq!(serialized, [0]);
+            let serialized = to_msgpack_vec(&0u16).unwrap();
+            assert_eq!(serialized, [0]);
+            let serialized = to_msgpack_vec(&0u32).unwrap();
+            assert_eq!(serialized, [0]);
+            let serialized = to_msgpack_vec(&0u64).unwrap();
+            assert_eq!(serialized, [0]);
+            let serialized = to_msgpack_vec(&0i64).unwrap();
+            assert_eq!(serialized, [0]);
+            let serialized = to_msgpack_vec(&7u8).unwrap();
+            assert_eq!(serialized, [7]);
+            let serialized = to_msgpack_vec(&7u16).unwrap();
+            assert_eq!(serialized, [7]);
+            let serialized = to_msgpack_vec(&7u32).unwrap();
+            assert_eq!(serialized, [7]);
+            let serialized = to_msgpack_vec(&7u64).unwrap();
+            assert_eq!(serialized, [7]);
+            let serialized = to_msgpack_vec(&127u32).unwrap();
+            assert_eq!(serialized, [127]);
+
+            // "negative fixint stores 5-bit negative integer"
+            let serialized = to_msgpack_vec(&-1i32).unwrap();
+            assert_eq!(serialized, [255]);
+            let serialized = to_msgpack_vec(&-1i64).unwrap();
+            assert_eq!(serialized, [255]);
+            let serialized = to_msgpack_vec(&-10i64).unwrap();
+            assert_eq!(serialized, [246]);
+            let serialized = to_msgpack_vec(&-24i64).unwrap();
+            assert_eq!(serialized, [232]);
+
+            // "uint 8 stores a 8-bit unsigned integer"
+            let serialized = to_msgpack_vec(&128u32).unwrap();
+            assert_eq!(serialized, [0xcc, 128]);
+            let serialized = to_msgpack_vec(&237u32).unwrap();
+            assert_eq!(serialized, [0xcc, 237]);
+
+            // "uint 16 stores a 16-bit big-endian unsigned integer"
+            let serialized = to_msgpack_vec(&1000u32).unwrap();
+            assert_eq!(serialized, [0xcd, 3, 232]);
+
+            // "uint 32 stores a 32-bit big-endian unsigned integer"
+            let serialized = to_msgpack_vec(&u32::MAX).unwrap();
+            assert_eq!(serialized, [0xce, 255, 255, 255, 255]);
+
+            // "uint 64 stores a 64-bit big-endian unsigned integer"
+            let serialized = to_msgpack_vec(&575747839886u64).unwrap();
+            assert_eq!(serialized, [0xcf, 0, 0, 0, 134, 13, 62, 215, 142]);
+            let serialized = to_msgpack_vec(&u64::MAX).unwrap();
+            assert_eq!(serialized, [0xcf, 255, 255, 255, 255, 255, 255, 255, 255]);
+
+            // "int 8 stores a 8-bit signed integer"
+            let serialized = to_msgpack_vec(&i8::MIN).unwrap();
+            assert_eq!(serialized, [0xd0, 128]);
+            let serialized = to_msgpack_vec(&-111i8).unwrap();
+            assert_eq!(serialized, [0xd0, 145]);
+
+            // "int 16 stores a 16-bit big-endian signed integer"
+            let serialized = to_msgpack_vec(&i16::MIN).unwrap();
+            assert_eq!(serialized, [0xd1, 128, 0]);
+
+            // "int 32 stores a 32-bit big-endian signed integer"
+            let serialized = to_msgpack_vec(&i32::MIN).unwrap();
+            assert_eq!(serialized, [0xd2, 128, 0, 0, 0]);
+
+            // "int 64 stores a 64-bit big-endian signed integer"
+            let serialized = to_msgpack_vec(&i64::MIN).unwrap();
+            assert_eq!(serialized, [0xd3, 128, 0, 0, 0, 0, 0, 0, 0]);
+        }
+
+        // u128/i128
+        // cannot be serialized as integers in messagepack due to the limitation
+        // "a value of an Integer object is limited from -(2^63) upto (2^64)-1"
+        {
+            // encoded as 16 bytes big endian
+            // i.e. takes 18 bytes of storage
+            assert_eq!(
+                to_msgpack_vec(&0u128).unwrap(),
+                [0xc4, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
+            );
+            assert_eq!(
+                to_msgpack_vec(&1u128).unwrap(),
+                [0xc4, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1]
+            );
+            assert_eq!(
+                to_msgpack_vec(&17u128).unwrap(),
+                [0xc4, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17]
+            );
+            assert_eq!(
+                to_msgpack_vec(&u128::MAX).unwrap(),
+                [
+                    0xc4, 16, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255,
+                    255, 255
+                ]
+            );
+
+            assert_eq!(
+                to_msgpack_vec(&0i128).unwrap(),
+                [0xc4, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
+            );
+            assert_eq!(
+                to_msgpack_vec(&1i128).unwrap(),
+                [0xc4, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1]
+            );
+            assert_eq!(
+                to_msgpack_vec(&17i128).unwrap(),
+                [0xc4, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17]
+            );
+            assert_eq!(
+                to_msgpack_vec(&-1i128).unwrap(),
+                [
+                    0xc4, 16, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255,
+                    255, 255
+                ]
+            );
+            assert_eq!(
+                to_msgpack_vec(&i128::MIN).unwrap(),
+                [0xc4, 16, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
+            );
+            assert_eq!(
+                to_msgpack_vec(&i128::MAX).unwrap(),
+                [
+                    0xc4, 16, 127, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255,
+                    255, 255
+                ]
+            );
+        }
+
+        // Uint64/Uint128/Uint256/Uint512
+        {
+            let s = to_msgpack_vec(&Uint64::zero()).unwrap();
+            assert_eq!(s, [0b10100000 ^ 1, b'0']); // string of lengths 1 with value "0"
+            let s = to_msgpack_vec(&Uint128::zero()).unwrap();
+            assert_eq!(s, [0b10100000 ^ 1, b'0']); // string of lengths 1 with value "0"
+            let s = to_msgpack_vec(&Uint256::zero()).unwrap();
+            assert_eq!(s, [0b10100000 ^ 1, b'0']); // string of lengths 1 with value "0"
+            let s = to_msgpack_vec(&Uint512::zero()).unwrap();
+            assert_eq!(s, [0b10100000 ^ 1, b'0']); // string of lengths 1 with value "0"
+
+            let s = to_msgpack_vec(&Uint64::one()).unwrap();
+            assert_eq!(s, [0b10100000 ^ 1, b'1']); // string of lengths 1 with value "1"
+            let s = to_msgpack_vec(&Uint128::one()).unwrap();
+            assert_eq!(s, [0b10100000 ^ 1, b'1']); // string of lengths 1 with value "1"
+            let s = to_msgpack_vec(&Uint256::one()).unwrap();
+            assert_eq!(s, [0b10100000 ^ 1, b'1']); // string of lengths 1 with value "1"
+            let s = to_msgpack_vec(&Uint512::one()).unwrap();
+            assert_eq!(s, [0b10100000 ^ 1, b'1']); // string of lengths 1 with value "1"
+
+            let s = to_msgpack_vec(&Uint64::MAX).unwrap();
+            assert_eq!(
+                s,
+                [
+                    0b10100000 ^ 20,
+                    b'1',
+                    b'8',
+                    b'4',
+                    b'4',
+                    b'6',
+                    b'7',
+                    b'4',
+                    b'4',
+                    b'0',
+                    b'7',
+                    b'3',
+                    b'7',
+                    b'0',
+                    b'9',
+                    b'5',
+                    b'5',
+                    b'1',
+                    b'6',
+                    b'1',
+                    b'5'
+                ]
+            ); // string of lengths 1 with value "1"
+        }
+
+        // Int64/Int128/Int256/Int512
+        {
+            let s = to_msgpack_vec(&Int64::zero()).unwrap();
+            assert_eq!(s, [0b10100000 ^ 1, b'0']); // string of lengths 1 with value "0"
+            let s = to_msgpack_vec(&Int128::zero()).unwrap();
+            assert_eq!(s, [0b10100000 ^ 1, b'0']); // string of lengths 1 with value "0"
+            let s = to_msgpack_vec(&Int256::zero()).unwrap();
+            assert_eq!(s, [0b10100000 ^ 1, b'0']); // string of lengths 1 with value "0"
+            let s = to_msgpack_vec(&Int512::zero()).unwrap();
+            assert_eq!(s, [0b10100000 ^ 1, b'0']); // string of lengths 1 with value "0"
+
+            let s = to_msgpack_vec(&Int64::one()).unwrap();
+            assert_eq!(s, [0b10100000 ^ 1, b'1']); // string of lengths 1 with value "1"
+            let s = to_msgpack_vec(&Int128::one()).unwrap();
+            assert_eq!(s, [0b10100000 ^ 1, b'1']); // string of lengths 1 with value "1"
+            let s = to_msgpack_vec(&Int256::one()).unwrap();
+            assert_eq!(s, [0b10100000 ^ 1, b'1']); // string of lengths 1 with value "1"
+            let s = to_msgpack_vec(&Int512::one()).unwrap();
+            assert_eq!(s, [0b10100000 ^ 1, b'1']); // string of lengths 1 with value "1"
+
+            let s = to_msgpack_vec(&Int64::from(15i32)).unwrap();
+            assert_eq!(s, [0b10100000 ^ 2, b'1', b'5']); // string of lengths 2 with value "15"
+            let s = to_msgpack_vec(&Int128::from(15i32)).unwrap();
+            assert_eq!(s, [0b10100000 ^ 2, b'1', b'5']); // string of lengths 2 with value "15"
+            let s = to_msgpack_vec(&Int256::from(15i32)).unwrap();
+            assert_eq!(s, [0b10100000 ^ 2, b'1', b'5']); // string of lengths 2 with value "15"
+            let s = to_msgpack_vec(&Int512::from(15i32)).unwrap();
+            assert_eq!(s, [0b10100000 ^ 2, b'1', b'5']); // string of lengths 2 with value "15"
+
+            let s = to_msgpack_vec(&Int64::from(-1i64)).unwrap();
+            assert_eq!(s, [0b10100000 ^ 2, b'-', b'1']); // string of lengths 2 with value "-1"
+            let s = to_msgpack_vec(&Int128::from(-1i64)).unwrap();
+            assert_eq!(s, [0b10100000 ^ 2, b'-', b'1']); // string of lengths 2 with value "-1"
+            let s = to_msgpack_vec(&Int256::from(-1i64)).unwrap();
+            assert_eq!(s, [0b10100000 ^ 2, b'-', b'1']); // string of lengths 2 with value "-1"
+            let s = to_msgpack_vec(&Int512::from(-1i64)).unwrap();
+            assert_eq!(s, [0b10100000 ^ 2, b'-', b'1']); // string of lengths 2 with value "-1"
+        }
+    }
 }