RDART-1062: Allow missing values (implicit null) (#1736)

* Allow missing values (implicit null)

* Update .expected files

* Rerun builder in example

* Update CHANGELOG

* Support default values during EJson deserialization

* Rerun builder in example

* Add tests

* Fix Decimal128 and RealmValue

* register allow super types to be specified

* Support DBRef

* Extend RealmValue serialization tests

* Accept a allowCustom argument on fromEJson

* Lookup PK dynmically using object schema, during RealmValue of RealmObject serialization

* Support deserializing RealmValue from DBKey

* Support Set

* Make SchemaObject const constructable again (unrelated to fix)

* Avoid allowCustom argument

* Update packages/realm_dart/test/serialization_test.dart

Co-authored-by: Nikola Irinchev <[email protected]>

* Update CHANGELOG.md

---------

Co-authored-by: Nikola Irinchev <[email protected]>

Author: Kasper Overgård Nielsen

CHANGELOG.md

@@ -3,6 +3,7 @@
 ### Enhancements
 * Added a new parameter of type `SyncTimeoutOptions` to `AppConfiguration`. It allows users to control sync timings, such as ping/pong intervals as well various connection timeouts. (Issue [#1763](https://github.com/realm/realm-dart/issues/1763))
 * Added a new parameter `cancelAsyncOperationsOnNonFatalErrors` on `Configuration.flexibleSync` that allows users to control whether non-fatal errors such as connection timeouts should be surfaced in the form of errors or if sync should try and reconnect in the background. (PR [#1764](https://github.com/realm/realm-dart/pull/1764))
+* Allow nullable and other optional fields to be absent in EJson, when deserializing realm objects. (Issue [#1735](https://github.com/realm/realm-dart/issues/1735))
 
 ### Fixed
 * Fixed an issue where creating a flexible sync configuration with an embedded object not referenced by any top-level object would throw a "No such table" exception with no meaningful information about the issue. Now a `RealmException` will be thrown that includes the offending object name, as well as more precise text for what the root cause of the error is. (PR [#1748](https://github.com/realm/realm-dart/pull/1748))

packages/CHANGELOG.ejson.md

@@ -1,3 +1,10 @@
+## 0.4.0
+
+- `fromEJson<T>` now accepts a `defaultValue` argument that is returned if  
+  `null` is passed as `ejson`.
+- `register<T>` takes an optional `superTypes` argument to specify the super 
+  types of `T` if needed.
+
 ## 0.3.1
 
 - Update sane_uuid dependency to ^1.0.0 (compensate for breaking change)

packages/ejson/lib/src/configuration.dart

@@ -9,8 +9,8 @@ import 'encoding.dart';
 
 /// Register custom EJSON [encoder] and [decoder] for a type [T].
 /// The last registered codec pair for a given type [T] will be used.
-void register<T>(EJsonEncoder<T> encoder, EJsonDecoder<T> decoder) {
-  TypePlus.add<T>();
+void register<T>(EJsonEncoder<T> encoder, EJsonDecoder<T> decoder, {Iterable<Type>? superTypes}) {
+  TypePlus.add<T>(superTypes: superTypes);
   customEncoders[T] = encoder;
   customDecoders[T] = decoder;
 }

packages/ejson/lib/src/decoding.dart

@@ -19,6 +19,7 @@ const _commonDecoders = {
   Object: _decodeAny,
   Iterable: _decodeArray,
   List: _decodeArray,
+  Set: _decodeSet,
   bool: _decodeBool,
   DateTime: _decodeDate,
   Defined: _decodeDefined,
@@ -32,34 +33,42 @@ const _commonDecoders = {
   Symbol: _decodeSymbol,
   Uint8List: _decodeBinary,
   Uuid: _decodeUuid,
+  DBRef: _decodeDBRef,
   Undefined: _decodeUndefined,
   UndefinedOr: _decodeUndefinedOr,
 };
 
-/// Custom decoders for specific types. Use `register` to add a custom decoder.
-final customDecoders = <Type, Function>{};
-
-final _decoders = () {
+/// Predefined decoders for common types
+final commonDecoders = () {
   // register extra common types on first access
   undefinedOr<T>(dynamic f) => f<UndefinedOr<T>>();
   TypePlus.addFactory(undefinedOr);
   TypePlus.addFactory(<T>(dynamic f) => f<Defined<T>>(), superTypes: [undefinedOr]);
   TypePlus.addFactory(<T>(dynamic f) => f<Undefined<T>>(), superTypes: [undefinedOr]);
+  TypePlus.addFactory(<T>(dynamic f) => f<DBRef<T>>());
   TypePlus.add<BsonKey>();
   TypePlus.add<ObjectId>();
   TypePlus.add<Uint8List>();
   TypePlus.add<Uuid>();
 
-  return CombinedMapView([customDecoders, _commonDecoders]);
+  return _commonDecoders;
 }();
 
+/// Custom decoders for specific types. Use `register` to add a custom decoder.
+final customDecoders = <Type, Function>{};
+
+final _decoders = CombinedMapView([customDecoders, commonDecoders]);
+
 /// Converts [ejson] to type [T].
 ///
+/// [defaultValue] is returned if set, and [ejson] is `null`.
+///
 /// Throws [InvalidEJson] if [ejson] is not valid for [T].
 /// Throws [MissingDecoder] if no decoder is registered for [T].
-T fromEJson<T>(EJsonValue ejson) {
+T fromEJson<T>(EJsonValue ejson, {T? defaultValue}) {
   final type = T;
   final nullable = type.isNullable;
+  if (ejson == null && defaultValue != null) return defaultValue;
   final decoder = nullable ? _decodeNullable : _decoders[type.base];
   if (decoder == null) {
     throw MissingDecoder._(ejson, type);
@@ -94,32 +103,30 @@ dynamic _decodeAny(EJsonValue ejson) {
     {'\$numberDouble': _} => _decodeDouble(ejson),
     {'\$numberInt': _} => _decodeInt(ejson),
     {'\$numberLong': _} => _decodeInt(ejson),
+    {'\$ref': _, '\$id': _} => _decodeDBRef<dynamic>(ejson),
     {'\$regex': _} => _decodeString(ejson),
     {'\$symbol': _} => _decodeSymbol(ejson),
     {'\$undefined': _} => _decodeUndefined<dynamic>(ejson),
     {'\$oid': _} => _decodeObjectId(ejson),
     {'\$binary': {'base64': _, 'subType': '04'}} => _decodeUuid(ejson),
     {'\$binary': _} => _decodeBinary(ejson),
-    List<dynamic> _ => _decodeArray<dynamic>(ejson),
-    Map<dynamic, dynamic> _ => _tryDecodeCustom(ejson) ?? _decodeDocument<String, dynamic>(ejson), // other maps goes last!!
+    List _ => _decodeArray<dynamic>(ejson),
+    Set _ => _decodeSet<dynamic>(ejson),
+    Map _ => _decodeDocument<String, dynamic>(ejson), // other maps goes last!!
     _ => raiseInvalidEJson<dynamic>(ejson),
   };
 }
 
-dynamic _tryDecodeCustom(EJsonValue ejson) {
-  for (final decoder in customDecoders.values) {
-    try {
-      return decoder(ejson);
-    } catch (_) {
-      // ignore
-    }
-  }
-  return null;
+List<T> _decodeArray<T>(EJsonValue ejson) {
+  return switch (ejson) {
+    Iterable i => i.map((ejson) => fromEJson<T>(ejson)).toList(),
+    _ => raiseInvalidEJson(ejson),
+  };
 }
 
-List<T> _decodeArray<T>(EJsonValue ejson) {
+Set<T> _decodeSet<T>(EJsonValue ejson) {
   return switch (ejson) {
-    Iterable<dynamic> i => i.map((ejson) => fromEJson<T>(ejson)).toList(),
+    Iterable i => i.map((ejson) => fromEJson<T>(ejson)).toSet(),
     _ => raiseInvalidEJson(ejson),
   };
 }
@@ -139,14 +146,24 @@ DateTime _decodeDate(EJsonValue ejson) {
   };
 }
 
+DBRef<KeyT> _decodeDBRef<KeyT>(EJsonValue ejson) {
+  switch (ejson) {
+    case {'\$ref': String collection, '\$id': EJsonValue id}:
+      KeyT key = fromEJson(id);
+      return DBRef.new.callWith(parameters: [collection, key], typeArguments: [key.runtimeType]) as DBRef<KeyT>;
+    default:
+      return raiseInvalidEJson(ejson);
+  }
+}
+
 Defined<T> _decodeDefined<T>(EJsonValue ejson) {
   if (ejson case {'\$undefined': 1}) return raiseInvalidEJson(ejson);
-  return Defined<T>(fromEJson<T>(ejson));
+  return Defined(fromEJson(ejson));
 }
 
 Map<K, V> _decodeDocument<K, V>(EJsonValue ejson) {
   return switch (ejson) {
-    Map<dynamic, dynamic> m => m.map((key, value) => MapEntry(key as K, fromEJson<V>(value))),
+    Map m => m.map((key, value) => MapEntry(key as K, fromEJson(value))),
     _ => raiseInvalidEJson(ejson),
   };
 }
@@ -229,14 +246,14 @@ Symbol _decodeSymbol(EJsonValue ejson) {
 
 Undefined<T> _decodeUndefined<T>(EJsonValue ejson) {
   return switch (ejson) {
-    {'\$undefined': 1} => Undefined<T>(),
+    {'\$undefined': 1} => Undefined(),
     _ => raiseInvalidEJson(ejson),
   };
 }
 
 UndefinedOr<T> _decodeUndefinedOr<T>(EJsonValue ejson) {
   return switch (ejson) {
-    {'\$undefined': 1} => Undefined<T>(),
+    {'\$undefined': 1} => Undefined(),
     _ => _decodeDefined(ejson),
   };
 }

packages/ejson/lib/src/encoding.dart

@@ -34,12 +34,13 @@ EJsonValue _encodeAny(Object? value) {
     null => null,
     bool b => _encodeBool(b),
     DateTime d => _encodeDate(d),
+    DBRef d => _encodeDBRef(d),
     Defined<dynamic> d => _encodeDefined(d),
     double d => _encodeDouble(d),
     int i => _encodeInt(i),
     BsonKey k => _encodeKey(k),
     Uint8List b => _encodeBinary(b, subtype: '00'),
-    Iterable<dynamic> l => _encodeArray(l),
+    Iterable<dynamic> l => _encodeArray(l), // handles List and Set as well
     Map<dynamic, dynamic> m => _encodeDocument(m),
     ObjectId o => _encodeObjectId(o),
     String s => _encodeString(s),
@@ -71,6 +72,13 @@ EJsonValue _encodeDate(DateTime value) {
   };
 }
 
+EJsonValue _encodeDBRef(DBRef<dynamic> d) {
+  return {
+    '\$ref': d.collection,
+    '\$id': toEJson(d.id),
+  };
+}
+
 EJsonValue _encodeDefined(Defined<dynamic> defined) => toEJson(defined.value);
 
 EJsonValue _encodeDocument(Map<dynamic, dynamic> map) => map.map((k, v) => MapEntry(k, toEJson(v)));
@@ -150,6 +158,12 @@ extension DateTimeEJsonEncoderExtension on DateTime {
   EJsonValue toEJson() => _encodeDate(this);
 }
 
+extension DBRefEJsonEncoderExtension on DBRef<dynamic> {
+  /// Converts this [DBRef] to EJson
+  @pragma('vm:prefer-inline')
+  EJsonValue toEJson() => _encodeDBRef(this);
+}
+
 extension DefinedEJsonEncoderExtension on Defined<dynamic> {
   /// Converts this [Defined] to EJson
   @pragma('vm:prefer-inline')

packages/ejson/lib/src/types.dart

@@ -5,24 +5,24 @@ enum EJsonType {
   array,
   binary,
   boolean,
-  date,
+  databaseRef,
+  date, // use this instead of timestamp
   decimal128,
   document,
   double,
   int32,
   int64,
   maxKey,
   minKey,
+  nil, // aka. null
   objectId,
   string,
   symbol,
-  nil, // aka. null
   undefined,
   // TODO: The following is not supported yet
   // code,
   // codeWithScope,
-  // databasePointer,
-  // databaseRef,
+  // databasePointer, // deprecated
   // regularExpression,
   // timestamp, // This is not what you think, see https://www.mongodb.com/docs/manual/reference/mongodb-extended-json/#mongodb-bsontype-Timestamp
 }
@@ -31,6 +31,22 @@ enum EJsonType {
 /// and [MinKey](https://www.mongodb.com/docs/manual/reference/mongodb-extended-json/#mongodb-bsontype-MinKey)
 enum BsonKey { min, max }
 
+/// See [DBRef](https://github.com/mongodb/specifications/blob/master/source/dbref.md)
+/// This is not technically a BSON type, but a common convention. 
+final class DBRef<KeyT> {
+  // Do we need to support the database name?
+  final String collection;
+  final KeyT id;
+
+  const DBRef(this.collection, this.id);
+
+  @override
+  int get hashCode => Object.hash(collection, id);
+
+  @override
+  bool operator ==(Object other) => other is DBRef<KeyT> && collection == other.collection && id == other.id;
+}
+
 sealed class UndefinedOr<T> {
   const UndefinedOr();
 }

packages/ejson/test/ejson_test.dart

@@ -10,9 +10,41 @@ import 'package:ejson/ejson.dart';
 import 'package:objectid/objectid.dart';
 import 'package:sane_uuid/uuid.dart';
 import 'package:test/test.dart';
+import 'package:type_plus/type_plus.dart';
 
 import 'person.dart';
 
+bool _canDecodeAny<T>([Type? type]) {
+  commonDecoders; // ensure common types has been registered;
+  type ??= T;
+  if (type.isNullable) return _canDecodeAny(type.base);
+  if ([
+    dynamic,
+    Null,
+    Object,
+    bool,
+    double,
+    int,
+    num,
+    String,
+    DateTime,
+    BsonKey,
+    Symbol,
+    ObjectId,
+    Uuid,
+    Uint8List,
+  ].contains(type)) return true;
+  if ([
+    List,
+    Set,
+    Map,
+    DBRef,
+    Undefined,
+    UndefinedOr,
+  ].contains(type.base)) return type.args.every(_canDecodeAny);
+  return false;
+}
+
 void _testCase<T>(T value, EJsonValue expected) {
   test('encode from $value of type $T', () {
     expect(toEJson(value), expected);
@@ -46,7 +78,7 @@ void _testCase<T>(T value, EJsonValue expected) {
     expect(() => fromEJson(expected), returnsNormally);
   });
 
-  if (value is! Defined) {
+  if (_canDecodeAny<T>()) {
     test('roundtrip $value of type $T as dynamic', () {
       // no <T> here, so dynamic
       expect(fromEJson(toEJson(value)), value);
@@ -82,6 +114,7 @@ void main() {
     expect([1, 2, 3].toEJson(), toEJson([1, 2, 3]));
     expect({'a': 1, 'b': 2}.toEJson(), toEJson({'a': 1, 'b': 2}));
     expect(DateTime(1974, 4, 10, 2, 42, 12, 202).toEJson(), toEJson(DateTime(1974, 4, 10, 2, 42, 12, 202)));
+    expect(DBRef('collection', 42).toEJson(), toEJson(DBRef('collection', 42)));
     expect((#sym).toEJson(), toEJson(#sym));
     expect(BsonKey.max.toEJson(), toEJson(BsonKey.max));
     expect(BsonKey.min.toEJson(), toEJson(BsonKey.min));
@@ -105,11 +138,17 @@ void main() {
   group('invalid', () {
     _invalidTestCase<bool>();
     _invalidTestCase<DateTime>();
+    _invalidTestCase<DBRef>();
+    _invalidTestCase<DBRef<int>>();
     _invalidTestCase<double>({'\$numberDouble': 'foobar'});
     _invalidTestCase<double>();
     _invalidTestCase<int>();
     _invalidTestCase<BsonKey>();
+    _invalidTestCase<List>();
     _invalidTestCase<List<int>>();
+    _invalidTestCase<Set>();
+    _invalidTestCase<Set<int>>();
+    _invalidTestCase<Map>([]);
     _invalidTestCase<Map<int, int>>([]);
     _invalidTestCase<Null>();
     _invalidTestCase<num>();
@@ -118,6 +157,7 @@ void main() {
     _invalidTestCase<String>();
     _invalidTestCase<Symbol>();
     _invalidTestCase<Uint8List>();
+    _invalidTestCase<Undefined>();
     _invalidTestCase<Undefined<int>>();
     _invalidTestCase<Uuid>();
 
@@ -151,6 +191,16 @@ void main() {
                 ]
               : [1, 2, 3],
         );
+        _testCase(
+          {1, 2, 3},
+          canonical
+              ? {
+                  {'\$numberInt': '1'},
+                  {'\$numberInt': '2'},
+                  {'\$numberInt': '3'},
+                }
+              : {1, 2, 3},
+        );
         _testCase(
           [1, 1.1],
           canonical
@@ -190,6 +240,10 @@ void main() {
         _testCase(#sym, {'\$symbol': 'sym'});
         _testCase(BsonKey.max, {'\$maxKey': 1});
         _testCase(BsonKey.min, {'\$minKey': 1});
+        _testCase(const DBRef<int>('collection', 42), {
+          '\$ref': 'collection',
+          '\$id': canonical ? {'\$numberInt': '42'} : 42,
+        });
         _testCase(undefined, {'\$undefined': 1});
         _testCase(const Undefined<int?>(), {'\$undefined': 1});
         _testCase(Undefined<int?>(), {'\$undefined': 1});

packages/ejson_generator/test/ctor_test.dart

@@ -85,13 +85,6 @@ void _testCase<T>(T value, EJsonValue expected) {
   test('roundtrip $expected as $T', () {
     expect(toEJson(fromEJson<T>(expected)), expected);
   });
-
-  test('roundtrip $expected of type $T as dynamic', () {
-    // no <T> here, so dynamic
-    final decoded = fromEJson<dynamic>(expected);
-    expect(decoded, isA<T>());
-    expect(toEJson(decoded), expected);
-  });
 }
 
 void main() {