Merge branch 'master' of github.com:MongoEngine/mongoengine into geonear_and_collstats_must_come_first_pipeline

Author: bagerard

docs/apireference.rst

@@ -77,7 +77,6 @@ Fields
 .. autoclass:: mongoengine.fields.EmailField
 .. autoclass:: mongoengine.fields.EnumField
 .. autoclass:: mongoengine.fields.IntField
-.. autoclass:: mongoengine.fields.LongField
 .. autoclass:: mongoengine.fields.FloatField
 .. autoclass:: mongoengine.fields.DecimalField
 .. autoclass:: mongoengine.fields.Decimal128Field

docs/changelog.rst

@@ -7,16 +7,19 @@ Changelog
 Development
 ===========
 - (Fill this out as you fix issues and develop your features).
+- Fix Document.compare_indexes() not working correctly for text indexes on multiple fields #2612
 - Add support for transaction through run_in_transaction (kudos to juannyG for this) #2569
   Some considerations:
     - make sure to read https://www.mongodb.com/docs/manual/core/transactions-in-applications/#callback-api-vs-core-api
     - run_in_transaction context manager relies on Pymongo coreAPI, it will retry automatically in case of `UnknownTransactionCommitResult` but not `TransientTransactionError` exceptions
     - Using .count() in a transaction will always use Collection.count_document (as estimated_document_count is not supported in transactions)
 - Fix use of $geoNear or $collStats in aggregate #2493
-- Further to the deprecation warning, remove ability to use an unpacked list to `Queryset.aggregate(*pipeline)`, a plain list must be provided instead `Queryset.aggregate(pipeline)`, as it's closer to pymongo interface
-- Further to the deprecation warning, remove `full_response` from `QuerySet.modify` as it wasn't supported with Pymongo 3+
+- BREAKING CHANGE: Further to the deprecation warning, remove ability to use an unpacked list to `Queryset.aggregate(*pipeline)`, a plain list must be provided instead `Queryset.aggregate(pipeline)`, as it's closer to pymongo interface
+- BREAKING CHANGE: Further to the deprecation warning, remove `full_response` from `QuerySet.modify` as it wasn't supported with Pymongo 3+
 - Fixed stacklevel of many warnings (to point places emitting the warning more accurately)
 - Add support for collation/hint/comment to delete/update and aggregate #2842
+- BREAKING CHANGE: Remove LongField as it's equivalent to IntField since we drop support to Python2 long time ago (User should simply switch to IntField) #2309
+- BugFix - Calling .clear on a ListField wasn't being marked as changed (and flushed to db upon .save()) #2858
 
 Changes in 0.29.0
 =================

docs/guide/defining-documents.rst

@@ -88,7 +88,6 @@ are as follows:
 * :class:`~mongoengine.fields.ImageField`
 * :class:`~mongoengine.fields.IntField`
 * :class:`~mongoengine.fields.ListField`
-* :class:`~mongoengine.fields.LongField`
 * :class:`~mongoengine.fields.MapField`
 * :class:`~mongoengine.fields.ObjectIdField`
 * :class:`~mongoengine.fields.ReferenceField`

mongoengine/base/datastructures.py

@@ -177,6 +177,7 @@ def __setitem__(self, key, value):
     remove = mark_as_changed_wrapper(list.remove)
     reverse = mark_as_changed_wrapper(list.reverse)
     sort = mark_as_changed_wrapper(list.sort)
+    clear = mark_as_changed_wrapper(list.clear)
     __delitem__ = mark_as_changed_wrapper(list.__delitem__)
     __iadd__ = mark_as_changed_wrapper(list.__iadd__)
     __imul__ = mark_as_changed_wrapper(list.__imul__)

mongoengine/base/utils.py

@@ -20,3 +20,13 @@ def __get__(self, instance, owner):
 
     def __set__(self, instance, value):
         raise AttributeError("Can not set attribute LazyRegexCompiler")
+
+
+class NonOrderedList(list):
+    """Simple utility class to compare lists without considering order (useful in context of indexes)"""
+
+    def __eq__(self, other):
+        if isinstance(other, list):
+            # Compare sorted versions of the lists
+            return sorted(self) == sorted(other)
+        return False

mongoengine/document.py

@@ -14,6 +14,7 @@
     TopLevelDocumentMetaclass,
     get_document,
 )
+from mongoengine.base.utils import NonOrderedList
 from mongoengine.common import _import_class
 from mongoengine.connection import (
     DEFAULT_CONNECTION_NAME,
@@ -1043,9 +1044,13 @@ def compare_indexes(cls):
                 # Useful for text indexes (but not only)
                 index_type = info["key"][0][1]
                 text_index_fields = info.get("weights").keys()
-                existing.append([(key, index_type) for key in text_index_fields])
+                # Use NonOrderedList to avoid order comparison, see #2612
+                existing.append(
+                    NonOrderedList([(key, index_type) for key in text_index_fields])
+                )
             else:
                 existing.append(info["key"])
+
         missing = [index for index in required if index not in existing]
         extra = [index for index in existing if index not in required]
 

mongoengine/fields.py

@@ -14,7 +14,6 @@
 import pymongo
 from bson import SON, Binary, DBRef, ObjectId
 from bson.decimal128 import Decimal128, create_decimal128_context
-from bson.int64 import Int64
 from pymongo import ReturnDocument
 
 try:
@@ -68,7 +67,6 @@
     "URLField",
     "EmailField",
     "IntField",
-    "LongField",
     "FloatField",
     "DecimalField",
     "BooleanField",
@@ -367,13 +365,6 @@ def prepare_query_value(self, op, value):
         return super().prepare_query_value(op, int(value))
 
 
-class LongField(IntField):
-    """64-bit integer field. (Equivalent to IntField since the support to Python2 was dropped)"""
-
-    def to_mongo(self, value):
-        return Int64(value)
-
-
 class FloatField(BaseField):
     """Floating point number field."""
 

tests/document/test_indexes.py

@@ -66,6 +66,8 @@ class BlogPost(InheritFrom):
         for expected in expected_specs:
             assert expected["fields"] in info
 
+        assert BlogPost.compare_indexes() == {"missing": [], "extra": []}
+
     def _index_test_inheritance(self, InheritFrom):
         class BlogPost(InheritFrom):
             date = DateTimeField(db_field="addDate", default=datetime.now)
@@ -949,6 +951,8 @@ class MyDoc(Document):
         ]["key"]
         assert info["provider_ids.foo_1_provider_ids.bar_1"]["sparse"]
 
+        assert MyDoc.compare_indexes() == {"missing": [], "extra": []}
+
     def test_text_indexes(self):
         class Book(Document):
             title = DictField()
@@ -968,6 +972,8 @@ class Book(Document):
         assert "ref_id_hashed" in indexes
         assert ("ref_id", "hashed") in indexes["ref_id_hashed"]["key"]
 
+        assert Book.compare_indexes() == {"missing": [], "extra": []}
+
     def test_indexes_after_database_drop(self):
         """
         Test to ensure that indexes are not re-created on a collection
@@ -1045,6 +1051,8 @@ class TestChildDoc(TestDoc):
         TestDoc.ensure_indexes()
         TestChildDoc.ensure_indexes()
 
+        assert TestDoc.compare_indexes() == {"missing": [], "extra": []}
+
         index_info = TestDoc._get_collection().index_information()
         for key in index_info:
             del index_info[key][
@@ -1082,9 +1090,34 @@ class TestDoc(Document):
         TestDoc.drop_collection()
         TestDoc.ensure_indexes()
 
+        assert TestDoc.compare_indexes() == {"missing": [], "extra": []}
+
         index_info = TestDoc._get_collection().index_information()
         assert "shard_1_1__cls_1_txt_1_1" in index_info
 
+    def test_compare_indexes_works_with_compound_text_indexes(self):
+        """The order of the fields in case of text indexes don't matter
+        so it's important to ensure that the compare_indexes method works that way
+        https://github.com/MongoEngine/mongoengine/issues/2612
+        """
+
+        class Sample1(Document):
+            a = StringField()
+            b = StringField()
+
+            meta = {"indexes": [{"fields": ["$a", "$b"]}]}
+
+        class Sample2(Document):
+            a = StringField()
+            b = StringField()
+
+            meta = {"indexes": [{"fields": ["$b", "$a"]}]}
+
+        Sample1.drop_collection()
+        Sample2.drop_collection()
+        assert Sample1.compare_indexes() == {"missing": [], "extra": []}
+        assert Sample2.compare_indexes() == {"missing": [], "extra": []}
+
 
 if __name__ == "__main__":
     unittest.main()

tests/fields/test_int_field.py

@@ -1,4 +1,5 @@
 import pytest
+from bson import Int64
 
 from mongoengine import *
 from tests.utils import MongoDBTestCase
@@ -42,3 +43,25 @@ class TestDocument(Document):
 
         assert 1 == TestDocument.objects(int_fld__ne=None).count()
         assert 1 == TestDocument.objects(int_fld__ne=1).count()
+
+    def test_int_field_long_field_migration(self):
+        class DeprecatedLongField(IntField):
+            """64-bit integer field. (Equivalent to IntField since the support to Python2 was dropped)"""
+
+            def to_mongo(self, value):
+                return Int64(value)
+
+        class TestDocument(Document):
+            long = DeprecatedLongField()
+
+        TestDocument.drop_collection()
+        TestDocument(long=10).save()
+
+        v = TestDocument.objects().first().long
+
+        # simulate a migration to IntField
+        class TestDocument(Document):
+            long = IntField()
+
+        assert TestDocument.objects(long=10).count() == 1
+        assert TestDocument.objects().first().long == v

tests/fields/test_long_field.py

@@ -5,11 +5,17 @@
 
 from mongoengine import *
 from mongoengine.queryset import Q, transform
+from tests.utils import MongoDBTestCase
 
 
-class TestTransform(unittest.TestCase):
-    def setUp(self):
-        connect(db="mongoenginetest")
+class TestTransform(MongoDBTestCase):
+
+    def test_transform_str_datetime(self):
+        data = {"date": {"$ne": "2015-12-01T00:00:00"}}
+        assert transform.query(**data) == {"date": {"$ne": "2015-12-01T00:00:00"}}
+        assert transform.query(date__ne="2015-12-01T00:00:00") == {
+            "date": {"$ne": "2015-12-01T00:00:00"}
+        }
 
     def test_transform_query(self):
         """Ensure that the _transform_query function operates correctly."""

tests/queryset/test_transform.py

@@ -185,7 +185,7 @@ def test___iter__(self):
         base_list = BaseList(values, instance=None, name="my_name")
         assert values == list(base_list)
 
-    def test___iter___allow_modification_while_iterating_withou_error(self):
+    def test___iter___allow_modification_while_iterating_without_error(self):
         # regular list allows for this, thus this subclass must comply to that
         base_list = BaseList([True, False, True, False], instance=None, name="my_name")
         for idx, val in enumerate(base_list):
@@ -365,6 +365,11 @@ def test_sort_calls_with_key(self):
         base_list.sort(key=lambda i: str(i))
         assert base_list == [1, 11, 2]
 
+    def test_clear_calls_mark_as_changed(self):
+        base_list = self._get_baselist([True, False])
+        base_list.clear()
+        assert base_list._instance._changed_fields == ["my_name"]
+
 
 class TestStrictDict(unittest.TestCase):
     def setUp(self):