Skip to content

Commit 20748c6

Browse files
bagerardbagerard
committed
Merge branch 'master' of github.com:MongoEngine/mongoengine into geonear_and_collstats_must_come_first_pipeline
2 parents 3b6d437 + 94b5b69 commit 20748c6

File tree

13 files changed

+277
-105
lines changed

13 files changed

+277
-105
lines changed

‎.github/workflows/start_mongo.sh

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,13 @@ mongodb_dir=$(find ${PWD}/ -type d -name "mongodb-linux-x86_64*")
66

77
mkdir $mongodb_dir/data
88

9-
$mongodb_dir/bin/mongod --dbpath $mongodb_dir/data --logpath $mongodb_dir/mongodb.log --fork --replSet mongoengine
9+
args=(--dbpath $mongodb_dir/data --logpath $mongodb_dir/mongodb.log --fork --replSet mongoengine)
10+
if (( $(echo "$MONGODB > 3.8" | bc -l) )); then
11+
args+=(--setParameter maxTransactionLockRequestTimeoutMillis=1000)
12+
fi
13+
14+
$mongodb_dir/bin/mongod "${args[@]}"
15+
1016
if (( $(echo "$MONGODB < 6.0" | bc -l) )); then
1117
mongo --verbose --eval "rs.initiate()"
1218
mongo --quiet --eval "rs.status().ok"

‎docs/changelog.rst

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,10 @@ Development
1313
- run_in_transaction context manager relies on Pymongo coreAPI, it will retry automatically in case of `UnknownTransactionCommitResult` but not `TransientTransactionError` exceptions
1414
- Using .count() in a transaction will always use Collection.count_document (as estimated_document_count is not supported in transactions)
1515
- Fix use of $geoNear or $collStats in aggregate #2493
16+
- Further to the deprecation warning, remove ability to use an unpacked list to `Queryset.aggregate(*pipeline)`, a plain list must be provided instead `Queryset.aggregate(pipeline)`, as it's closer to pymongo interface
17+
- Further to the deprecation warning, remove `full_response` from `QuerySet.modify` as it wasn't supported with Pymongo 3+
18+
- Fixed stacklevel of many warnings (to point places emitting the warning more accurately)
19+
- Add support for collation/hint/comment to delete/update and aggregate #2842
1620

1721
Changes in 0.29.0
1822
=================

‎entrypoint.sh

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,4 +3,4 @@
33
mongod --replSet mongoengine --fork --logpath=/var/log/mongodb.log
44
mongo db --eval "rs.initiate()"
55
mongod --shutdown
6-
mongod --replSet mongoengine --bind_ip 0.0.0.0
6+
mongod --replSet mongoengine --bind_ip 0.0.0.0 --setParameter maxTransactionLockRequestTimeoutMillis=1000

‎mongoengine/base/document.py

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -455,6 +455,7 @@ def to_json(self, *args, **kwargs):
455455
"representation to use. This will be changed to "
456456
"uuid_representation=UNSPECIFIED in a future release.",
457457
DeprecationWarning,
458+
stacklevel=2,
458459
)
459460
kwargs["json_options"] = LEGACY_JSON_OPTIONS
460461
return json_util.dumps(self.to_mongo(use_db_field), *args, **kwargs)
@@ -486,6 +487,7 @@ def from_json(cls, json_data, created=False, **kwargs):
486487
"representation to use. This will be changed to "
487488
"uuid_representation=UNSPECIFIED in a future release.",
488489
DeprecationWarning,
490+
stacklevel=2,
489491
)
490492
kwargs["json_options"] = LEGACY_JSON_OPTIONS
491493
return cls._from_son(json_util.loads(json_data, **kwargs), created=created)

‎mongoengine/base/metaclasses.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -307,7 +307,7 @@ def __new__(mcs, name, bases, attrs):
307307
and not parent_doc_cls._meta.get("abstract", True)
308308
):
309309
msg = "Trying to set a collection on a subclass (%s)" % name
310-
warnings.warn(msg, SyntaxWarning)
310+
warnings.warn(msg, SyntaxWarning, stacklevel=2)
311311
del attrs["_meta"]["collection"]
312312

313313
# Ensure abstract documents have abstract bases

‎mongoengine/connection.py

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -211,6 +211,7 @@ def _get_connection_settings(
211211
"older drivers in those languages. This will be changed to "
212212
"'unspecified' in a future release.",
213213
DeprecationWarning,
214+
stacklevel=3,
214215
)
215216
kwargs["uuidRepresentation"] = "pythonLegacy"
216217

‎mongoengine/queryset/base.py

Lines changed: 45 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -519,8 +519,20 @@ def delete(self, write_concern=None, _from_doc_delete=False, cascade_refs=None):
519519
write_concern=write_concern, **{"pull_all__%s" % field_name: self}
520520
)
521521

522+
kwargs = {}
523+
if self._hint not in (-1, None):
524+
kwargs["hint"] = self._hint
525+
if self._collation:
526+
kwargs["collation"] = self._collation
527+
if self._comment:
528+
kwargs["comment"] = self._comment
529+
522530
with set_write_concern(queryset._collection, write_concern) as collection:
523-
result = collection.delete_many(queryset._query, session=_get_session())
531+
result = collection.delete_many(
532+
queryset._query,
533+
session=_get_session(),
534+
**kwargs,
535+
)
524536

525537
# If we're using an unack'd write concern, we don't really know how
526538
# many items have been deleted at this point, hence we only return
@@ -582,6 +594,15 @@ def update(
582594
update["$set"]["_cls"] = queryset._document._class_name
583595
else:
584596
update["$set"] = {"_cls": queryset._document._class_name}
597+
598+
kwargs = {}
599+
if self._hint not in (-1, None):
600+
kwargs["hint"] = self._hint
601+
if self._collation:
602+
kwargs["collation"] = self._collation
603+
if self._comment:
604+
kwargs["comment"] = self._comment
605+
585606
try:
586607
with set_read_write_concern(
587608
queryset._collection, write_concern, read_concern
@@ -595,6 +616,7 @@ def update(
595616
upsert=upsert,
596617
array_filters=array_filters,
597618
session=_get_session(),
619+
**kwargs,
598620
)
599621
if full_result:
600622
return result
@@ -675,7 +697,6 @@ def update_one(
675697
def modify(
676698
self,
677699
upsert=False,
678-
full_response=False,
679700
remove=False,
680701
new=False,
681702
array_filters=None,
@@ -687,15 +708,7 @@ def modify(
687708
parameter. If no documents match the query and `upsert` is false,
688709
returns ``None``. If upserting and `new` is false, returns ``None``.
689710
690-
If the full_response parameter is ``True``, the return value will be
691-
the entire response object from the server, including the 'ok' and
692-
'lastErrorObject' fields, rather than just the modified document.
693-
This is useful mainly because the 'lastErrorObject' document holds
694-
information about the command's execution.
695-
696711
:param upsert: insert if document doesn't exist (default ``False``)
697-
:param full_response: return the entire response object from the
698-
server (default ``False``, not available for PyMongo 3+)
699712
:param remove: remove rather than updating (default ``False``)
700713
:param new: return updated rather than original document
701714
(default ``False``)
@@ -719,9 +732,6 @@ def modify(
719732
sort = queryset._ordering
720733

721734
try:
722-
if full_response:
723-
msg = "With PyMongo 3+, it is not possible anymore to get the full response."
724-
warnings.warn(msg, DeprecationWarning)
725735
if remove:
726736
result = queryset._collection.find_one_and_delete(
727737
query, sort=sort, session=_get_session(), **self._cursor_args
@@ -746,12 +756,8 @@ def modify(
746756
except pymongo.errors.OperationFailure as err:
747757
raise OperationError("Update failed (%s)" % err)
748758

749-
if full_response:
750-
if result["value"] is not None:
751-
result["value"] = self._document._from_son(result["value"])
752-
else:
753-
if result is not None:
754-
result = self._document._from_son(result)
759+
if result is not None:
760+
result = self._document._from_son(result)
755761

756762
return result
757763

@@ -1222,7 +1228,7 @@ def snapshot(self, enabled):
12221228
:param enabled: whether or not snapshot mode is enabled
12231229
"""
12241230
msg = "snapshot is deprecated as it has no impact when using PyMongo 3+."
1225-
warnings.warn(msg, DeprecationWarning)
1231+
warnings.warn(msg, DeprecationWarning, stacklevel=2)
12261232
queryset = self.clone()
12271233
queryset._snapshot = enabled
12281234
return queryset
@@ -1331,6 +1337,7 @@ def to_json(self, *args, **kwargs):
13311337
"representation to use. This will be changed to "
13321338
"uuid_representation=UNSPECIFIED in a future release.",
13331339
DeprecationWarning,
1340+
stacklevel=2,
13341341
)
13351342
kwargs["json_options"] = LEGACY_JSON_OPTIONS
13361343
return json_util.dumps(self.as_pymongo(), *args, **kwargs)
@@ -1340,7 +1347,7 @@ def from_json(self, json_data):
13401347
son_data = json_util.loads(json_data)
13411348
return [self._document._from_son(data) for data in son_data]
13421349

1343-
def aggregate(self, pipeline, *suppl_pipeline, **kwargs):
1350+
def aggregate(self, pipeline, **kwargs):
13441351
"""Perform an aggregate function based on your queryset params
13451352
13461353
If the queryset contains a query or skip/limit/sort or if the target Document class
@@ -1353,19 +1360,13 @@ def aggregate(self, pipeline, *suppl_pipeline, **kwargs):
13531360
13541361
:param pipeline: list of aggregation commands,
13551362
see: https://www.mongodb.com/docs/manual/core/aggregation-pipeline/
1356-
:param suppl_pipeline: unpacked list of pipeline (added to support deprecation of the old interface)
1357-
parameter will be removed shortly
13581363
:param kwargs: (optional) kwargs dictionary to be passed to pymongo's aggregate call
13591364
See https://pymongo.readthedocs.io/en/stable/api/pymongo/collection.html#pymongo.collection.Collection.aggregate
13601365
"""
1361-
using_deprecated_interface = isinstance(pipeline, dict) or bool(suppl_pipeline)
1362-
user_pipeline = [pipeline] if isinstance(pipeline, dict) else list(pipeline)
1363-
1364-
if using_deprecated_interface:
1365-
msg = "Calling .aggregate() with un unpacked list (*pipeline) is deprecated, it will soon change and will expect a list (similar to pymongo.Collection.aggregate interface), see documentation"
1366-
warnings.warn(msg, DeprecationWarning)
1367-
1368-
user_pipeline += suppl_pipeline
1366+
if not isinstance(pipeline, (tuple, list)):
1367+
raise TypeError(
1368+
f"Starting from 1.0 release pipeline must be a list/tuple, received: {type(pipeline)}"
1369+
)
13691370

13701371
initial_pipeline = []
13711372
if self._none or self._empty:
@@ -1391,7 +1392,7 @@ def aggregate(self, pipeline, *suppl_pipeline, **kwargs):
13911392
# geoNear and collStats must be the first stages in the pipeline if present
13921393
first_step = []
13931394
new_user_pipeline = []
1394-
for step_step in user_pipeline:
1395+
for step_step in pipeline:
13951396
if "$geoNear" in step_step:
13961397
first_step.append(step_step)
13971398
elif "$collStats" in step_step:
@@ -1407,8 +1408,18 @@ def aggregate(self, pipeline, *suppl_pipeline, **kwargs):
14071408
read_preference=self._read_preference, read_concern=self._read_concern
14081409
)
14091410

1411+
if self._hint not in (-1, None):
1412+
kwargs.setdefault("hint", self._hint)
1413+
if self._collation:
1414+
kwargs.setdefault("collation", self._collation)
1415+
if self._comment:
1416+
kwargs.setdefault("comment", self._comment)
1417+
14101418
return collection.aggregate(
1411-
final_pipeline, cursor={}, session=_get_session(), **kwargs
1419+
final_pipeline,
1420+
cursor={},
1421+
session=_get_session(),
1422+
**kwargs,
14121423
)
14131424

14141425
# JS functionality
@@ -1712,7 +1723,7 @@ def _cursor_args(self):
17121723
# TODO: evaluate similar possibilities using modifiers
17131724
if self._snapshot:
17141725
msg = "The snapshot option is not anymore available with PyMongo 3+"
1715-
warnings.warn(msg, DeprecationWarning)
1726+
warnings.warn(msg, DeprecationWarning, stacklevel=3)
17161727

17171728
cursor_args = {}
17181729
if not self._timeout:

‎tests/document/test_instance.py

Lines changed: 100 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,12 @@
4242
PickleSignalsTest,
4343
PickleTest,
4444
)
45-
from tests.utils import MongoDBTestCase, get_as_pymongo
45+
from tests.utils import (
46+
MongoDBTestCase,
47+
db_ops_tracker,
48+
get_as_pymongo,
49+
requires_mongodb_gte_44,
50+
)
4651

4752
TEST_IMAGE_PATH = os.path.join(os.path.dirname(__file__), "../fields/mongoengine.png")
4853

@@ -1871,6 +1876,53 @@ class User(self.Person):
18711876
person = self.Person.objects.get()
18721877
assert not person.comments_dict["first_post"].published
18731878

1879+
@requires_mongodb_gte_44
1880+
def test_update_propagates_hint_collation_and_comment(self):
1881+
"""Make sure adding a hint/comment/collation to the query gets added to the query"""
1882+
mongo_ver = get_mongodb_version()
1883+
1884+
base = {"locale": "en", "strength": 2}
1885+
index_name = "name_1"
1886+
1887+
class AggPerson(Document):
1888+
name = StringField()
1889+
meta = {
1890+
"indexes": [{"fields": ["name"], "name": index_name, "collation": base}]
1891+
}
1892+
1893+
AggPerson.drop_collection()
1894+
_ = AggPerson.objects.first()
1895+
1896+
comment = "test_comment"
1897+
1898+
if PYMONGO_VERSION >= (4, 1):
1899+
with db_ops_tracker() as q:
1900+
_ = AggPerson.objects.comment(comment).update_one(name="something")
1901+
query_op = q.db.system.profile.find(
1902+
{"ns": "mongoenginetest.agg_person"}
1903+
)[0]
1904+
CMD_QUERY_KEY = "command" if mongo_ver >= MONGODB_36 else "query"
1905+
assert "hint" not in query_op[CMD_QUERY_KEY]
1906+
assert query_op[CMD_QUERY_KEY]["comment"] == comment
1907+
assert "collation" not in query_op[CMD_QUERY_KEY]
1908+
1909+
with db_ops_tracker() as q:
1910+
_ = AggPerson.objects.hint(index_name).update_one(name="something")
1911+
query_op = q.db.system.profile.find({"ns": "mongoenginetest.agg_person"})[0]
1912+
CMD_QUERY_KEY = "command" if mongo_ver >= MONGODB_36 else "query"
1913+
1914+
assert query_op[CMD_QUERY_KEY]["hint"] == {"$hint": index_name}
1915+
assert "comment" not in query_op[CMD_QUERY_KEY]
1916+
assert "collation" not in query_op[CMD_QUERY_KEY]
1917+
1918+
with db_ops_tracker() as q:
1919+
_ = AggPerson.objects.collation(base).update_one(name="something")
1920+
query_op = q.db.system.profile.find({"ns": "mongoenginetest.agg_person"})[0]
1921+
CMD_QUERY_KEY = "command" if mongo_ver >= MONGODB_36 else "query"
1922+
assert "hint" not in query_op[CMD_QUERY_KEY]
1923+
assert "comment" not in query_op[CMD_QUERY_KEY]
1924+
assert query_op[CMD_QUERY_KEY]["collation"] == base
1925+
18741926
def test_delete(self):
18751927
"""Ensure that document may be deleted using the delete method."""
18761928
person = self.Person(name="Test User", age=30)
@@ -1879,6 +1931,53 @@ def test_delete(self):
18791931
person.delete()
18801932
assert self.Person.objects.count() == 0
18811933

1934+
@requires_mongodb_gte_44
1935+
def test_delete_propagates_hint_collation_and_comment(self):
1936+
"""Make sure adding a hint/comment/collation to the query gets added to the query"""
1937+
mongo_ver = get_mongodb_version()
1938+
1939+
base = {"locale": "en", "strength": 2}
1940+
index_name = "name_1"
1941+
1942+
class AggPerson(Document):
1943+
name = StringField()
1944+
meta = {
1945+
"indexes": [{"fields": ["name"], "name": index_name, "collation": base}]
1946+
}
1947+
1948+
AggPerson.drop_collection()
1949+
_ = AggPerson.objects.first()
1950+
1951+
comment = "test_comment"
1952+
1953+
if PYMONGO_VERSION >= (4, 1):
1954+
with db_ops_tracker() as q:
1955+
_ = AggPerson.objects().comment(comment).delete()
1956+
query_op = q.db.system.profile.find(
1957+
{"ns": "mongoenginetest.agg_person"}
1958+
)[0]
1959+
CMD_QUERY_KEY = "command" if mongo_ver >= MONGODB_36 else "query"
1960+
assert "hint" not in query_op[CMD_QUERY_KEY]
1961+
assert query_op[CMD_QUERY_KEY]["comment"] == comment
1962+
assert "collation" not in query_op[CMD_QUERY_KEY]
1963+
1964+
with db_ops_tracker() as q:
1965+
_ = AggPerson.objects.hint(index_name).delete()
1966+
query_op = q.db.system.profile.find({"ns": "mongoenginetest.agg_person"})[0]
1967+
CMD_QUERY_KEY = "command" if mongo_ver >= MONGODB_36 else "query"
1968+
1969+
assert query_op[CMD_QUERY_KEY]["hint"] == {"$hint": index_name}
1970+
assert "comment" not in query_op[CMD_QUERY_KEY]
1971+
assert "collation" not in query_op[CMD_QUERY_KEY]
1972+
1973+
with db_ops_tracker() as q:
1974+
_ = AggPerson.objects.collation(base).delete()
1975+
query_op = q.db.system.profile.find({"ns": "mongoenginetest.agg_person"})[0]
1976+
CMD_QUERY_KEY = "command" if mongo_ver >= MONGODB_36 else "query"
1977+
assert "hint" not in query_op[CMD_QUERY_KEY]
1978+
assert "comment" not in query_op[CMD_QUERY_KEY]
1979+
assert query_op[CMD_QUERY_KEY]["collation"] == base
1980+
18821981
def test_save_custom_id(self):
18831982
"""Ensure that a document may be saved with a custom _id."""
18841983

‎tests/queryset/test_modify.py

Lines changed: 0 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,5 @@
11
import unittest
22

3-
import pytest
4-
53
from mongoengine import (
64
Document,
75
IntField,
@@ -32,13 +30,6 @@ def test_modify(self):
3230
assert old_doc.to_json() == doc.to_json()
3331
self._assert_db_equal([{"_id": 0, "value": 0}, {"_id": 1, "value": -1}])
3432

35-
def test_modify_full_response_raise_value_error_for_recent_mongo(self):
36-
Doc(id=0, value=0).save()
37-
Doc(id=1, value=1).save()
38-
39-
with pytest.raises(ValueError):
40-
Doc.objects(id=1).modify(set__value=-1, full_response=True)
41-
4233
def test_modify_with_new(self):
4334
Doc(id=0, value=0).save()
4435
doc = Doc(id=1, value=1).save()

0 commit comments

Comments
 (0)