Transaction support (doctrine#2586)

* Run CI workflows on feature branches

* Refactor commit logic (doctrine#2580)

* Add tests for commit consistency showing wrong behaviour

* Clear scheduled document changes at the end of a commit operation

* Rename private variables to communicate intent

* Remove obsolete comment

* Use different error code

* Use single mongos in consistency tests

This ensures that the fail points are created on the same server that the write operations take place on, which can't be guaranteed in a sharded cluster with multiple mongoses.

* Rename test methods for clarity

* Explain reasoning for index error

* Extract helper method to create failpoint

* Add configuration setting for transactional flush (doctrine#2587)

* Add configuration setting for transactional flush

* Use classic setters for transactional flush setting

* Add logic for transactional commits (doctrine#2589)

* Extract commit logic

* Support transactional commit operations

* Always use transactional flush if supported

With this commit, all tests using the document manager use transactional flush as long as transactions are supported. Certain tests can use the static $allowsTransactions variable to disable this behaviour.

* Test with MongoDB 7.0

* Update test names

* Update phpstan baseline

* Fix query selection in shard key tests

* Flip transaction options constant by default

* Use supportsTransaction method when skipping tests

* Apply review feedback to tests

* Add separate test to check write concern in commit options

* Strip write options when in transaction

* Use majority write concern in test

* Update events to play nice with transactions (doctrine#2594)

* Pass session and transaction information to event args

* Only dispatch lifecycle events once per commit operation

* Remove isInTransaction property in event args

* Split method signature for readability

* Use property promotion for event args classes

* Extract construction of eventArgs

* Inline spl_object_hash calls

* Avoid injecting test instance

* Add session to commitOptions in persister

* Add session assertions in LifecycleEventManager

* Only retry transaction once (doctrine#2604)

* Only retry transaction once

* Rename variable

* Update transaction documentation (doctrine#2606)

* Remove references to transactions where not applicable

* Update transaction documentation

* Apply suggestions from code review

Co-authored-by: Jeremy Mikola <[email protected]>

* Change title level in event documentation

* Add documentation about transactions to event docs

---------

Co-authored-by: Jeremy Mikola <[email protected]>

* Address code review items

* Test highest dependencies on MongoDB 7.0

* Regenerate psalm baseline to silence unrelated errors

---------

Co-authored-by: Jeremy Mikola <[email protected]>

.github/workflows/coding-standards.yml

@@ -5,6 +5,7 @@ on:
   pull_request:
     branches:
       - "*.x"
+      - "feature/*"
   push:
 
 jobs:

.github/workflows/continuous-integration.yml

@@ -4,6 +4,7 @@ on:
   pull_request:
     branches:
       - "*.x"
+      - "feature/*"
   push:
 
 env:
@@ -21,6 +22,7 @@ jobs:
           - "8.2"
           - "8.3"
         mongodb-version:
+          - "7.0"
           - "6.0"
           - "5.0"
           - "4.4"
@@ -33,24 +35,34 @@ jobs:
         symfony-version:
           - "stable"
         include:
+          # Test against lowest dependencies
           - dependencies: "lowest"
             php-version: "8.1"
             mongodb-version: "4.4"
             driver-version: "1.11.0"
             topology: "server"
             symfony-version: "stable"
-          - topology: "sharded_cluster"
+          # Test with highest dependencies
+          - topology: "server"
+            php-version: "8.2"
+            mongodb-version: "7.0"
+            driver-version: "stable"
+            dependencies: "highest"
+            symfony-version: "7"
+          # Test with a 4.4 replica set
+          - topology: "replica_set"
             php-version: "8.2"
             mongodb-version: "4.4"
             driver-version: "stable"
             dependencies: "highest"
             symfony-version: "stable"
-          - topology: "server"
+          # Test with a 4.4 sharded cluster
+          - topology: "sharded_cluster"
             php-version: "8.2"
-            mongodb-version: "6.0"
+            mongodb-version: "4.4"
             driver-version: "stable"
             dependencies: "highest"
-            symfony-version: "7"
+            symfony-version: "stable"
 
     steps:
       - name: "Checkout"

.github/workflows/performance.yml

@@ -4,6 +4,7 @@ on:
   pull_request:
     branches:
       - "*.x"
+      - "feature/*"
   push:
 
 jobs:

.github/workflows/static-analysis.yml

@@ -5,6 +5,7 @@ on:
   pull_request:
     branches:
       - "*.x"
+      - "feature/*"
   push:
 
 jobs:

docs/en/cookbook/validation-of-documents.rst

@@ -86,7 +86,7 @@ Now validation is performed whenever you call
 ``DocumentManager#persist($order)`` or when you call
 ``DocumentManager#flush()`` and an order is about to be updated. Any
 Exception that happens in the lifecycle callbacks will be cached by
-the DocumentManager and the current transaction is rolled back.
+the DocumentManager.
 
 Of course you can do any type of primitive checks, not null,
 email-validation, string size, integer and date ranges in your

docs/en/reference/architecture.rst

@@ -56,7 +56,7 @@ A document instance can be characterized as being NEW, MANAGED, DETACHED or REMO
    DocumentManager and a UnitOfWork.
 -  A REMOVED document instance is an instance with a persistent
    identity, associated with a DocumentManager, that will be removed
-   from the database upon transaction commit.
+   from the database upon UnitOfWork commit.
 
 Persistent fields
 ~~~~~~~~~~~~~~~~~
@@ -103,7 +103,7 @@ persistent objects.
 Transactional write-behind
 ~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-An ``DocumentManager`` and the underlying ``UnitOfWork`` employ a
+The ``DocumentManager`` and the underlying ``UnitOfWork`` employ a
 strategy called "transactional write-behind" that delays the
 execution of query statements in order to execute them in the most
 efficient way and to execute them at the end of a transaction so

docs/en/reference/events.rst

@@ -42,7 +42,7 @@ Now we can add some event listeners to the ``$evm``. Let's create a
             $evm->addEventListener([self::preFoo, self::postFoo], $this);
         }
 
-        public function preFoo(EventArgs $e): void 
+        public function preFoo(EventArgs $e): void
         {
             $this->preFooInvoked = true;
         }
@@ -345,6 +345,38 @@ follow this restrictions very carefully since operations in the
 wrong event may produce lots of different errors, such as
 inconsistent data and lost updates/persists/removes.
 
+Handling Transactional Flushes
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+When a flush operation is executed in a transaction, all queries inside a lifecycle event listener also have to make use
+of the session used during the flush operation. This session object is exposed through the ``LifecycleEventArgs``
+parameter passed to the listener. Passing the session to queries ensures that the query will become part of the
+transaction and will see data that has not been committed yet.
+
+.. code-block:: php
+
+    <?php
+
+    public function someEventListener(\Doctrine\ODM\MongoDB\Event\LifecycleEventArgs $eventArgs): void
+    {
+        // To check if a transaction is active:
+        if ($eventArgs->isInTransaction()) {
+            // Do something
+        }
+
+        // Pass the session to any query you execute
+        $eventArgs->getDocumentManager()->createQueryBuilder(User::class)
+            // Query logic
+            ->getQuery(['session' => $eventArgs->session])
+            ->execute();
+    }
+
+.. note::
+
+    Event listeners are only called during the first transaction attempt. If the transaction is retried, event listeners
+    will not be invoked again. Make sure to run any persistence logic through the UnitOfWork instead of modifying data
+    directly through queries run in an event listener.
+
 prePersist
 ~~~~~~~~~~
 
@@ -693,8 +725,8 @@ Define the ``EventTest`` class with a ``postCollectionLoad()`` method:
         }
     }
 
-Load ClassMetadata Event
-------------------------
+loadClassMetadata
+~~~~~~~~~~~~~~~~~
 
 When the mapping information for a document is read, it is
 populated in to a ``ClassMetadata`` instance. You can hook in to

docs/en/reference/transactions-and-concurrency.rst

@@ -9,26 +9,78 @@ Transactions
 As per the `documentation <https://docs.mongodb.com/manual/core/write-operations-atomicity/#atomicity-and-transactions>`_, MongoDB
 write operations are "atomic on the level of a single document".
 
-Even when updating multiple documents within a single write operation,
-though the modification of each document is atomic,
-the operation as a whole is not and other operations may interleave.
+Even when updating multiple documents within a single write operation, though the modification of each document is
+atomic, the operation as a whole is not and other operations may interleave.
 
-As stated in the `FAQ <https://docs.mongodb.com/manual/faq/fundamentals/#does-mongodb-support-transactions>`_,
-"MongoDB does not support multi-document transactions" and neither does Doctrine MongoDB ODM.
+Transaction support
+~~~~~~~~~~~~~~~~~~~
+
+MongoDB supports multi-document transactions on replica sets (starting in MongoDB 4.2) and sharded clusters (MongoDB
+4.4). Standalone topologies do not support multi-document transactions.
+
+Transaction Support in Doctrine MongoDB ODM
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. note::
+    Transaction support in MongoDB ODM was introduced in version 2.7.
+
+You can instruct the ODM to use transactions when writing changes to the databases by enabling the
+``useTransactionalFlush`` setting in your configuration:
+
+.. code-block:: php
+
+    $config = new Configuration();
+    $config->setUseTransactionalFlush(true);
+    // Other configuration
+
+    $dm = DocumentManager::create(null, $config);
+
+From then onwards, any call to ``DocumentManager::flush`` will start a transaction, apply the write operations, then
+commit the transaction.
+
+To enable or disable transaction usage for a single flush operation, use the ``withTransaction`` write option when
+calling ``DocumentManager::flush``:
+
+.. code-block:: php
+
+    // To explicitly enable transaction for this write
+    $dm->flush(['withTransaction' => true]);
+
+    // To disable transaction usage for a write, regardless of the ``useTransactionalFlush`` config:
+    $dm->flush(['withTransaction' => false]);
+
+.. note::
+
+    Please note that transactions are only used for write operations executed during the ``flush`` operation. For any
+    other operations, e.g. manually executed queries or aggregation pipelines, transactions will not be used and you
+    will have to rely on the MongoDB driver's transaction mechanism.
+
+Lifecycle Events and Transactions
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+When using transactional flushes, either through the configuration or explicitly, there are a couple of important things
+to note regarding lifecycle events. Due to the way MongoDB transactions work, it is possible that ODM attempts write
+operations multiple times. However, to preserve the expectation that lifecycle events are only triggered once per flush
+operation, lifecycle events will not be dispatched when the transaction is retried. This maintains current functionality
+when a lifecycle event modifies the unit of work, as this change is automatically carried over when the transaction is
+retried.
 
-Limitation
-~~~~~~~~~~
-At the moment, Doctrine MongoDB ODM does not provide any native strategy to emulate multi-document transactions.
+Lifecycle events now expose a ``MongoDB\Driver\Session`` object which needs to be used if it is set. Since MongoDB
+transactions are not tied to the connection but only to a session, any command that should be part of the transaction
+needs to be told about the session to be used. This does not only apply to write commands, but also to read commands
+that need to see the transaction state. If a session is given in a lifecycle event, this session should always be used
+regardless of whether a transaction is active or not.
 
-Workaround
-~~~~~~~~~~
-To work around this limitation, one can utilize `two phase commits <https://docs.mongodb.com/manual/tutorial/perform-two-phase-commits/>`_.
 
-Concurrency
------------
+Other Concurrency Controls
+--------------------------
 
-Doctrine MongoDB ODM offers native support for pessimistic and optimistic locking strategies.
-This allows for very fine-grained control over what kind of locking is required for documents in your application.
+Multi-Document transactions provide certain guarantees regarding your database writes and prevent two simultaneous write
+operations from interfering with each other. Depending on your use case, this is not enough, as the transactional
+guarantee will only apply once you start writing to the database as part of the ``DocumentManager::flush()`` call. This
+could still lead to data loss if you replace data that was written to the database by a different process in between you
+reading the data and starting the transaction. To solve this problem, optimistic and pessimistic locking strategies can
+be used, allowing for fine-grained control over what kind of locking is required for documents in your application.
 
 .. _transactions_and_concurrency_optimistic_locking:
 

lib/Doctrine/ODM/MongoDB/Configuration.php

@@ -117,6 +117,8 @@ class Configuration
 
     private int $autoGenerateProxyClasses = self::AUTOGENERATE_EVAL;
 
+    private bool $useTransactionalFlush = false;
+
     public function __construct()
     {
         $this->proxyManagerConfiguration = new ProxyManagerConfiguration();
@@ -596,6 +598,16 @@ public function getProxyManagerConfiguration(): ProxyManagerConfiguration
     {
         return $this->proxyManagerConfiguration;
     }
+
+    public function setUseTransactionalFlush(bool $useTransactionalFlush): void
+    {
+        $this->useTransactionalFlush = $useTransactionalFlush;
+    }
+
+    public function isTransactionalFlushEnabled(): bool
+    {
+        return $this->useTransactionalFlush;
+    }
 }
 
 interface_exists(MappingDriver::class);

lib/Doctrine/ODM/MongoDB/Event/LifecycleEventArgs.php

@@ -6,6 +6,8 @@
 
 use Doctrine\ODM\MongoDB\DocumentManager;
 use Doctrine\Persistence\Event\LifecycleEventArgs as BaseLifecycleEventArgs;
+use Doctrine\Persistence\ObjectManager;
+use MongoDB\Driver\Session;
 
 /**
  * Lifecycle Events are triggered by the UnitOfWork during lifecycle transitions
@@ -15,6 +17,14 @@
  */
 class LifecycleEventArgs extends BaseLifecycleEventArgs
 {
+    public function __construct(
+        object $object,
+        ObjectManager $objectManager,
+        public readonly ?Session $session = null,
+    ) {
+        parent::__construct($object, $objectManager);
+    }
+
     public function getDocument(): object
     {
         return $this->getObject();
@@ -24,4 +34,9 @@ public function getDocumentManager(): DocumentManager
     {
         return $this->getObjectManager();
     }
+
+    public function isInTransaction(): bool
+    {
+        return $this->session?->isInTransaction() ?? false;
+    }
 }

lib/Doctrine/ODM/MongoDB/Event/PreLoadEventArgs.php

@@ -5,21 +5,21 @@
 namespace Doctrine\ODM\MongoDB\Event;
 
 use Doctrine\ODM\MongoDB\DocumentManager;
+use MongoDB\Driver\Session;
 
 /**
  * Class that holds event arguments for a preLoad event.
  */
 final class PreLoadEventArgs extends LifecycleEventArgs
 {
-    /** @var array<string, mixed> */
-    private array $data;
-
     /** @param array<string, mixed> $data */
-    public function __construct(object $document, DocumentManager $dm, array &$data)
-    {
-        parent::__construct($document, $dm);
-
-        $this->data =& $data;
+    public function __construct(
+        object $document,
+        DocumentManager $dm,
+        private array &$data,
+        ?Session $session = null,
+    ) {
+        parent::__construct($document, $dm, $session);
     }
 
     /**