PHPLIB-1541: Include specs repository as a submodule (#1429)

* Add submodule for specifications

* Run spec tests from submodule

* Add dependabot configuration to update spec submodule

* Add submodule update to contributing docs

* Initialise spec submodule through composer

* Add guidance for fixing spec test failures in dependabot PRs

* PHPLIB-1458: Update aggregate-write-readPreference tests

* Update spec test paths

* Allow skipping duplicate tests in data providers

* Skip all clientBulkWrite tests

* PHPLIB-1450: Consolidate and improve skipping of tests

* Update to latest specs commit

* Use str_starts_with instead of regex logic

* Add annotations to type-qualify arrays

* Always require passing test group in provideTests

* Consistent wrapping

* PHPLIB-1492: Skip spec tests for sort in update operations

* Remove obsolete imports

Author: alcaeus

.github/dependabot.yml

@@ -4,3 +4,7 @@ updates:
     directory: "/"
     schedule:
       interval: "weekly"
+  - package-ecosystem: "gitsubmodule"
+    directory: "/"
+    schedule:
+      interval: "weekly"

.github/workflows/tests.yml

@@ -60,6 +60,7 @@ jobs:
         uses: "actions/checkout@v4"
         with:
           fetch-depth: 2
+          submodules: true
 
       - id: setup-mongodb
         uses: mongodb-labs/drivers-evergreen-tools@master

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "specifications"]
+	path = tests/specifications
+	url = https://github.com/mongodb/specifications

CONTRIBUTING.md

@@ -14,6 +14,7 @@ $ composer update
 In addition to installing project dependencies, Composer will check that the
 required extension version is installed. Directions for installing the extension
 may be found [here](https://php.net/manual/en/mongodb.installation.php).
+Composer will also install the submodule required for running spec tests.
 
 Installation directions for Composer may be found in its
 [Getting Started](https://getcomposer.org/doc/00-intro.md) guide.
@@ -100,6 +101,35 @@ The following environment variables are used for [CSFLE testing](https://github.
  * `KMS_TLS_CA_FILE`
  * `KMS_TLS_CERTIFICATE_KEY_FILE`
 
+### Updating spec tests
+
+Tests from the MongoDB Specifications repository are included through a
+submodule and updated automatically through Dependabot. To update tests
+manually, switch to the `tests/specifications` directory and update the
+repository to the appropriate commit. Remember to commit this change to the
+library repository.
+
+#### Handling test failures on updates
+
+Failures on updates can occur for multiple reasons, and the remedy to this will
+depend on the type of failure. Note that only tests for implemented
+specifications are run in the test runner.
+
+ * If a specification is not fully implemented (e.g. a recent change to the spec
+   has not been applied yet), skip the test in question with a reference to the
+   ticket that covers the change
+ * If a test fails because it uses features not yet implemented in the unified
+   test runner, skip the corresponding test with a reference to the ticket that
+   covers implementing the new features
+ * If the test failure points to a bug in the spec, consider the effort required
+   to fix the failure. If it's a small change, commit and push the fix directly
+   to the pull request. Otherwise, skip the test with a reference to a ticket to
+   fix the failing test.
+
+The goal is that the library passes tests with the latest spec version at all
+times, either by implementing small changes quickly, or by skipping tests as
+necessary.
+
 ## Code quality
 
 Before submitting a pull request, please ensure that your code adheres to the

composer.json

@@ -39,6 +39,8 @@
         }
     },
     "scripts": {
+        "pre-install-cmd": "git submodule update --init",
+        "pre-update-cmd": "git submodule update --init",
         "bench": "cd benchmark && composer update && vendor/bin/phpbench run --report=aggregate",
         "checks": [
             "@check:cs",

tests/SpecTests/ClientSideEncryption/Prose22_RangeExplicitEncryptionTest.php

@@ -38,6 +38,8 @@ class Prose22_RangeExplicitEncryptionTest extends FunctionalTestCase
     private ?Client $encryptedClient = null;
     private $key1Id;
 
+    private static string $specDir = __DIR__ . '/../../specifications/source/client-side-encryption';
+
     public function setUp(): void
     {
         parent::setUp();
@@ -50,7 +52,7 @@ public function setUp(): void
 
         $client = static::createTestClient();
 
-        $key1Document = $this->decodeJson(file_get_contents(__DIR__ . '/../client-side-encryption/etc/data/keys/key1-document.json'));
+        $key1Document = $this->decodeJson(file_get_contents(self::$specDir . '/etc/data/keys/key1-document.json'));
         $this->key1Id = $key1Document->_id;
 
         // Drop the key vault collection and insert key1Document with a majority write concern
@@ -85,7 +87,7 @@ public function setUpWithTypeAndRangeOpts(string $type, array $rangeOpts): void
          * for 64-bit integers. This means that DropEncryptedCollection and
          * CreateEncryptedCollection will be unable to inspect the option for
          * metadata collection names, but that's not necessary for the test. */
-        $encryptedFields = Document::fromJSON(file_get_contents(__DIR__ . '/../client-side-encryption/etc/data/range-encryptedFields-' . $type . '.json'));
+        $encryptedFields = Document::fromJSON(file_get_contents(self::$specDir . '/etc/data/range-encryptedFields-' . $type . '.json'));
 
         $database = $this->encryptedClient->selectDatabase($this->getDatabaseName());
         $database->dropCollection('explicit_encryption', ['encryptedFields' => $encryptedFields]);

tests/SpecTests/ClientSideEncryptionSpecTest.php

@@ -118,6 +118,8 @@ class ClientSideEncryptionSpecTest extends FunctionalTestCase
         'fle2v2-Compact: Compact works' => 'Failing due to bug in libmongocrypt (LIBMONGOCRYPT-699)',
     ];
 
+    private static string $specDir = __DIR__ . '/../specifications/source/client-side-encryption';
+
     public function setUp(): void
     {
         parent::setUp();
@@ -222,7 +224,7 @@ public static function provideTests()
     {
         $testArgs = [];
 
-        foreach (glob(__DIR__ . '/client-side-encryption/tests/*.json') as $filename) {
+        foreach (glob(self::$specDir . '/tests/legacy/*.json') as $filename) {
             $group = basename($filename, '.json');
 
             /* Some tests need to differentiate int32 and int64 BSON types.
@@ -421,7 +423,7 @@ public function testExternalKeyVault($withExternalKeyVault): void
         $client->selectCollection('db', 'coll')->drop();
 
         self::insertKeyVaultData($client, [
-            $this->decodeJson(file_get_contents(__DIR__ . '/client-side-encryption/external/external-key.json')),
+            $this->decodeJson(file_get_contents(self::$specDir . '/external/external-key.json')),
         ]);
 
         $encryptionOpts = [
@@ -437,7 +439,7 @@ public function testExternalKeyVault($withExternalKeyVault): void
 
         $autoEncryptionOpts = $encryptionOpts + [
             'schemaMap' => [
-                'db.coll' => $this->decodeJson(file_get_contents(__DIR__ . '/client-side-encryption/external/external-schema.json')),
+                'db.coll' => $this->decodeJson(file_get_contents(self::$specDir . '/external/external-schema.json')),
             ],
         ];
 
@@ -567,10 +569,10 @@ public function testBSONSizeLimitsAndBatchSplitting(Closure $test): void
         $client = static::createTestClient();
 
         $client->selectCollection('db', 'coll')->drop();
-        $client->selectDatabase('db')->createCollection('coll', ['validator' => ['$jsonSchema' => $this->decodeJson(file_get_contents(__DIR__ . '/client-side-encryption/limits/limits-schema.json'))]]);
+        $client->selectDatabase('db')->createCollection('coll', ['validator' => ['$jsonSchema' => $this->decodeJson(file_get_contents(self::$specDir . '/limits/limits-schema.json'))]]);
 
         self::insertKeyVaultData($client, [
-            $this->decodeJson(file_get_contents(__DIR__ . '/client-side-encryption/limits/limits-key.json')),
+            $this->decodeJson(file_get_contents(self::$specDir . '/limits/limits-key.json')),
         ]);
 
         $autoEncryptionOpts = [
@@ -585,7 +587,7 @@ public function testBSONSizeLimitsAndBatchSplitting(Closure $test): void
 
         $collection = $clientEncrypted->selectCollection('db', 'coll');
 
-        $document = json_decode(file_get_contents(__DIR__ . '/client-side-encryption/limits/limits-doc.json'), true, 512, JSON_THROW_ON_ERROR);
+        $document = json_decode(file_get_contents(self::$specDir . '/limits/limits-doc.json'), true, 512, JSON_THROW_ON_ERROR);
 
         $test($this, $collection, $document);
     }
@@ -634,18 +636,18 @@ public function testCorpus($schemaMap = true): void
         $client = static::createTestClient();
         $client->selectDatabase('db')->dropCollection('coll');
 
-        $schema = $this->decodeJson(file_get_contents(__DIR__ . '/client-side-encryption/corpus/corpus-schema.json'));
+        $schema = $this->decodeJson(file_get_contents(self::$specDir . '/corpus/corpus-schema.json'));
 
         if (! $schemaMap) {
             $client->selectDatabase('db')->createCollection('coll', ['validator' => ['$jsonSchema' => $schema]]);
         }
 
         self::insertKeyVaultData($client, [
-            $this->decodeJson(file_get_contents(__DIR__ . '/client-side-encryption/corpus/corpus-key-local.json')),
-            $this->decodeJson(file_get_contents(__DIR__ . '/client-side-encryption/corpus/corpus-key-aws.json')),
-            $this->decodeJson(file_get_contents(__DIR__ . '/client-side-encryption/corpus/corpus-key-azure.json')),
-            $this->decodeJson(file_get_contents(__DIR__ . '/client-side-encryption/corpus/corpus-key-gcp.json')),
-            $this->decodeJson(file_get_contents(__DIR__ . '/client-side-encryption/corpus/corpus-key-kmip.json')),
+            $this->decodeJson(file_get_contents(self::$specDir . '/corpus/corpus-key-local.json')),
+            $this->decodeJson(file_get_contents(self::$specDir . '/corpus/corpus-key-aws.json')),
+            $this->decodeJson(file_get_contents(self::$specDir . '/corpus/corpus-key-azure.json')),
+            $this->decodeJson(file_get_contents(self::$specDir . '/corpus/corpus-key-gcp.json')),
+            $this->decodeJson(file_get_contents(self::$specDir . '/corpus/corpus-key-kmip.json')),
         ]);
 
         $encryptionOpts = [
@@ -670,7 +672,7 @@ public function testCorpus($schemaMap = true): void
             ];
         }
 
-        $corpus = (array) $this->decodeJson(file_get_contents(__DIR__ . '/client-side-encryption/corpus/corpus.json'));
+        $corpus = (array) $this->decodeJson(file_get_contents(self::$specDir . '/corpus/corpus.json'));
         $corpusCopied = [];
 
         $clientEncrypted = static::createTestClient(null, [], ['autoEncryption' => $autoEncryptionOpts]);
@@ -701,7 +703,7 @@ public function testCorpus($schemaMap = true): void
 
         $this->assertDocumentsMatch($corpus, $corpusDecrypted);
 
-        $corpusEncryptedExpected = (array) $this->decodeJson(file_get_contents(__DIR__ . '/client-side-encryption/corpus/corpus-encrypted.json'));
+        $corpusEncryptedExpected = (array) $this->decodeJson(file_get_contents(self::$specDir . '/corpus/corpus-encrypted.json'));
         $corpusEncryptedActual = $client->selectCollection('db', 'coll')->findOne(['_id' => 'client_side_encryption_corpus'], ['typeMap' => ['root' => 'array', 'document' => stdClass::class, 'array' => 'array']]);
 
         foreach ($corpusEncryptedExpected as $fieldName => $expectedData) {
@@ -913,7 +915,7 @@ public function testBypassSpawningMongocryptdViaLoadingSharedLibrary(): void
                 'local' => ['key' => new Binary(base64_decode(self::LOCAL_MASTERKEY))],
             ],
             'schemaMap' => [
-                'db.coll' => $this->decodeJson(file_get_contents(__DIR__ . '/client-side-encryption/external/external-schema.json')),
+                'db.coll' => $this->decodeJson(file_get_contents(self::$specDir . '/external/external-schema.json')),
             ],
             'extraOptions' => [
                 'mongocryptdBypassSpawn' => true,
@@ -955,7 +957,7 @@ public function testBypassSpawningMongocryptdViaBypassSpawn(): void
                 'local' => ['key' => new Binary(base64_decode(self::LOCAL_MASTERKEY))],
             ],
             'schemaMap' => [
-                'db.coll' => $this->decodeJson(file_get_contents(__DIR__ . '/client-side-encryption/external/external-schema.json')),
+                'db.coll' => $this->decodeJson(file_get_contents(self::$specDir . '/external/external-schema.json')),
             ],
             'extraOptions' => [
                 'mongocryptdBypassSpawn' => true,
@@ -1335,8 +1337,8 @@ public function testExplicitEncryption(Closure $test): void
         $this->skipIfServerVersion('<', '7.0.0', 'Explicit encryption tests require MongoDB 7.0 or later');
 
         // Test setup
-        $encryptedFields = $this->decodeJson(file_get_contents(__DIR__ . '/client-side-encryption/etc/data/encryptedFields.json'));
-        $key1Document = $this->decodeJson(file_get_contents(__DIR__ . '/client-side-encryption/etc/data/keys/key1-document.json'));
+        $encryptedFields = $this->decodeJson(file_get_contents(self::$specDir . '/etc/data/encryptedFields.json'));
+        $key1Document = $this->decodeJson(file_get_contents(self::$specDir . '/etc/data/keys/key1-document.json'));
         $key1Id = $key1Document->_id;
 
         $client = static::createTestClient();