Merge pull request #356 from magento/develop

MFTF 2.4.1 - Merge develop to master

Author: KevinBKozan

.github/CONTRIBUTING.md

@@ -40,7 +40,7 @@ All contributors are required to submit a click-through form to agree to the ter
 
    - Unit/integration test coverage
    - Proposed documentation update.
-     For the documentation contribution guidelines, see [DevDocs Contributing][devdocs].
+     For the documentation contribution guidelines, see [DOCUMENTATION_TEMPLATE][].
 7. For large features or changes, [open an issue][issue] to discuss first.
    This may prevent duplicate or unnecessary effort, and it may gain you some additional contributors.
 8. To report a bug, [open an issue][issue], and follow [guidelines about bugfix issues][issue reporting].
@@ -160,7 +160,7 @@ Label| Description
 [create issue]: https://help.github.com/articles/creating-an-issue/
 [create pr]: https://help.github.com/articles/creating-a-pull-request/
 [Definition of Done]: https://devdocs.magento.com/guides/v2.2/contributor-guide/contributing_dod.html
-[devdocs]: https://github.com/magento/devdocs/blob/master/.github/CONTRIBUTING.html
+[DOCUMENTATION_TEMPLATE]: https://github.com/magento/devdocs/blob/master/.github/DOCUMENTATION_TEMPLATE.md
 [existing issues]: https://github.com/magento/magento2-functional-testing-framework/issues?q=is%3Aopen+is%3Aissue
 [existing PRs]: https://github.com/magento/magento2-functional-testing-framework/pulls?q=is%3Aopen+is%3Apr
 [GitHub documentation]: https://help.github.com/articles/syncing-a-fork
@@ -171,4 +171,4 @@ Label| Description
 [Magento Contributor Agreement]: http://www.magento.com/legaldocuments/mca
 [MFTF repository]: https://github.com/magento/magento2-functional-testing-framework
 [open new issue]: https://github.com/magento/magento2-functional-testing-framework/issues/new
-[Travis CI]: https://travis-ci.com/magento/magento2-functional-testing-framework/pull_requests
+[Travis CI]: https://travis-ci.com/magento/magento2-functional-testing-framework/pull_requests

.github/DOCUMENTATION_TEMPLATE.md

@@ -0,0 +1,26 @@
+# How to contribute to MFTF docs
+
+We welcome contributions to the MFTF documentation, which is kept within the `docs/` folder in this repository.
+This page describes the submitting process and serves as a template for a properly written content.
+The contribution workflow is the same as submitting code.
+
+1. Create a branch from the `develop` branch in the [MFTF repository][].
+1. Make edits/additions/deletions as needed. Read the [Basic Template][] for tips on how to write with Markdown.
+1. Submit your pull request to the `develop` branch.
+
+Once submitted, a member of the documentation team will review and merge it.
+We will inform you if it needs any additional processing.
+
+The documentation in this repository is used as the source for the [MFTF documentation][].
+Any changes to the table of contents will need to be made through a separate pull request in the regular [Magento Developer documentation repository][].
+
+Read more about how to [Contribute to Magento Devdocs][].
+
+<!-- For readability, we abstract the link URLS to the bottom of the page. The extra set of square brackets denotes it is a link, rather than plain brackets. >
+
+<!-- Link Definitions -->
+[Magento Developer documentation repository]: https://github.com/magento/devdocs/blob/master/_data/toc/mftf.yml
+[MFTF repository]: https://github.com/magento/magento2-functional-testing-framework
+[Contribute to Magento Devdocs]: https://github.com/magento/devdocs/blob/master/.github/CONTRIBUTING.md
+[MFTF documentation]: https://devdocs.magento.com/mftf/docs/introduction.html
+[Basic Template]: https://devdocs.magento.com/guides/v2.3/contributor-guide/templates/basic_template.html

CHANGELOG.md

@@ -1,6 +1,16 @@
 Magento Functional Testing Framework Changelog
 ================================================
 
+2.4.1
+-----
+* Traceability
+    * XSD Schema validation is now enabled by default in `generate:tests`, `run:test`, `run:failed`, `run:group`
+    * `--debug` option for the above commands has been updated to include different debug levels
+        * See DevDocs for details
+
+### Fixes
+* Fixed an issue where `skipReadiness` attribute would cause false XSD Schema validation errors.
+
 2.4.0
 -----
 ### Enhancements

bin/mftf

@@ -29,7 +29,7 @@ try {
 try {
     $application = new Symfony\Component\Console\Application();
     $application->setName('Magento Functional Testing Framework CLI');
-    $application->setVersion('2.4.0');
+    $application->setVersion('2.4.1');
     /** @var \Magento\FunctionalTestingFramework\Console\CommandListInterface $commandList */
     $commandList = new \Magento\FunctionalTestingFramework\Console\CommandList;
     foreach ($commandList->getCommands() as $command) {

composer.json

@@ -2,7 +2,7 @@
     "name": "magento/magento2-functional-testing-framework",
     "description": "Magento2 Functional Testing Framework",
     "type": "library",
-    "version": "2.4.0",
+    "version": "2.4.1",
     "license": "AGPL-3.0",
     "keywords": ["magento", "automation", "functional", "testing"],
     "config": {

composer.lock

@@ -32,7 +32,7 @@
     true,
     \Magento\FunctionalTestingFramework\Config\MftfApplicationConfig::UNIT_TEST_PHASE,
     true,
-    false
+    \Magento\FunctionalTestingFramework\Config\MftfApplicationConfig::LEVEL_NONE
 );
 
 // Load needed framework env params

dev/tests/_bootstrap.php

@@ -60,9 +60,7 @@ class BasicFunctionalTestCest
 	{
 		$I->comment("");
 		$I->comment("");
-		$I->skipReadinessCheck(true);
-		$I->comment("skipReadiness");
-		$I->skipReadinessCheck(false);
+		$I->comment("seeComment");
 		$someVarDefinition = $I->grabValueFrom();
 		$I->acceptPopup();
 		$I->amOnPage("/test/url");

dev/tests/verification/Resources/BasicFunctionalTest.txt

@@ -115,10 +115,6 @@
         <see selector=".selector" userInput="{{sameStepKeyAsArg}}" stepKey="arg1" />
     </actionGroup>
 
-    <actionGroup name="actionGroupWithSkipReadinessActions">
-        <comment userInput="ActionGroupSkipReadiness" stepKey="skip" skipReadiness="true"/>
-    </actionGroup>
-
     <actionGroup name="actionGroupWithSectionAndData">
         <arguments>
             <argument name="content" type="string"/>

dev/tests/verification/TestModule/ActionGroup/BasicActionGroup.xml

@@ -24,7 +24,7 @@
         </after>
         <comment stepKey="basicCommentWithNoData" userInput="{{emptyData.noData}}"/>
         <comment stepKey="basicCommentWithDefinitelyNoData" userInput="{{emptyData.definitelyNoData}}"/>
-        <comment stepKey="ReadinessCheckSkipped" userInput="skipReadiness" skipReadiness="true"/>
+        <comment stepKey="basicCommentWithData" userInput="seeComment"/>
         <grabValueFrom stepKey="someVarDefinition"/>
         <acceptPopup stepKey="acceptPopupKey1"/>
         <amOnPage stepKey="amOnPageKey1" url="/test/url"/>

dev/tests/verification/TestModule/Test/BasicFunctionalTest.xml

@@ -185,17 +185,6 @@ public function testActionGroupWithArgContainingStepKey()
         $this->generateAndCompareTest('ActionGroupContainsStepKeyInArgText');
     }
 
-    /**
-     * Test an action group with an arg containing stepKey text
-     *
-     * @throws \Exception
-     * @throws \Magento\FunctionalTestingFramework\Exceptions\TestReferenceException
-     */
-    public function testActionGroupWithSkipReadiness()
-    {
-        $this->generateAndCompareTest('ActionGroupSkipReadiness');
-    }
-
     /**
      * Test an action group with an arg containing stepKey text
      *

dev/tests/verification/Tests/ActionGroupGenerationTest.php

@@ -19,7 +19,7 @@ class SchemaValidationTest extends MftfTestCase
      */
     public function testInvalidTestSchema()
     {
-        AspectMock::double(MftfApplicationConfig::class, ['debugEnabled' => true]);
+        AspectMock::double(MftfApplicationConfig::class, ['getDebugLevel' => MftfApplicationConfig::LEVEL_DEVELOPER]);
         $testFile = ['testFile.xml' => "<tests><test name='testName'><annotations>a</annotations></test></tests>"];
         $expectedError = TESTS_MODULE_PATH .
             DIRECTORY_SEPARATOR .

dev/tests/verification/Tests/SchemaValidationTest.php

@@ -105,7 +105,7 @@ vendor/bin/mftf build:project --MAGENTO_BASE_URL=http://magento.local/ --MAGENTO
 
 #### Description
 
-Generate PHP code from the tests defined in XML files.
+Perform XML schema validation and generate PHP code from the tests defined in XML files.
 The path is set in the `TESTS_MODULE_PATH` [configuration] parameter.
 
 #### Usage
@@ -122,7 +122,7 @@ vendor/bin/mftf generate:tests [option] [<test name>] [<test name>] [--remove]
 | `--force` | Forces test generation, regardless of the module merge order defined in the Magento instance. Example: `generate:tests --force`. |
 | `-i,--time` | Set time in minutes to determine the group size when `--config=parallel` is used. The __default value__ is `10`. Example: `generate:tests --config=parallel --time=15`|
 | `--tests` | Defines the test configuration as a JSON string.|
-| `--debug` | Returns additional debug information (such as the filename where an error occurred) when test generation fails because of an invalid XML schema. This parameter takes extra processing time. Use it after test generation has failed once. |
+| `--debug or --debug=[<none>]`| Performs schema validations on XML files. <br/> DEFAULT: `generate:tests` implicitly performs schema validation on merged files. It does not indicate the file name where the error is encountered. <br/> DEVELOPER: `--debug` performs per-file validation and returns additional debug information (such as the filename where an error occurred) when test generation fails because of an invalid XML schema. This option takes extra processing time. Use it after test generation has failed once.</br><br/> NONE: `--debug=none` skips debugging during test generation. Added for backward compatibility, it will be removed in the next MAJOR release.</br>|
 | `-r,--remove`| Removes the existing generated suites and tests cleaning up the `_generated` directory before the actual run. For example, `generate:tests SampleTest --remove` cleans up the entire `_generated` directory and generates `SampleTest` only.|
 
 #### Examples of the JSON configuration
@@ -295,6 +295,7 @@ vendor/bin/mftf run:group [--skip-generate|--remove] [--] <group1> [<group2>]
 | --------------------- | --------------------------------------------------------------------------------------------------------- |
 | `-k, --skip-generate` | Skips generating from the source XML. Instead, the command executes previously-generated groups of tests. |
 | `-r, --remove`        | Removes previously generated suites and tests before the actual generation and run.                       |
+| `--debug or --debug=[<none>]`| Performs schema validations on XML files. `run:group` implicitly performs schema validation on merged files. It does not indicate the file name where the error is encountered. `--debug` performs per-file validation and returns additional debug information (such as the filename where an error occurred). `--debug=none` skips debugging during test run. Added for backward compatibility, it will be removed in the next MAJOR release.|
 
 #### Examples
 
@@ -326,6 +327,7 @@ vendor/bin/mftf run:test [--skip-generate|--remove] [--] <name1> [<name2>]
 |-----------------------|-----------------------------------------------------------------------------------------------------------|
 | `-k, --skip-generate` | Skips generating from the source XML. Instead, the command executes previously-generated groups of tests. |
 | `-r, --remove`        | Remove previously generated suites and tests.                                                             |
+| `--debug or --debug=[<none>]`| Performs schema validations on XML files. `run:test` implicitly performs schema validation on merged files. It does not indicate the file name where the error is encountered. `--debug` performs per-file validation and returns additional debug information (such as the filename where an error occurred). `--debug=none` skips debugging during test run. Added for backward compatibility, it will be removed in the next MAJOR release.
 
 #### Examples
 
@@ -347,6 +349,11 @@ For more details about `failed`, refer to [Reporting][].
 ```bash
 vendor/bin/mftf run:failed
 ```
+#### Options
+
+| Option                | Description                                                                                               |
+|-----------------------|-----------------------------------------------------------------------------------------------------------|
+| `--debug or --debug=[<none>]`| Performs schema validations on XML files. `run:failed` implicitly performs schema validation on merged files. It does not indicate the file name where the error is encountered. `--debug` performs per-file validation and returns additional debug information (such as the filename where an error occurred). Use it after test run has failed once. `--debug=none` skips debugging during test run. Added for backward compatibility, it will be removed in the next MAJOR release.|
 
 #### Examples
 

docs/commands/mftf.md

@@ -30,7 +30,6 @@ X.Y.Z
 | | +-- Backward Compatible changes (Patch release - bug fixes, small additions)
 | +---- Backward Compatible changes (Minor release - small new features, bug fixes)
 +------ Backward Incompatible changes (Major release - new features and/or major changes)
-
 ```
 
 For example:
@@ -56,3 +55,14 @@ It MAY include patch level changes. Patch version MUST be reset to 0 when minor
 Major version **X** MUST be incremented for a release that introduces backward incompatible changes.
 A major release can also include minor and patch level changes.
 You must reset the patch and minor version to 0 when you change the major version.
+
+## Magento 2  compatibility
+
+This table lists the version of the MFTF that was released with a particular version of Magento.
+
+|Magento version| MFTF  version|
+|---|---|
+| 2.3.1 | 2.3.13 |
+| 2.3.0 | 2.3.9 |
+| 2.2.8 | 2.3.13 |
+| 2.2.7 | 2.3.8 |

docs/versioning.md

@@ -14,6 +14,14 @@ class MftfApplicationConfig
     const UNIT_TEST_PHASE = "testing";
     const MFTF_PHASES = [self::GENERATION_PHASE, self::EXECUTION_PHASE, self::UNIT_TEST_PHASE];
 
+    /**
+     * Mftf debug levels
+     */
+    const LEVEL_DEFAULT = "default";
+    const LEVEL_DEVELOPER = "developer";
+    const LEVEL_NONE = "none";
+    const MFTF_DEBUG_LEVEL = [self::LEVEL_DEFAULT, self::LEVEL_DEVELOPER, self::LEVEL_NONE];
+
     /**
      * Determines whether the user has specified a force option for generation
      *
@@ -36,11 +44,11 @@ class MftfApplicationConfig
     private $verboseEnabled;
 
     /**
-     * Determines whether the user would like to execute mftf in a verbose run.
+     * String which identifies the current debug level of mftf execution
      *
-     * @var boolean
+     * @var string
      */
-    private $debugEnabled;
+    private $debugLevel;
 
     /**
      * MftfApplicationConfig Singelton Instance
@@ -55,14 +63,14 @@ class MftfApplicationConfig
      * @param boolean $forceGenerate
      * @param string  $phase
      * @param boolean $verboseEnabled
-     * @param boolean $debugEnabled
+     * @param string  $debugLevel
      * @throws TestFrameworkException
      */
     private function __construct(
         $forceGenerate = false,
         $phase = self::EXECUTION_PHASE,
         $verboseEnabled = null,
-        $debugEnabled = null
+        $debugLevel = self::LEVEL_NONE
     ) {
         $this->forceGenerate = $forceGenerate;
 
@@ -72,7 +80,15 @@ private function __construct(
 
         $this->phase = $phase;
         $this->verboseEnabled = $verboseEnabled;
-        $this->debugEnabled = $debugEnabled;
+        switch ($debugLevel) {
+            case self::LEVEL_DEVELOPER:
+            case self::LEVEL_DEFAULT:
+            case self::LEVEL_NONE:
+                $this->debugLevel = $debugLevel;
+                break;
+            default:
+                $this->debugLevel = self::LEVEL_DEVELOPER;
+        }
     }
 
     /**
@@ -82,14 +98,14 @@ private function __construct(
      * @param boolean $forceGenerate
      * @param string  $phase
      * @param boolean $verboseEnabled
-     * @param boolean $debugEnabled
+     * @param string  $debugLevel
      * @return void
      */
-    public static function create($forceGenerate, $phase, $verboseEnabled, $debugEnabled)
+    public static function create($forceGenerate, $phase, $verboseEnabled, $debugLevel)
     {
         if (self::$MFTF_APPLICATION_CONTEXT == null) {
             self::$MFTF_APPLICATION_CONTEXT =
-                new MftfApplicationConfig($forceGenerate, $phase, $verboseEnabled, $debugEnabled);
+                new MftfApplicationConfig($forceGenerate, $phase, $verboseEnabled, $debugLevel);
         }
     }
 
@@ -132,14 +148,13 @@ public function verboseEnabled()
     }
 
     /**
-     * Returns a boolean indicating whether the user has indicated a debug run, which will lengthy validation
-     * with some extra error messaging to be run
+     * Returns a string which indicates the debug level of mftf execution.
      *
-     * @return boolean
+     * @return string
      */
-    public function debugEnabled()
+    public function getDebugLevel()
     {
-        return $this->debugEnabled ?? getenv('MFTF_DEBUG');
+        return $this->debugLevel ?? getenv('MFTF_DEBUG');
     }
 
     /**

src/Magento/FunctionalTestingFramework/Config/MftfApplicationConfig.php

@@ -157,7 +157,7 @@ protected function readFiles($fileList)
                 } else {
                     $configMerger->merge($content);
                 }
-                if (MftfApplicationConfig::getConfig()->debugEnabled()) {
+                if (MftfApplicationConfig::getConfig()->getDebugLevel() === MftfApplicationConfig::LEVEL_DEVELOPER) {
                     $this->validateSchema($configMerger, $fileList->getFilename());
                 }
             } catch (\Magento\FunctionalTestingFramework\Config\Dom\ValidationException $e) {
@@ -231,8 +231,14 @@ protected function validateSchema($configMerger, $filename = null)
         if ($this->validationState->isValidationRequired()) {
             $errors = [];
             if ($configMerger && !$configMerger->validate($this->schemaFile, $errors)) {
-                $message = $filename ? $filename . PHP_EOL . "Invalid Document \n" : PHP_EOL . "Invalid Document \n";
-                throw new \Exception($message . implode("\n", $errors));
+                foreach ($errors as $error) {
+                    $error = str_replace(PHP_EOL, "", $error);
+                    LoggingUtil::getInstance()->getLogger(Filesystem::class)->criticalFailure(
+                        "Schema validation error ",
+                        ($filename ? [ "file"=> $filename, "error" => $error]: ["error" => $error])
+                    );
+                }
+                throw new \Exception("Schema validation errors found in xml file(s)" . $filename);
             }
         }
     }