diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md index 61805aafe..271b66244 100644 --- a/.github/CONTRIBUTING.md +++ b/.github/CONTRIBUTING.md @@ -40,7 +40,7 @@ All contributors are required to submit a click-through form to agree to the ter - Unit/integration test coverage - Proposed documentation update. - For the documentation contribution guidelines, see [DevDocs Contributing][devdocs]. + For the documentation contribution guidelines, see [DOCUMENTATION_TEMPLATE][]. 7. For large features or changes, [open an issue][issue] to discuss first. This may prevent duplicate or unnecessary effort, and it may gain you some additional contributors. 8. To report a bug, [open an issue][issue], and follow [guidelines about bugfix issues][issue reporting]. @@ -160,7 +160,7 @@ Label| Description [create issue]: https://help.github.com/articles/creating-an-issue/ [create pr]: https://help.github.com/articles/creating-a-pull-request/ [Definition of Done]: https://devdocs.magento.com/guides/v2.2/contributor-guide/contributing_dod.html -[devdocs]: https://github.com/magento/devdocs/blob/master/.github/CONTRIBUTING.html +[DOCUMENTATION_TEMPLATE]: https://github.com/magento/devdocs/blob/master/.github/DOCUMENTATION_TEMPLATE.md [existing issues]: https://github.com/magento/magento2-functional-testing-framework/issues?q=is%3Aopen+is%3Aissue [existing PRs]: https://github.com/magento/magento2-functional-testing-framework/pulls?q=is%3Aopen+is%3Apr [GitHub documentation]: https://help.github.com/articles/syncing-a-fork @@ -171,4 +171,4 @@ Label| Description [Magento Contributor Agreement]: http://www.magento.com/legaldocuments/mca [MFTF repository]: https://github.com/magento/magento2-functional-testing-framework [open new issue]: https://github.com/magento/magento2-functional-testing-framework/issues/new -[Travis CI]: https://travis-ci.com/magento/magento2-functional-testing-framework/pull_requests \ No newline at end of file +[Travis CI]: https://travis-ci.com/magento/magento2-functional-testing-framework/pull_requests diff --git a/.github/DOCUMENTATION_TEMPLATE.md b/.github/DOCUMENTATION_TEMPLATE.md new file mode 100644 index 000000000..222ff23be --- /dev/null +++ b/.github/DOCUMENTATION_TEMPLATE.md @@ -0,0 +1,26 @@ +# How to contribute to MFTF docs + +We welcome contributions to the MFTF documentation, which is kept within the `docs/` folder in this repository. +This page describes the submitting process and serves as a template for a properly written content. +The contribution workflow is the same as submitting code. + +1. Create a branch from the `develop` branch in the [MFTF repository][]. +1. Make edits/additions/deletions as needed. Read the [Basic Template][] for tips on how to write with Markdown. +1. Submit your pull request to the `develop` branch. + +Once submitted, a member of the documentation team will review and merge it. +We will inform you if it needs any additional processing. + +The documentation in this repository is used as the source for the [MFTF documentation][]. +Any changes to the table of contents will need to be made through a separate pull request in the regular [Magento Developer documentation repository][]. + +Read more about how to [Contribute to Magento Devdocs][]. + + +[Magento Developer documentation repository]: https://github.com/magento/devdocs/blob/master/_data/toc/mftf.yml +[MFTF repository]: https://github.com/magento/magento2-functional-testing-framework +[Contribute to Magento Devdocs]: https://github.com/magento/devdocs/blob/master/.github/CONTRIBUTING.md +[MFTF documentation]: https://devdocs.magento.com/mftf/docs/introduction.html +[Basic Template]: https://devdocs.magento.com/guides/v2.3/contributor-guide/templates/basic_template.html diff --git a/CHANGELOG.md b/CHANGELOG.md index afcb80adc..ee0d665e5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,16 @@ Magento Functional Testing Framework Changelog ================================================ +2.4.1 +----- +* Traceability + * XSD Schema validation is now enabled by default in `generate:tests`, `run:test`, `run:failed`, `run:group` + * `--debug` option for the above commands has been updated to include different debug levels + * See DevDocs for details + +### Fixes +* Fixed an issue where `skipReadiness` attribute would cause false XSD Schema validation errors. + 2.4.0 ----- ### Enhancements diff --git a/bin/mftf b/bin/mftf index bebc55b94..207dab6ff 100755 --- a/bin/mftf +++ b/bin/mftf @@ -29,7 +29,7 @@ try { try { $application = new Symfony\Component\Console\Application(); $application->setName('Magento Functional Testing Framework CLI'); - $application->setVersion('2.4.0'); + $application->setVersion('2.4.1'); /** @var \Magento\FunctionalTestingFramework\Console\CommandListInterface $commandList */ $commandList = new \Magento\FunctionalTestingFramework\Console\CommandList; foreach ($commandList->getCommands() as $command) { diff --git a/composer.json b/composer.json index 21a2670c0..d226371ea 100755 --- a/composer.json +++ b/composer.json @@ -2,7 +2,7 @@ "name": "magento/magento2-functional-testing-framework", "description": "Magento2 Functional Testing Framework", "type": "library", - "version": "2.4.0", + "version": "2.4.1", "license": "AGPL-3.0", "keywords": ["magento", "automation", "functional", "testing"], "config": { diff --git a/composer.lock b/composer.lock index 2e539fcf2..c858ceff4 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "4ae7e6856deb8d93cae8f3d7c2fbb3c0", + "content-hash": "6b2b6a074f053fe28df2633e8bce095f", "packages": [ { "name": "allure-framework/allure-codeception", diff --git a/dev/tests/_bootstrap.php b/dev/tests/_bootstrap.php index 31b4462f6..d9a4208b2 100644 --- a/dev/tests/_bootstrap.php +++ b/dev/tests/_bootstrap.php @@ -32,7 +32,7 @@ true, \Magento\FunctionalTestingFramework\Config\MftfApplicationConfig::UNIT_TEST_PHASE, true, - false + \Magento\FunctionalTestingFramework\Config\MftfApplicationConfig::LEVEL_NONE ); // Load needed framework env params diff --git a/dev/tests/verification/Resources/BasicFunctionalTest.txt b/dev/tests/verification/Resources/BasicFunctionalTest.txt index 9970e81ec..fd5f0f165 100644 --- a/dev/tests/verification/Resources/BasicFunctionalTest.txt +++ b/dev/tests/verification/Resources/BasicFunctionalTest.txt @@ -60,9 +60,7 @@ class BasicFunctionalTestCest { $I->comment(""); $I->comment(""); - $I->skipReadinessCheck(true); - $I->comment("skipReadiness"); - $I->skipReadinessCheck(false); + $I->comment("seeComment"); $someVarDefinition = $I->grabValueFrom(); $I->acceptPopup(); $I->amOnPage("/test/url"); diff --git a/dev/tests/verification/TestModule/ActionGroup/BasicActionGroup.xml b/dev/tests/verification/TestModule/ActionGroup/BasicActionGroup.xml index 245447f7d..45ccdbd24 100644 --- a/dev/tests/verification/TestModule/ActionGroup/BasicActionGroup.xml +++ b/dev/tests/verification/TestModule/ActionGroup/BasicActionGroup.xml @@ -115,10 +115,6 @@ - - - - diff --git a/dev/tests/verification/TestModule/Test/BasicFunctionalTest.xml b/dev/tests/verification/TestModule/Test/BasicFunctionalTest.xml index d120e5c31..bdeb4e0c6 100644 --- a/dev/tests/verification/TestModule/Test/BasicFunctionalTest.xml +++ b/dev/tests/verification/TestModule/Test/BasicFunctionalTest.xml @@ -24,7 +24,7 @@ - + diff --git a/dev/tests/verification/Tests/ActionGroupGenerationTest.php b/dev/tests/verification/Tests/ActionGroupGenerationTest.php index dec8c6d14..613d6cb72 100644 --- a/dev/tests/verification/Tests/ActionGroupGenerationTest.php +++ b/dev/tests/verification/Tests/ActionGroupGenerationTest.php @@ -185,17 +185,6 @@ public function testActionGroupWithArgContainingStepKey() $this->generateAndCompareTest('ActionGroupContainsStepKeyInArgText'); } - /** - * Test an action group with an arg containing stepKey text - * - * @throws \Exception - * @throws \Magento\FunctionalTestingFramework\Exceptions\TestReferenceException - */ - public function testActionGroupWithSkipReadiness() - { - $this->generateAndCompareTest('ActionGroupSkipReadiness'); - } - /** * Test an action group with an arg containing stepKey text * diff --git a/dev/tests/verification/Tests/SchemaValidationTest.php b/dev/tests/verification/Tests/SchemaValidationTest.php index 86828e9a5..92442bd37 100644 --- a/dev/tests/verification/Tests/SchemaValidationTest.php +++ b/dev/tests/verification/Tests/SchemaValidationTest.php @@ -19,7 +19,7 @@ class SchemaValidationTest extends MftfTestCase */ public function testInvalidTestSchema() { - AspectMock::double(MftfApplicationConfig::class, ['debugEnabled' => true]); + AspectMock::double(MftfApplicationConfig::class, ['getDebugLevel' => MftfApplicationConfig::LEVEL_DEVELOPER]); $testFile = ['testFile.xml' => "a"]; $expectedError = TESTS_MODULE_PATH . DIRECTORY_SEPARATOR . diff --git a/docs/commands/mftf.md b/docs/commands/mftf.md index 4d91bb37f..2301d6f44 100644 --- a/docs/commands/mftf.md +++ b/docs/commands/mftf.md @@ -105,7 +105,7 @@ vendor/bin/mftf build:project --MAGENTO_BASE_URL=http://magento.local/ --MAGENTO #### Description -Generate PHP code from the tests defined in XML files. +Perform XML schema validation and generate PHP code from the tests defined in XML files. The path is set in the `TESTS_MODULE_PATH` [configuration] parameter. #### Usage @@ -122,7 +122,7 @@ vendor/bin/mftf generate:tests [option] [] [] [--remove] | `--force` | Forces test generation, regardless of the module merge order defined in the Magento instance. Example: `generate:tests --force`. | | `-i,--time` | Set time in minutes to determine the group size when `--config=parallel` is used. The __default value__ is `10`. Example: `generate:tests --config=parallel --time=15`| | `--tests` | Defines the test configuration as a JSON string.| -| `--debug` | Returns additional debug information (such as the filename where an error occurred) when test generation fails because of an invalid XML schema. This parameter takes extra processing time. Use it after test generation has failed once. | +| `--debug or --debug=[]`| Performs schema validations on XML files.
DEFAULT: `generate:tests` implicitly performs schema validation on merged files. It does not indicate the file name where the error is encountered.
DEVELOPER: `--debug` performs per-file validation and returns additional debug information (such as the filename where an error occurred) when test generation fails because of an invalid XML schema. This option takes extra processing time. Use it after test generation has failed once.

NONE: `--debug=none` skips debugging during test generation. Added for backward compatibility, it will be removed in the next MAJOR release.
| | `-r,--remove`| Removes the existing generated suites and tests cleaning up the `_generated` directory before the actual run. For example, `generate:tests SampleTest --remove` cleans up the entire `_generated` directory and generates `SampleTest` only.| #### Examples of the JSON configuration @@ -295,6 +295,7 @@ vendor/bin/mftf run:group [--skip-generate|--remove] [--] [] | --------------------- | --------------------------------------------------------------------------------------------------------- | | `-k, --skip-generate` | Skips generating from the source XML. Instead, the command executes previously-generated groups of tests. | | `-r, --remove` | Removes previously generated suites and tests before the actual generation and run. | +| `--debug or --debug=[]`| Performs schema validations on XML files. `run:group` implicitly performs schema validation on merged files. It does not indicate the file name where the error is encountered. `--debug` performs per-file validation and returns additional debug information (such as the filename where an error occurred). `--debug=none` skips debugging during test run. Added for backward compatibility, it will be removed in the next MAJOR release.| #### Examples @@ -326,6 +327,7 @@ vendor/bin/mftf run:test [--skip-generate|--remove] [--] [] |-----------------------|-----------------------------------------------------------------------------------------------------------| | `-k, --skip-generate` | Skips generating from the source XML. Instead, the command executes previously-generated groups of tests. | | `-r, --remove` | Remove previously generated suites and tests. | +| `--debug or --debug=[]`| Performs schema validations on XML files. `run:test` implicitly performs schema validation on merged files. It does not indicate the file name where the error is encountered. `--debug` performs per-file validation and returns additional debug information (such as the filename where an error occurred). `--debug=none` skips debugging during test run. Added for backward compatibility, it will be removed in the next MAJOR release. #### Examples @@ -347,6 +349,11 @@ For more details about `failed`, refer to [Reporting][]. ```bash vendor/bin/mftf run:failed ``` +#### Options + +| Option | Description | +|-----------------------|-----------------------------------------------------------------------------------------------------------| +| `--debug or --debug=[]`| Performs schema validations on XML files. `run:failed` implicitly performs schema validation on merged files. It does not indicate the file name where the error is encountered. `--debug` performs per-file validation and returns additional debug information (such as the filename where an error occurred). Use it after test run has failed once. `--debug=none` skips debugging during test run. Added for backward compatibility, it will be removed in the next MAJOR release.| #### Examples diff --git a/docs/versioning.md b/docs/versioning.md index 026cab8f8..660b88a40 100644 --- a/docs/versioning.md +++ b/docs/versioning.md @@ -30,7 +30,6 @@ X.Y.Z | | +-- Backward Compatible changes (Patch release - bug fixes, small additions) | +---- Backward Compatible changes (Minor release - small new features, bug fixes) +------ Backward Incompatible changes (Major release - new features and/or major changes) - ``` For example: @@ -56,3 +55,14 @@ It MAY include patch level changes. Patch version MUST be reset to 0 when minor Major version **X** MUST be incremented for a release that introduces backward incompatible changes. A major release can also include minor and patch level changes. You must reset the patch and minor version to 0 when you change the major version. + +## Magento 2 compatibility + +This table lists the version of the MFTF that was released with a particular version of Magento. + +|Magento version| MFTF version| +|---|---| +| 2.3.1 | 2.3.13 | +| 2.3.0 | 2.3.9 | +| 2.2.8 | 2.3.13 | +| 2.2.7 | 2.3.8 | diff --git a/src/Magento/FunctionalTestingFramework/Config/MftfApplicationConfig.php b/src/Magento/FunctionalTestingFramework/Config/MftfApplicationConfig.php index eeabdf117..29278f761 100644 --- a/src/Magento/FunctionalTestingFramework/Config/MftfApplicationConfig.php +++ b/src/Magento/FunctionalTestingFramework/Config/MftfApplicationConfig.php @@ -14,6 +14,14 @@ class MftfApplicationConfig const UNIT_TEST_PHASE = "testing"; const MFTF_PHASES = [self::GENERATION_PHASE, self::EXECUTION_PHASE, self::UNIT_TEST_PHASE]; + /** + * Mftf debug levels + */ + const LEVEL_DEFAULT = "default"; + const LEVEL_DEVELOPER = "developer"; + const LEVEL_NONE = "none"; + const MFTF_DEBUG_LEVEL = [self::LEVEL_DEFAULT, self::LEVEL_DEVELOPER, self::LEVEL_NONE]; + /** * Determines whether the user has specified a force option for generation * @@ -36,11 +44,11 @@ class MftfApplicationConfig private $verboseEnabled; /** - * Determines whether the user would like to execute mftf in a verbose run. + * String which identifies the current debug level of mftf execution * - * @var boolean + * @var string */ - private $debugEnabled; + private $debugLevel; /** * MftfApplicationConfig Singelton Instance @@ -55,14 +63,14 @@ class MftfApplicationConfig * @param boolean $forceGenerate * @param string $phase * @param boolean $verboseEnabled - * @param boolean $debugEnabled + * @param string $debugLevel * @throws TestFrameworkException */ private function __construct( $forceGenerate = false, $phase = self::EXECUTION_PHASE, $verboseEnabled = null, - $debugEnabled = null + $debugLevel = self::LEVEL_NONE ) { $this->forceGenerate = $forceGenerate; @@ -72,7 +80,15 @@ private function __construct( $this->phase = $phase; $this->verboseEnabled = $verboseEnabled; - $this->debugEnabled = $debugEnabled; + switch ($debugLevel) { + case self::LEVEL_DEVELOPER: + case self::LEVEL_DEFAULT: + case self::LEVEL_NONE: + $this->debugLevel = $debugLevel; + break; + default: + $this->debugLevel = self::LEVEL_DEVELOPER; + } } /** @@ -82,14 +98,14 @@ private function __construct( * @param boolean $forceGenerate * @param string $phase * @param boolean $verboseEnabled - * @param boolean $debugEnabled + * @param string $debugLevel * @return void */ - public static function create($forceGenerate, $phase, $verboseEnabled, $debugEnabled) + public static function create($forceGenerate, $phase, $verboseEnabled, $debugLevel) { if (self::$MFTF_APPLICATION_CONTEXT == null) { self::$MFTF_APPLICATION_CONTEXT = - new MftfApplicationConfig($forceGenerate, $phase, $verboseEnabled, $debugEnabled); + new MftfApplicationConfig($forceGenerate, $phase, $verboseEnabled, $debugLevel); } } @@ -132,14 +148,13 @@ public function verboseEnabled() } /** - * Returns a boolean indicating whether the user has indicated a debug run, which will lengthy validation - * with some extra error messaging to be run + * Returns a string which indicates the debug level of mftf execution. * - * @return boolean + * @return string */ - public function debugEnabled() + public function getDebugLevel() { - return $this->debugEnabled ?? getenv('MFTF_DEBUG'); + return $this->debugLevel ?? getenv('MFTF_DEBUG'); } /** diff --git a/src/Magento/FunctionalTestingFramework/Config/Reader/Filesystem.php b/src/Magento/FunctionalTestingFramework/Config/Reader/Filesystem.php index 4c3aa581b..2f2cb8ad3 100644 --- a/src/Magento/FunctionalTestingFramework/Config/Reader/Filesystem.php +++ b/src/Magento/FunctionalTestingFramework/Config/Reader/Filesystem.php @@ -157,7 +157,7 @@ protected function readFiles($fileList) } else { $configMerger->merge($content); } - if (MftfApplicationConfig::getConfig()->debugEnabled()) { + if (MftfApplicationConfig::getConfig()->getDebugLevel() === MftfApplicationConfig::LEVEL_DEVELOPER) { $this->validateSchema($configMerger, $fileList->getFilename()); } } catch (\Magento\FunctionalTestingFramework\Config\Dom\ValidationException $e) { @@ -231,8 +231,14 @@ protected function validateSchema($configMerger, $filename = null) if ($this->validationState->isValidationRequired()) { $errors = []; if ($configMerger && !$configMerger->validate($this->schemaFile, $errors)) { - $message = $filename ? $filename . PHP_EOL . "Invalid Document \n" : PHP_EOL . "Invalid Document \n"; - throw new \Exception($message . implode("\n", $errors)); + foreach ($errors as $error) { + $error = str_replace(PHP_EOL, "", $error); + LoggingUtil::getInstance()->getLogger(Filesystem::class)->criticalFailure( + "Schema validation error ", + ($filename ? [ "file"=> $filename, "error" => $error]: ["error" => $error]) + ); + } + throw new \Exception("Schema validation errors found in xml file(s)" . $filename); } } } diff --git a/src/Magento/FunctionalTestingFramework/Config/Reader/MftfFilesystem.php b/src/Magento/FunctionalTestingFramework/Config/Reader/MftfFilesystem.php index 728c4cb3a..cd075e57c 100644 --- a/src/Magento/FunctionalTestingFramework/Config/Reader/MftfFilesystem.php +++ b/src/Magento/FunctionalTestingFramework/Config/Reader/MftfFilesystem.php @@ -24,6 +24,7 @@ public function readFiles($fileList) $exceptionCollector = new ExceptionCollector(); /** @var \Magento\FunctionalTestingFramework\Test\Config\Dom $configMerger */ $configMerger = null; + $debugLevel = MftfApplicationConfig::getConfig()->getDebugLevel(); foreach ($fileList as $key => $content) { //check if file is empty and continue to next if it is if (!parent::verifyFileEmpty($content, $fileList->getFilename())) { @@ -40,7 +41,8 @@ public function readFiles($fileList) } else { $configMerger->merge($content, $fileList->getFilename(), $exceptionCollector); } - if (MftfApplicationConfig::getConfig()->debugEnabled()) { + // run per file validation with generate:tests -d + if ($debugLevel === MftfApplicationConfig::LEVEL_DEVELOPER) { $this->validateSchema($configMerger, $fileList->getFilename()); } } catch (\Magento\FunctionalTestingFramework\Config\Dom\ValidationException $e) { @@ -48,8 +50,10 @@ public function readFiles($fileList) } } $exceptionCollector->throwException(); - if ($fileList->valid()) { - $this->validateSchema($configMerger, $fileList->getFilename()); + + //run validation on merged file with generate:tests + if ($debugLevel === MftfApplicationConfig::LEVEL_DEFAULT) { + $this->validateSchema($configMerger); } $output = []; diff --git a/src/Magento/FunctionalTestingFramework/Console/GenerateDocsCommand.php b/src/Magento/FunctionalTestingFramework/Console/GenerateDocsCommand.php index db77bc74f..4ed2b6b29 100644 --- a/src/Magento/FunctionalTestingFramework/Console/GenerateDocsCommand.php +++ b/src/Magento/FunctionalTestingFramework/Console/GenerateDocsCommand.php @@ -67,7 +67,7 @@ protected function execute(InputInterface $input, OutputInterface $output) $force, MftfApplicationConfig::GENERATION_PHASE, false, - false + MftfApplicationConfig::LEVEL_NONE ); $allActionGroups = ActionGroupObjectHandler::getInstance()->getAllObjects(); diff --git a/src/Magento/FunctionalTestingFramework/Console/GenerateTestsCommand.php b/src/Magento/FunctionalTestingFramework/Console/GenerateTestsCommand.php index fcd977060..23651b492 100644 --- a/src/Magento/FunctionalTestingFramework/Console/GenerateTestsCommand.php +++ b/src/Magento/FunctionalTestingFramework/Console/GenerateTestsCommand.php @@ -29,7 +29,7 @@ class GenerateTestsCommand extends BaseGenerateCommand protected function configure() { $this->setName('generate:tests') - ->setDescription('This command generates all test files and suites based on xml declarations') + ->setDescription('Run validation and generate all test files and suites based on xml declarations') ->addArgument( 'name', InputArgument::OPTIONAL | InputArgument::IS_ARRAY, @@ -39,7 +39,7 @@ protected function configure() "force", 'f', InputOption::VALUE_NONE, - 'force generation of tests regardless of Magento Instance Configuration' + 'Force generation of tests regardless of Magento Instance Configuration' )->addOption( 'time', 'i', @@ -54,8 +54,10 @@ protected function configure() )->addOption( 'debug', 'd', - InputOption::VALUE_NONE, - 'run extra validation when generating tests' + InputOption::VALUE_OPTIONAL, + 'Run extra validation when generating tests. Use option \'none\' to turn off debugging -- + added for backward compatibility, will be removed in the next MAJOR release', + MftfApplicationConfig::LEVEL_DEFAULT ); parent::configure(); @@ -78,9 +80,8 @@ protected function execute(InputInterface $input, OutputInterface $output) $json = $input->getOption('tests'); $force = $input->getOption('force'); $time = $input->getOption('time') * 60 * 1000; // convert from minutes to milliseconds - $debug = $input->getOption('debug'); + $debug = $input->getOption('debug') ?? MftfApplicationConfig::LEVEL_DEVELOPER; // for backward compatibility $remove = $input->getOption('remove'); - $verbose = $output->isVerbose(); if ($json !== null && !json_decode($json)) { @@ -95,7 +96,8 @@ protected function execute(InputInterface $input, OutputInterface $output) // Remove previous GENERATED_DIR if --remove option is used if ($remove) { - $this->removeGeneratedDirectory($output, $verbose || $debug); + $this->removeGeneratedDirectory($output, $verbose || + ($debug !== MftfApplicationConfig::LEVEL_NONE)); } $testConfiguration = $this->createTestConfiguration($json, $tests, $force, $debug, $verbose); @@ -124,13 +126,13 @@ protected function execute(InputInterface $input, OutputInterface $output) * @param string $json * @param array $tests * @param boolean $force - * @param boolean $debug + * @param string $debug * @param boolean $verbose * @return array * @throws \Magento\FunctionalTestingFramework\Exceptions\TestReferenceException * @throws \Magento\FunctionalTestingFramework\Exceptions\XmlException */ - private function createTestConfiguration($json, array $tests, bool $force, bool $debug, bool $verbose) + private function createTestConfiguration($json, array $tests, bool $force, $debug, bool $verbose) { // set our application configuration so we can references the user options in our framework MftfApplicationConfig::create( diff --git a/src/Magento/FunctionalTestingFramework/Console/RunTestCommand.php b/src/Magento/FunctionalTestingFramework/Console/RunTestCommand.php index 33a29b2e8..76a8bc8f4 100644 --- a/src/Magento/FunctionalTestingFramework/Console/RunTestCommand.php +++ b/src/Magento/FunctionalTestingFramework/Console/RunTestCommand.php @@ -7,6 +7,7 @@ namespace Magento\FunctionalTestingFramework\Console; +use Magento\FunctionalTestingFramework\Config\MftfApplicationConfig; use Symfony\Component\Console\Input\ArrayInput; use Symfony\Component\Console\Input\InputArgument; use Symfony\Component\Console\Input\InputInterface; @@ -36,6 +37,13 @@ protected function configure() 'f', InputOption::VALUE_NONE, 'force generation of tests regardless of Magento Instance Configuration' + )->addOption( + 'debug', + 'd', + InputOption::VALUE_OPTIONAL, + 'Run extra validation when running tests. Use option \'none\' to turn off debugging -- + added for backward compatibility, will be removed in the next MAJOR release', + MftfApplicationConfig::LEVEL_DEFAULT ); parent::configure(); @@ -57,6 +65,7 @@ protected function execute(InputInterface $input, OutputInterface $output): int $skipGeneration = $input->getOption('skip-generate'); $force = $input->getOption('force'); $remove = $input->getOption('remove'); + $debug = $input->getOption('debug') ?? MftfApplicationConfig::LEVEL_DEVELOPER; // for backward compatibility if ($skipGeneration and $remove) { // "skip-generate" and "remove" options cannot be used at the same time @@ -73,7 +82,8 @@ protected function execute(InputInterface $input, OutputInterface $output): int 'suites' => null ]), '--force' => $force, - '--remove' => $remove + '--remove' => $remove, + '--debug' => $debug ]; $command->run(new ArrayInput($args), $output); } diff --git a/src/Magento/FunctionalTestingFramework/Console/RunTestFailedCommand.php b/src/Magento/FunctionalTestingFramework/Console/RunTestFailedCommand.php index 8257162b9..d6a580d52 100644 --- a/src/Magento/FunctionalTestingFramework/Console/RunTestFailedCommand.php +++ b/src/Magento/FunctionalTestingFramework/Console/RunTestFailedCommand.php @@ -13,6 +13,7 @@ use Symfony\Component\Console\Output\OutputInterface; use Symfony\Component\Process\Process; use Magento\FunctionalTestingFramework\Exceptions\TestFrameworkException; +use Symfony\Component\Console\Input\InputOption; class RunTestFailedCommand extends BaseGenerateCommand { @@ -49,7 +50,15 @@ class RunTestFailedCommand extends BaseGenerateCommand protected function configure() { $this->setName('run:failed') - ->setDescription('Execute a set of tests referenced via failed file'); + ->setDescription('Execute a set of tests referenced via failed file') + ->addOption( + 'debug', + 'd', + InputOption::VALUE_OPTIONAL, + 'Run extra validation when running failed tests. Use option \'none\' to turn off debugging -- + added for backward compatibility, will be removed in the next MAJOR release', + MftfApplicationConfig::LEVEL_DEFAULT + ); parent::configure(); } @@ -67,12 +76,13 @@ protected function configure() */ protected function execute(InputInterface $input, OutputInterface $output): int { + $debug = $input->getOption('debug') ?? MftfApplicationConfig::LEVEL_DEVELOPER; // for backward compatibility // Create Mftf Configuration MftfApplicationConfig::create( false, MftfApplicationConfig::GENERATION_PHASE, false, - false + $debug ); $testConfiguration = $this->getFailedTestList(); @@ -83,7 +93,11 @@ protected function execute(InputInterface $input, OutputInterface $output): int } $command = $this->getApplication()->find('generate:tests'); - $args = ['--tests' => $testConfiguration, '--remove' => true]; + $args = [ + '--tests' => $testConfiguration, + '--remove' => true, + '--debug' => $debug + ]; $command->run(new ArrayInput($args), $output); diff --git a/src/Magento/FunctionalTestingFramework/Console/RunTestGroupCommand.php b/src/Magento/FunctionalTestingFramework/Console/RunTestGroupCommand.php index e895fb66a..d714a1dd1 100644 --- a/src/Magento/FunctionalTestingFramework/Console/RunTestGroupCommand.php +++ b/src/Magento/FunctionalTestingFramework/Console/RunTestGroupCommand.php @@ -39,6 +39,13 @@ protected function configure() 'f', InputOption::VALUE_NONE, 'force generation of tests regardless of Magento Instance Configuration' + )->addOption( + 'debug', + 'd', + InputOption::VALUE_OPTIONAL, + 'Run extra validation when running tests. Use option \'none\' to turn off debugging -- + added for backward compatibility, will be removed in the next MAJOR release', + MftfApplicationConfig::LEVEL_DEFAULT )->addArgument( 'groups', InputArgument::IS_ARRAY | InputArgument::REQUIRED, @@ -64,6 +71,7 @@ protected function execute(InputInterface $input, OutputInterface $output): int $force = $input->getOption('force'); $groups = $input->getArgument('groups'); $remove = $input->getOption('remove'); + $debug = $input->getOption('debug') ?? MftfApplicationConfig::LEVEL_DEVELOPER; // for backward compatibility if ($skipGeneration and $remove) { // "skip-generate" and "remove" options cannot be used at the same time @@ -77,7 +85,7 @@ protected function execute(InputInterface $input, OutputInterface $output): int $force, MftfApplicationConfig::GENERATION_PHASE, false, - false + $debug ); if (!$skipGeneration) { @@ -86,7 +94,8 @@ protected function execute(InputInterface $input, OutputInterface $output): int $args = [ '--tests' => $testConfiguration, '--force' => $force, - '--remove' => $remove + '--remove' => $remove, + '--debug' => $debug ]; $command->run(new ArrayInput($args), $output); diff --git a/src/Magento/FunctionalTestingFramework/StaticCheck/TestDependencyCheck.php b/src/Magento/FunctionalTestingFramework/StaticCheck/TestDependencyCheck.php index e0b3cefd6..1a149a657 100644 --- a/src/Magento/FunctionalTestingFramework/StaticCheck/TestDependencyCheck.php +++ b/src/Magento/FunctionalTestingFramework/StaticCheck/TestDependencyCheck.php @@ -75,7 +75,7 @@ public function execute(InputInterface $input) true, MftfApplicationConfig::UNIT_TEST_PHASE, false, - false + MftfApplicationConfig::LEVEL_NONE ); ModuleResolver::getInstance()->getModulesPath(); diff --git a/src/Magento/FunctionalTestingFramework/Test/etc/Actions/commonAttributes.xsd b/src/Magento/FunctionalTestingFramework/Test/etc/Actions/commonAttributes.xsd index 22c05ea7d..c39d779ed 100644 --- a/src/Magento/FunctionalTestingFramework/Test/etc/Actions/commonAttributes.xsd +++ b/src/Magento/FunctionalTestingFramework/Test/etc/Actions/commonAttributes.xsd @@ -31,13 +31,6 @@ - - - - Flag for skipping readiness check - - - diff --git a/src/Magento/FunctionalTestingFramework/Test/etc/mergedTestSchema.xsd b/src/Magento/FunctionalTestingFramework/Test/etc/mergedTestSchema.xsd index b663bb99d..fe45945ee 100644 --- a/src/Magento/FunctionalTestingFramework/Test/etc/mergedTestSchema.xsd +++ b/src/Magento/FunctionalTestingFramework/Test/etc/mergedTestSchema.xsd @@ -129,7 +129,13 @@ + + + + Flag for skipping readiness check. + + + - diff --git a/src/Magento/FunctionalTestingFramework/Util/Logger/MftfLogger.php b/src/Magento/FunctionalTestingFramework/Util/Logger/MftfLogger.php index 5844858a0..0a52f6b6b 100644 --- a/src/Magento/FunctionalTestingFramework/Util/Logger/MftfLogger.php +++ b/src/Magento/FunctionalTestingFramework/Util/Logger/MftfLogger.php @@ -13,7 +13,7 @@ class MftfLogger extends Logger { /** - * Prints a deprecation warning, as well as adding a log at the WARNING level. + * Prints a deprecation warning, as well as adds a log at the WARNING level. * * @param string $message The log message. * @param array $context The log context. @@ -28,4 +28,21 @@ public function deprecation($message, array $context = []) } parent::warning($message, $context); } + + /** + * Prints a critical failure, as well as adds a log at the CRITICAL level. + * + * @param string $message The log message. + * @param array $context The log context. + * @return void + */ + public function criticalFailure($message, array $context = []) + { + $message = "FAILURE: " . $message; + // Suppress print during unit testing + if (MftfApplicationConfig::getConfig()->getPhase() !== MftfApplicationConfig::UNIT_TEST_PHASE) { + print ($message . implode("\n", $context) . "\n"); + } + parent::critical($message, $context); + } }