diff --git a/docs/commands/mftf.md b/docs/commands/mftf.md index 12e1ead28..fa6f1646c 100644 --- a/docs/commands/mftf.md +++ b/docs/commands/mftf.md @@ -122,19 +122,9 @@ vendor/bin/mftf generate:tests [option] [] [] [--remove] | `--force` | Forces test generation, regardless of the module merge order defined in the Magento instance. Example: `generate:tests --force`. | | `-i,--time` | Set time in minutes to determine the group size when `--config=parallel` is used. The __default value__ is `10`. Example: `generate:tests --config=parallel --time=15`| | `--tests` | Defines the test configuration as a JSON string.| -| `--debug=[ or ]`| Set debug level to `default` when the option is not specified, to `developer` with `--debug developer` or `--debug`, to `none` with `--debug none` | | +| `--debug or --debug=[]`| Performs schema validations on XML files.

DEFAULT: `generate:tests` implicitly performs schema validation on merged files. It does not indicate the file name where the error is encountered.
DEVELOPER: `--debug` performs per-file validation and returns additional debug information (such as the filename where an error occurred) when test generation fails because of an invalid XML schema. This option takes extra processing time. Use it after test generation has failed once.

NONE: `--debug=none` skips debugging during test generation. Added for backward compatibility, it will be removed in the next MAJOR release.
| | `-r,--remove`| Removes the existing generated suites and tests cleaning up the `_generated` directory before the actual run. For example, `generate:tests SampleTest --remove` cleans up the entire `_generated` directory and generates `SampleTest` only.| -#### Debugging levels - -You can run `generate:tests` with any of the following debug levels. XML schema validation errors will be logged as CRITICAL failures. - -| Debug level | Description | -| ---| --- | -|`default` `[generate:tests]` | Perform XML schema validation on merged files. Does not indicate the file name where the error is encountered. Use `developer` level for enhanced debugging. | -|`developer` `[generate:tests --debug or generate:tests --debug developer]` | Returns additional debug information (such as the filename where an error occurred) when test generation fails because of an invalid XML schema. Note: This option takes extra processing time. Use it after test generation has failed once. | -|`none` `[generate:tests --debug none]`| Skip debugging during test generation. Added for backward compatibility, it will be removed with the next MAJOR release.| - #### Examples of the JSON configuration The configuration to generate a single test with no suites: @@ -305,6 +295,7 @@ vendor/bin/mftf run:group [--skip-generate|--remove] [--] [] | --------------------- | --------------------------------------------------------------------------------------------------------- | | `-k, --skip-generate` | Skips generating from the source XML. Instead, the command executes previously-generated groups of tests. | | `-r, --remove` | Removes previously generated suites and tests before the actual generation and run. | +| `--debug or --debug=[]`| Performs schema validations on XML files. `run:group` implicitly performs schema validation on merged files. It does not indicate the file name where the error is encountered. `--debug` performs per-file validation and returns additional debug information (such as the filename where an error occurred). `--debug=none` skips debugging during test run. Added for backward compatibility, it will be removed in the next MAJOR release.| #### Examples @@ -336,6 +327,7 @@ vendor/bin/mftf run:test [--skip-generate|--remove] [--] [] |-----------------------|-----------------------------------------------------------------------------------------------------------| | `-k, --skip-generate` | Skips generating from the source XML. Instead, the command executes previously-generated groups of tests. | | `-r, --remove` | Remove previously generated suites and tests. | +| `--debug or --debug=[]`| Performs schema validations on XML files. `run:test` implicitly performs schema validation on merged files. It does not indicate the file name where the error is encountered. `--debug` performs per-file validation and returns additional debug information (such as the filename where an error occurred). `--debug=none` skips debugging during test run. Added for backward compatibility, it will be removed in the next MAJOR release. #### Examples @@ -357,6 +349,11 @@ For more details about `failed`, refer to [Reporting][]. ```bash vendor/bin/mftf run:failed ``` +#### Options + +| Option | Description | +|-----------------------|-----------------------------------------------------------------------------------------------------------| +| `--debug or --debug=[]`| Performs schema validations on XML files. `run:failed` implicitly performs schema validation on merged files. It does not indicate the file name where the error is encountered. `--debug` performs per-file validation and returns additional debug information (such as the filename where an error occurred). Use it after test run has failed once. `--debug=none` skips debugging during test run. Added for backward compatibility, it will be removed in the next MAJOR release.| #### Examples diff --git a/src/Magento/FunctionalTestingFramework/Config/Reader/MftfFilesystem.php b/src/Magento/FunctionalTestingFramework/Config/Reader/MftfFilesystem.php index 6fbc2fdfc..cd075e57c 100644 --- a/src/Magento/FunctionalTestingFramework/Config/Reader/MftfFilesystem.php +++ b/src/Magento/FunctionalTestingFramework/Config/Reader/MftfFilesystem.php @@ -42,7 +42,7 @@ public function readFiles($fileList) $configMerger->merge($content, $fileList->getFilename(), $exceptionCollector); } // run per file validation with generate:tests -d - if ($debugLevel == MftfApplicationConfig::LEVEL_DEVELOPER) { + if ($debugLevel === MftfApplicationConfig::LEVEL_DEVELOPER) { $this->validateSchema($configMerger, $fileList->getFilename()); } } catch (\Magento\FunctionalTestingFramework\Config\Dom\ValidationException $e) { @@ -52,7 +52,7 @@ public function readFiles($fileList) $exceptionCollector->throwException(); //run validation on merged file with generate:tests - if ($debugLevel == MftfApplicationConfig::LEVEL_DEFAULT) { + if ($debugLevel === MftfApplicationConfig::LEVEL_DEFAULT) { $this->validateSchema($configMerger); } diff --git a/src/Magento/FunctionalTestingFramework/Console/GenerateTestsCommand.php b/src/Magento/FunctionalTestingFramework/Console/GenerateTestsCommand.php index bc8c76142..23651b492 100644 --- a/src/Magento/FunctionalTestingFramework/Console/GenerateTestsCommand.php +++ b/src/Magento/FunctionalTestingFramework/Console/GenerateTestsCommand.php @@ -57,7 +57,7 @@ protected function configure() InputOption::VALUE_OPTIONAL, 'Run extra validation when generating tests. Use option \'none\' to turn off debugging -- added for backward compatibility, will be removed in the next MAJOR release', - 'default' + MftfApplicationConfig::LEVEL_DEFAULT ); parent::configure(); diff --git a/src/Magento/FunctionalTestingFramework/Console/RunTestCommand.php b/src/Magento/FunctionalTestingFramework/Console/RunTestCommand.php index bd9b4b99b..76a8bc8f4 100644 --- a/src/Magento/FunctionalTestingFramework/Console/RunTestCommand.php +++ b/src/Magento/FunctionalTestingFramework/Console/RunTestCommand.php @@ -37,6 +37,13 @@ protected function configure() 'f', InputOption::VALUE_NONE, 'force generation of tests regardless of Magento Instance Configuration' + )->addOption( + 'debug', + 'd', + InputOption::VALUE_OPTIONAL, + 'Run extra validation when running tests. Use option \'none\' to turn off debugging -- + added for backward compatibility, will be removed in the next MAJOR release', + MftfApplicationConfig::LEVEL_DEFAULT ); parent::configure(); @@ -58,6 +65,7 @@ protected function execute(InputInterface $input, OutputInterface $output): int $skipGeneration = $input->getOption('skip-generate'); $force = $input->getOption('force'); $remove = $input->getOption('remove'); + $debug = $input->getOption('debug') ?? MftfApplicationConfig::LEVEL_DEVELOPER; // for backward compatibility if ($skipGeneration and $remove) { // "skip-generate" and "remove" options cannot be used at the same time @@ -75,7 +83,7 @@ protected function execute(InputInterface $input, OutputInterface $output): int ]), '--force' => $force, '--remove' => $remove, - '--debug' => MftfApplicationConfig::LEVEL_NONE + '--debug' => $debug ]; $command->run(new ArrayInput($args), $output); } diff --git a/src/Magento/FunctionalTestingFramework/Console/RunTestFailedCommand.php b/src/Magento/FunctionalTestingFramework/Console/RunTestFailedCommand.php index 899fa8312..d6a580d52 100644 --- a/src/Magento/FunctionalTestingFramework/Console/RunTestFailedCommand.php +++ b/src/Magento/FunctionalTestingFramework/Console/RunTestFailedCommand.php @@ -13,6 +13,7 @@ use Symfony\Component\Console\Output\OutputInterface; use Symfony\Component\Process\Process; use Magento\FunctionalTestingFramework\Exceptions\TestFrameworkException; +use Symfony\Component\Console\Input\InputOption; class RunTestFailedCommand extends BaseGenerateCommand { @@ -49,7 +50,15 @@ class RunTestFailedCommand extends BaseGenerateCommand protected function configure() { $this->setName('run:failed') - ->setDescription('Execute a set of tests referenced via failed file'); + ->setDescription('Execute a set of tests referenced via failed file') + ->addOption( + 'debug', + 'd', + InputOption::VALUE_OPTIONAL, + 'Run extra validation when running failed tests. Use option \'none\' to turn off debugging -- + added for backward compatibility, will be removed in the next MAJOR release', + MftfApplicationConfig::LEVEL_DEFAULT + ); parent::configure(); } @@ -67,12 +76,13 @@ protected function configure() */ protected function execute(InputInterface $input, OutputInterface $output): int { + $debug = $input->getOption('debug') ?? MftfApplicationConfig::LEVEL_DEVELOPER; // for backward compatibility // Create Mftf Configuration MftfApplicationConfig::create( false, MftfApplicationConfig::GENERATION_PHASE, false, - MftfApplicationConfig::LEVEL_NONE + $debug ); $testConfiguration = $this->getFailedTestList(); @@ -83,7 +93,11 @@ protected function execute(InputInterface $input, OutputInterface $output): int } $command = $this->getApplication()->find('generate:tests'); - $args = ['--tests' => $testConfiguration, '--remove' => true]; + $args = [ + '--tests' => $testConfiguration, + '--remove' => true, + '--debug' => $debug + ]; $command->run(new ArrayInput($args), $output); diff --git a/src/Magento/FunctionalTestingFramework/Console/RunTestGroupCommand.php b/src/Magento/FunctionalTestingFramework/Console/RunTestGroupCommand.php index 174fd8130..d714a1dd1 100644 --- a/src/Magento/FunctionalTestingFramework/Console/RunTestGroupCommand.php +++ b/src/Magento/FunctionalTestingFramework/Console/RunTestGroupCommand.php @@ -39,6 +39,13 @@ protected function configure() 'f', InputOption::VALUE_NONE, 'force generation of tests regardless of Magento Instance Configuration' + )->addOption( + 'debug', + 'd', + InputOption::VALUE_OPTIONAL, + 'Run extra validation when running tests. Use option \'none\' to turn off debugging -- + added for backward compatibility, will be removed in the next MAJOR release', + MftfApplicationConfig::LEVEL_DEFAULT )->addArgument( 'groups', InputArgument::IS_ARRAY | InputArgument::REQUIRED, @@ -64,6 +71,7 @@ protected function execute(InputInterface $input, OutputInterface $output): int $force = $input->getOption('force'); $groups = $input->getArgument('groups'); $remove = $input->getOption('remove'); + $debug = $input->getOption('debug') ?? MftfApplicationConfig::LEVEL_DEVELOPER; // for backward compatibility if ($skipGeneration and $remove) { // "skip-generate" and "remove" options cannot be used at the same time @@ -77,7 +85,7 @@ protected function execute(InputInterface $input, OutputInterface $output): int $force, MftfApplicationConfig::GENERATION_PHASE, false, - MftfApplicationConfig::LEVEL_NONE + $debug ); if (!$skipGeneration) { @@ -86,7 +94,8 @@ protected function execute(InputInterface $input, OutputInterface $output): int $args = [ '--tests' => $testConfiguration, '--force' => $force, - '--remove' => $remove + '--remove' => $remove, + '--debug' => $debug ]; $command->run(new ArrayInput($args), $output);