diff --git a/docs/commands/mftf.md b/docs/commands/mftf.md index aea53c96f..ac029779e 100644 --- a/docs/commands/mftf.md +++ b/docs/commands/mftf.md @@ -58,6 +58,14 @@ vendor/bin/mftf run:test LoginAsAdminTest LoginAsCustomerTest -r This command cleans up the previously generated tests; generates and runs the `LoginAsAdminTest` and `LoginAsCustomerTest` tests. +### Generate and run a testManifest.txt file + +```bash +vendor/bin/mftf run:manifest path/to/your/testManifest.txt +``` + +This command runs all tests specified in a `testManifest.txt` file. When you generate tests, a `testManifest.txt` file is also generated for you. You can pass this file directly to the `run:manifest` command and it will execute all listed tests. You can also create your own file in the same format to execute a subset of tests. Note: This command does not generate tests. + ### Generate and run previously failed tests ```bash @@ -338,6 +346,30 @@ Generate the `LoginCustomerTest` and `StorefrontCreateCustomerTest` tests from X vendor/bin/mftf run:test LoginCustomerTest StorefrontCreateCustomerTest ``` +### `run:manifest` + +Runs a testManifest.txt file. + +This command runs all tests specified in a testManifest.xml file. It does not generate tests for you. You must do that as first. + +#### Usage + +```bash +vendor/bin/mftf run:manifest path/to/your/testManifest.txt +``` + +#### Example testManifest.xml file + +Each line should contain either: one test path or one group (-g) reference. + +``` +tests/functional/tests/MFTF/_generated/default/AdminLoginTestCest.php +-g PaypalTestSuite +tests/functional/tests/MFTF/_generated/default/SomeOtherTestCest.php +tests/functional/tests/MFTF/_generated/default/ThirdTestCest.php +-g SomeOtherSuite +``` + ### `run:failed` Regenerates and reruns tests that previously failed. diff --git a/src/Magento/FunctionalTestingFramework/Console/CommandList.php b/src/Magento/FunctionalTestingFramework/Console/CommandList.php index 5bcf1ed31..34d221840 100644 --- a/src/Magento/FunctionalTestingFramework/Console/CommandList.php +++ b/src/Magento/FunctionalTestingFramework/Console/CommandList.php @@ -37,6 +37,7 @@ public function __construct(array $commands = []) 'run:test' => new RunTestCommand(), 'run:group' => new RunTestGroupCommand(), 'run:failed' => new RunTestFailedCommand(), + 'run:manifest' => new RunManifestCommand(), 'setup:env' => new SetupEnvCommand(), 'upgrade:tests' => new UpgradeTestsCommand(), 'generate:docs' => new GenerateDocsCommand(), diff --git a/src/Magento/FunctionalTestingFramework/Console/RunManifestCommand.php b/src/Magento/FunctionalTestingFramework/Console/RunManifestCommand.php new file mode 100644 index 000000000..e487d16cf --- /dev/null +++ b/src/Magento/FunctionalTestingFramework/Console/RunManifestCommand.php @@ -0,0 +1,152 @@ +setName("run:manifest") + ->setDescription("runs a manifest file") + ->addArgument("path", InputArgument::REQUIRED, "path to a manifest file"); + } + + /** + * Executes the run:manifest command. + * + * @param InputInterface $input + * @param OutputInterface $output + * @throws TestFrameworkException + * @return integer + */ + protected function execute(InputInterface $input, OutputInterface $output): int + { + $path = $input->getArgument("path"); + + if (!file_exists($path)) { + throw new TestFrameworkException("Could not find file $path. Check the path and try again."); + } + + $manifestFile = file($path, FILE_IGNORE_NEW_LINES); + + // Delete the Codeception failed file just in case it exists from any previous test runs + $this->deleteFailedFile(); + + foreach ($manifestFile as $manifestLine) { + if (empty($manifestLine)) { + continue; + } + + $this->runManifestLine($manifestLine, $output); + $this->aggregateFailed(); + } + + if (!empty($this->failedTests)) { + $this->deleteFailedFile(); + $this->writeFailedFile(); + } + + return $this->returnCode; + } + + /** + * Runs a test (or group) line from the manifest file + * + * @param string $manifestLine + * @param OutputInterface $output + * @return void + * + * @SuppressWarnings(PHPMD.UnusedLocalVariable) Need this because of the unused $type variable in the closure + */ + private function runManifestLine(string $manifestLine, OutputInterface $output) + { + $codeceptionCommand = realpath(PROJECT_ROOT . "/vendor/bin/codecept") + . " run functional --verbose --steps " + . $manifestLine; + + // run the codecept command in a sub process + $process = new Process($codeceptionCommand); + $process->setWorkingDirectory(TESTS_BP); + $process->setIdleTimeout(600); + $process->setTimeout(0); + $subReturnCode = $process->run(function ($type, $buffer) use ($output) { + $output->write($buffer); + }); + $this->returnCode = max($this->returnCode, $subReturnCode); + } + + /** + * Keeps track of any tests that failed while running the manifest file. + * + * Each codecept command executions overwrites the failed file. Since we are running multiple codecept commands, + * we need to hold on to any failures in order to write a final failed file containing all tests. + * + * @return void + */ + private function aggregateFailed() + { + if (file_exists(RunTestFailedCommand::TESTS_FAILED_FILE)) { + $currentFile = file(RunTestFailedCommand::TESTS_FAILED_FILE, FILE_IGNORE_NEW_LINES); + $this->failedTests = array_merge( + $this->failedTests, + $currentFile + ); + } + } + + /** + * Delete the Codeception failed file. + * + * @return void + */ + private function deleteFailedFile() + { + if (file_exists(RunTestFailedCommand::TESTS_FAILED_FILE)) { + unlink(RunTestFailedCommand::TESTS_FAILED_FILE); + } + } + + /** + * Writes any tests that failed to the Codeception failed file. + * + * @return void + */ + private function writeFailedFile() + { + foreach ($this->failedTests as $test) { + file_put_contents(RunTestFailedCommand::TESTS_FAILED_FILE, $test . "\n", FILE_APPEND); + } + } +}