diff --git a/CHANGELOG.md b/CHANGELOG.md index 5a24b5f29..fd555ee88 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,41 @@ Magento Functional Testing Framework Changelog ================================================ +3.1.0 +________ + +### Enhancements + +* Customizability + * Introduced the new `return` action that allows action groups to return a value. See the [actions page](./docs/test/actions.md#return) for details. + * Introduced new MFTF command that invokes `vendor/bin/codecept run`. See the [mftf page](./docs/commands/mftf.md#codeceptrun) for details. + +* Usability + * Introduced new action `pause`, to invoke codeception interactive pause for debugging during test execution. See the [Interactive Pause](./docs/interactive-pause.md) page for details. + * Introduced a new `.env` configuration option `ENABLE_PAUSE`, to enable the new pause feature. + +* Maintainability + * Added a new static check that checks for the usage of the `pause` action. See the [command page](./docs/commands/mftf.md#static-checks) for details. + +* Modularity + * MFTF now supports the use of actions from multiple modules within suites. + +* Traceability + * A deprecation notice is now added at test execution time for deprecated metadata usage. + +### Fixes + +* Fixed issue with suite precondition failure for `createData` with required entity. + +### GitHub Issues/Pull requests: + + * [#547](https://github.com/magento/magento2-functional-testing-framework/pull/547) -- Fix invalid behavior of MAGENTO_BACKEND_BASE_URL + * [#742](https://github.com/magento/magento2-functional-testing-framework/pull/742) -- Fix Waits In MagentoPwaWebDriver + * [#750](https://github.com/magento/magento2-functional-testing-framework/pull/750) -- Docs: Outlining the difference between Allure severity levels + * [#683](https://github.com/magento/magento2-functional-testing-framework/pull/683) -- Docs: Renamed sample test name with the correct one + * [#691](https://github.com/magento/magento2-functional-testing-framework/pull/691) -- Docs: Branch name updates + * [#678](https://github.com/magento/magento2-functional-testing-framework/pull/678) -- Docs: Command added to modify the web server rewrites configuration + * [#745](https://github.com/magento/magento2-functional-testing-framework/pull/745) -- Docs: Remove invalid sample test name + 3.0.0 --------- diff --git a/composer.json b/composer.json index 6f8b0c2c2..813ab382b 100755 --- a/composer.json +++ b/composer.json @@ -2,7 +2,7 @@ "name": "magento/magento2-functional-testing-framework", "description": "Magento2 Functional Testing Framework", "type": "library", - "version": "3.0.0", + "version": "3.1.0", "license": "AGPL-3.0", "keywords": ["magento", "automation", "functional", "testing"], "config": { diff --git a/composer.lock b/composer.lock index 91fa2c772..7f7648096 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "98733866973fb51e11e316b98c0af563", + "content-hash": "278e33e2c7d183d0b7689b5a76127d29", "packages": [ { "name": "allure-framework/allure-codeception", @@ -3664,6 +3664,7 @@ "keywords": [ "tokenizer" ], + "abandoned": true, "time": "2020-02-07T06:19:00+00:00" }, { @@ -7469,5 +7470,6 @@ "ext-json": "*", "ext-openssl": "*" }, - "platform-dev": [] + "platform-dev": [], + "plugin-api-version": "1.1.0" } diff --git a/dev/tests/unit/Magento/FunctionalTestFramework/Test/Handlers/TestObjectHandlerTest.php b/dev/tests/unit/Magento/FunctionalTestFramework/Test/Handlers/TestObjectHandlerTest.php index 036656cdd..939dd496e 100644 --- a/dev/tests/unit/Magento/FunctionalTestFramework/Test/Handlers/TestObjectHandlerTest.php +++ b/dev/tests/unit/Magento/FunctionalTestFramework/Test/Handlers/TestObjectHandlerTest.php @@ -260,25 +260,6 @@ public function testGetAllTestObjectsWithInvalidExtends() $toh->getAllObjects(); } - /** - * Function used to set mock for parser return and force init method to run between tests. - * - * @param array $data - * @throws \Exception - */ - private function setMockParserOutput($data) - { - // clear test object handler value to inject parsed content - $property = new \ReflectionProperty(TestObjectHandler::class, 'testObjectHandler'); - $property->setAccessible(true); - $property->setValue(null); - - $mockDataParser = AspectMock::double(TestDataParser::class, ['readTestData' => $data])->make(); - $instance = AspectMock::double(ObjectManager::class, ['create' => $mockDataParser]) - ->make(); // bypass the private constructor - AspectMock::double(ObjectManagerFactory::class, ['getObjectManager' => $instance]); - } - /** * Validate test object when ENABLE_PAUSE is set to true * @@ -299,7 +280,7 @@ public function testGetTestObjectWhenEnablePause() $resolverMock = new MockModuleResolverBuilder(); $resolverMock->setup(); - $this->setMockParserOutput($mockData); + ObjectHandlerUtil::mockTestObjectHandlerWitData($mockData); // run object handler method $toh = TestObjectHandler::getInstance(); @@ -325,7 +306,7 @@ public function testGetTestObjectWhenEnablePause() $expectedFailedActionObject2 = new ActionObject( 'pauseWhenFailed', 'pause', - [] + [ActionObject::PAUSE_ACTION_INTERNAL_ATTRIBUTE => true] ); $expectedBeforeHookObject = new TestHookObject( diff --git a/docs/test/actions.md b/docs/test/actions.md index f69ee1d66..42ecde053 100644 --- a/docs/test/actions.md +++ b/docs/test/actions.md @@ -1251,8 +1251,9 @@ Attribute|Type|Use|Description `stepKey`|string|required| A unique identifier of the action. #### Example + ```xml - ``` diff --git a/src/Magento/FunctionalTestingFramework/Console/BaseGenerateCommand.php b/src/Magento/FunctionalTestingFramework/Console/BaseGenerateCommand.php index 525711e31..401153c7b 100644 --- a/src/Magento/FunctionalTestingFramework/Console/BaseGenerateCommand.php +++ b/src/Magento/FunctionalTestingFramework/Console/BaseGenerateCommand.php @@ -33,6 +33,7 @@ class BaseGenerateCommand extends Command const MFTF_NOTICES = "Placeholder text for MFTF notices\n"; const CODECEPT_RUN = 'codecept:run'; const CODECEPT_RUN_FUNCTIONAL = self::CODECEPT_RUN . ' functional '; + const CODECEPT_RUN_OPTION_NO_EXIT = ' --no-exit '; /** * Enable pause() diff --git a/src/Magento/FunctionalTestingFramework/Console/RunManifestCommand.php b/src/Magento/FunctionalTestingFramework/Console/RunManifestCommand.php index 89ae90888..683be1b53 100644 --- a/src/Magento/FunctionalTestingFramework/Console/RunManifestCommand.php +++ b/src/Magento/FunctionalTestingFramework/Console/RunManifestCommand.php @@ -81,12 +81,17 @@ protected function execute(InputInterface $input, OutputInterface $output): int // Delete the Codeception failed file just in case it exists from any previous test runs $this->deleteFailedFile(); - foreach ($manifestFile as $manifestLine) { - if (empty($manifestLine)) { + for ($line = 0; $line < count($manifestFile); $line++) { + if (empty($manifestFile[$line])) { continue; } - $this->runManifestLine($manifestLine, $output); + if ($line == count($manifestFile) - 1) { + $this->runManifestLine($manifestFile[$line], $output, true); + } else { + $this->runManifestLine($manifestFile[$line], $output); + } + $this->aggregateFailed(); } @@ -103,18 +108,23 @@ protected function execute(InputInterface $input, OutputInterface $output): int * * @param string $manifestLine * @param OutputInterface $output + * @param boolean $exit * @return void * @throws \Exception * * @SuppressWarnings(PHPMD.UnusedLocalVariable) Need this because of the unused $type variable in the closure */ - private function runManifestLine(string $manifestLine, OutputInterface $output) + private function runManifestLine($manifestLine, $output, $exit = false) { if (getenv('ENABLE_PAUSE') === 'true') { $codeceptionCommand = BaseGenerateCommand::CODECEPT_RUN_FUNCTIONAL - . '--verbose --steps --debug ' . $manifestLine; + . '--verbose --steps --debug '; + if (!$exit) { + $codeceptionCommand .= BaseGenerateCommand::CODECEPT_RUN_OPTION_NO_EXIT; + } + $codeceptionCommand .= $manifestLine; $input = new StringInput($codeceptionCommand); - $command = $this->getApplication()->find('codecept:run'); + $command = $this->getApplication()->find(BaseGenerateCommand::CODECEPT_RUN); $subReturnCode = $command->run($input, $output); } else { $codeceptionCommand = realpath(PROJECT_ROOT . "/vendor/bin/codecept") diff --git a/src/Magento/FunctionalTestingFramework/Console/RunTestCommand.php b/src/Magento/FunctionalTestingFramework/Console/RunTestCommand.php index 3d99b6fad..6931e8303 100644 --- a/src/Magento/FunctionalTestingFramework/Console/RunTestCommand.php +++ b/src/Magento/FunctionalTestingFramework/Console/RunTestCommand.php @@ -135,8 +135,8 @@ private function runTests(array $tests, OutputInterface $output) TestGenerator::DEFAULT_DIR . DIRECTORY_SEPARATOR ; - foreach ($tests as $test) { - $testName = $test . 'Cest.php'; + for ($i = 0; $i < count($tests); $i++) { + $testName = $tests[$i] . 'Cest.php'; if (!realpath($testsDirectory . $testName)) { throw new TestFrameworkException( $testName . " is not available under " . $testsDirectory @@ -145,6 +145,9 @@ private function runTests(array $tests, OutputInterface $output) if ($this->pauseEnabled()) { $fullCommand = $codeceptionCommand . $testsDirectory . $testName . ' --verbose --steps --debug'; + if ($i != count($tests) - 1) { + $fullCommand .= self::CODECEPT_RUN_OPTION_NO_EXIT; + } $this->returnCode = max($this->returnCode, $this->codeceptRunTest($fullCommand, $output)); } else { $fullCommand = $codeceptionCommand . $testsDirectory . $testName . ' --verbose --steps'; @@ -169,10 +172,18 @@ private function runTestsInSuite(array $suitesConfig, OutputInterface $output) $codeceptionCommand = realpath(PROJECT_ROOT . '/vendor/bin/codecept') . ' run functional --verbose --steps '; } + + $count = count($suitesConfig); + $index = 0; //for tests in suites, run them as a group to run before and after block foreach (array_keys($suitesConfig) as $suite) { $fullCommand = $codeceptionCommand . " -g {$suite}"; + + $index += 1; if ($this->pauseEnabled()) { + if ($index != $count) { + $fullCommand .= self::CODECEPT_RUN_OPTION_NO_EXIT; + } $this->returnCode = max($this->returnCode, $this->codeceptRunTest($fullCommand, $output)); } else { $this->returnCode = max($this->returnCode, $this->executeTestCommand($fullCommand, $output)); diff --git a/src/Magento/FunctionalTestingFramework/Console/RunTestFailedCommand.php b/src/Magento/FunctionalTestingFramework/Console/RunTestFailedCommand.php index 87606d53d..6e6954c70 100644 --- a/src/Magento/FunctionalTestingFramework/Console/RunTestFailedCommand.php +++ b/src/Magento/FunctionalTestingFramework/Console/RunTestFailedCommand.php @@ -116,13 +116,16 @@ protected function execute(InputInterface $input, OutputInterface $output): int $testManifestList = $this->readTestManifestFile(); $returnCode = 0; - foreach ($testManifestList as $testCommand) { + for ($i = 0; $i < count($testManifestList); $i++) { if ($this->pauseEnabled()) { - $codeceptionCommand = self::CODECEPT_RUN_FUNCTIONAL . $testCommand . ' --debug'; + $codeceptionCommand = self::CODECEPT_RUN_FUNCTIONAL . $testManifestList[$i] . ' --debug '; + if ($i != count($testManifestList) - 1) { + $codeceptionCommand .= self::CODECEPT_RUN_OPTION_NO_EXIT; + } $returnCode = $this->codeceptRunTest($codeceptionCommand, $output); } else { $codeceptionCommand = realpath(PROJECT_ROOT . '/vendor/bin/codecept') . ' run functional '; - $codeceptionCommand .= $testCommand; + $codeceptionCommand .= $testManifestList[$i]; $process = new Process($codeceptionCommand); $process->setWorkingDirectory(TESTS_BP); @@ -142,6 +145,7 @@ function ($type, $buffer) use ($output) { ); } } + foreach ($this->failedList as $test) { $this->writeFailedTestToFile($test, $this->testsFailedFile); } diff --git a/src/Magento/FunctionalTestingFramework/Console/RunTestGroupCommand.php b/src/Magento/FunctionalTestingFramework/Console/RunTestGroupCommand.php index 896218efc..f18b9dc30 100644 --- a/src/Magento/FunctionalTestingFramework/Console/RunTestGroupCommand.php +++ b/src/Magento/FunctionalTestingFramework/Console/RunTestGroupCommand.php @@ -52,6 +52,7 @@ protected function configure() * @throws \Exception * * @SuppressWarnings(PHPMD.UnusedLocalVariable) + * @SuppressWarnings(PHPMD.CyclomaticComplexity) */ protected function execute(InputInterface $input, OutputInterface $output): int { @@ -102,10 +103,13 @@ protected function execute(InputInterface $input, OutputInterface $output): int $exitCode = -1; $returnCodes = []; - foreach ($groups as $group) { - $codeceptionCommandString = $commandString . " -g {$group}"; + for ($i = 0; $i < count($groups); $i++) { + $codeceptionCommandString = $commandString . ' -g ' . $groups[$i]; if ($this->pauseEnabled()) { + if ($i != count($groups) - 1) { + $codeceptionCommandString .= self::CODECEPT_RUN_OPTION_NO_EXIT; + } $returnCodes[] = $this->codeceptRunTest($codeceptionCommandString, $output); } else { $process = new Process($codeceptionCommandString); diff --git a/src/Magento/FunctionalTestingFramework/Module/MagentoWebDriver.php b/src/Magento/FunctionalTestingFramework/Module/MagentoWebDriver.php index 13d44a358..00e295606 100644 --- a/src/Magento/FunctionalTestingFramework/Module/MagentoWebDriver.php +++ b/src/Magento/FunctionalTestingFramework/Module/MagentoWebDriver.php @@ -54,7 +54,9 @@ class MagentoWebDriver extends WebDriver { use AttachmentSupport; - use Pause; + use Pause { + pause as codeceptPause; + } const MAGENTO_CRON_INTERVAL = 60; const MAGENTO_CRON_COMMAND = 'cron:run'; @@ -843,7 +845,7 @@ public function _failed(TestInterface $test, $fail) if ($this->pngReport === null && $this->htmlReport === null) { $this->saveScreenshot(); if (getenv('ENABLE_PAUSE') === 'true') { - $this->pause(); + $this->pause(true); } } @@ -1028,4 +1030,23 @@ public function switchToIFrame($locator = null) $this->webDriver->switchTo()->frame($els[0]); } } + + /** + * Invoke Codeption pause() + * + * @param boolean $pauseOnFail + * @return void + */ + public function pause($pauseOnFail = false) + { + if (!\Codeception\Util\Debug::isEnabled()) { + return; + } + + if ($pauseOnFail) { + print(PHP_EOL . "Failure encountered. Pausing execution..." . PHP_EOL . PHP_EOL); + } + + $this->codeceptPause(); + } } diff --git a/src/Magento/FunctionalTestingFramework/Test/Objects/ActionObject.php b/src/Magento/FunctionalTestingFramework/Test/Objects/ActionObject.php index 8d4e7a945..8e4aa973c 100644 --- a/src/Magento/FunctionalTestingFramework/Test/Objects/ActionObject.php +++ b/src/Magento/FunctionalTestingFramework/Test/Objects/ActionObject.php @@ -77,6 +77,7 @@ class ActionObject const ACTION_TYPE_COMMENT = 'comment'; const ACTION_TYPE_HELPER = 'helper'; const INVISIBLE_STEP_ACTIONS = ['retrieveEntityField', 'getSecret']; + const PAUSE_ACTION_INTERNAL_ATTRIBUTE = 'pauseOnFail'; /** * The unique identifier for the action diff --git a/src/Magento/FunctionalTestingFramework/Test/Util/TestHookObjectExtractor.php b/src/Magento/FunctionalTestingFramework/Test/Util/TestHookObjectExtractor.php index a68f09491..e9c163d31 100644 --- a/src/Magento/FunctionalTestingFramework/Test/Util/TestHookObjectExtractor.php +++ b/src/Magento/FunctionalTestingFramework/Test/Util/TestHookObjectExtractor.php @@ -66,7 +66,11 @@ public function createDefaultFailedHook($parentName) { $defaultSteps['saveScreenshot'] = new ActionObject("saveScreenshot", "saveScreenshot", []); if (getenv('ENABLE_PAUSE') === 'true') { - $defaultSteps['pauseWhenFailed'] = new ActionObject('pauseWhenFailed', 'pause', []); + $defaultSteps['pauseWhenFailed'] = new ActionObject( + 'pauseWhenFailed', + 'pause', + [ActionObject::PAUSE_ACTION_INTERNAL_ATTRIBUTE => true] + ); } $hook = new TestHookObject( diff --git a/src/Magento/FunctionalTestingFramework/Util/TestGenerator.php b/src/Magento/FunctionalTestingFramework/Util/TestGenerator.php index 82fb2c06a..8caffc565 100644 --- a/src/Magento/FunctionalTestingFramework/Util/TestGenerator.php +++ b/src/Magento/FunctionalTestingFramework/Util/TestGenerator.php @@ -1441,6 +1441,16 @@ public function generateStepsPhp($actionObjects, $generationScope = TestGenerato $testSteps .= $dateGenerateCode; break; + case "pause": + $pauseAttr = $actionObject->getCustomActionAttributes( + ActionObject::PAUSE_ACTION_INTERNAL_ATTRIBUTE + ); + if ($pauseAttr) { + $testSteps .= sprintf("\t\t$%s->%s(%s);", $actor, $actionObject->getType(), 'true'); + } else { + $testSteps .= sprintf("\t\t$%s->%s();", $actor, $actionObject->getType()); + } + break; case "comment": $input = $input === null ? strtr($value, ['$' => '\$', '{' => '\{', '}' => '\}']) : $input; // Combining userInput from native XML comment and action to fall-through 'default' case