diff --git a/CHANGELOG.md b/CHANGELOG.md
index 5a24b5f29..fd555ee88 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,5 +1,41 @@
Magento Functional Testing Framework Changelog
================================================
+3.1.0
+________
+
+### Enhancements
+
+* Customizability
+ * Introduced the new `return` action that allows action groups to return a value. See the [actions page](./docs/test/actions.md#return) for details.
+ * Introduced new MFTF command that invokes `vendor/bin/codecept run`. See the [mftf page](./docs/commands/mftf.md#codeceptrun) for details.
+
+* Usability
+ * Introduced new action `pause`, to invoke codeception interactive pause for debugging during test execution. See the [Interactive Pause](./docs/interactive-pause.md) page for details.
+ * Introduced a new `.env` configuration option `ENABLE_PAUSE`, to enable the new pause feature.
+
+* Maintainability
+ * Added a new static check that checks for the usage of the `pause` action. See the [command page](./docs/commands/mftf.md#static-checks) for details.
+
+* Modularity
+ * MFTF now supports the use of actions from multiple modules within suites.
+
+* Traceability
+ * A deprecation notice is now added at test execution time for deprecated metadata usage.
+
+### Fixes
+
+* Fixed issue with suite precondition failure for `createData` with required entity.
+
+### GitHub Issues/Pull requests:
+
+ * [#547](https://github.com/magento/magento2-functional-testing-framework/pull/547) -- Fix invalid behavior of MAGENTO_BACKEND_BASE_URL
+ * [#742](https://github.com/magento/magento2-functional-testing-framework/pull/742) -- Fix Waits In MagentoPwaWebDriver
+ * [#750](https://github.com/magento/magento2-functional-testing-framework/pull/750) -- Docs: Outlining the difference between Allure severity levels
+ * [#683](https://github.com/magento/magento2-functional-testing-framework/pull/683) -- Docs: Renamed sample test name with the correct one
+ * [#691](https://github.com/magento/magento2-functional-testing-framework/pull/691) -- Docs: Branch name updates
+ * [#678](https://github.com/magento/magento2-functional-testing-framework/pull/678) -- Docs: Command added to modify the web server rewrites configuration
+ * [#745](https://github.com/magento/magento2-functional-testing-framework/pull/745) -- Docs: Remove invalid sample test name
+
3.0.0
---------
diff --git a/composer.json b/composer.json
index 6f8b0c2c2..813ab382b 100755
--- a/composer.json
+++ b/composer.json
@@ -2,7 +2,7 @@
"name": "magento/magento2-functional-testing-framework",
"description": "Magento2 Functional Testing Framework",
"type": "library",
- "version": "3.0.0",
+ "version": "3.1.0",
"license": "AGPL-3.0",
"keywords": ["magento", "automation", "functional", "testing"],
"config": {
diff --git a/composer.lock b/composer.lock
index 91fa2c772..7f7648096 100644
--- a/composer.lock
+++ b/composer.lock
@@ -4,7 +4,7 @@
"Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies",
"This file is @generated automatically"
],
- "content-hash": "98733866973fb51e11e316b98c0af563",
+ "content-hash": "278e33e2c7d183d0b7689b5a76127d29",
"packages": [
{
"name": "allure-framework/allure-codeception",
@@ -3664,6 +3664,7 @@
"keywords": [
"tokenizer"
],
+ "abandoned": true,
"time": "2020-02-07T06:19:00+00:00"
},
{
@@ -7469,5 +7470,6 @@
"ext-json": "*",
"ext-openssl": "*"
},
- "platform-dev": []
+ "platform-dev": [],
+ "plugin-api-version": "1.1.0"
}
diff --git a/dev/tests/unit/Magento/FunctionalTestFramework/Test/Handlers/TestObjectHandlerTest.php b/dev/tests/unit/Magento/FunctionalTestFramework/Test/Handlers/TestObjectHandlerTest.php
index 036656cdd..939dd496e 100644
--- a/dev/tests/unit/Magento/FunctionalTestFramework/Test/Handlers/TestObjectHandlerTest.php
+++ b/dev/tests/unit/Magento/FunctionalTestFramework/Test/Handlers/TestObjectHandlerTest.php
@@ -260,25 +260,6 @@ public function testGetAllTestObjectsWithInvalidExtends()
$toh->getAllObjects();
}
- /**
- * Function used to set mock for parser return and force init method to run between tests.
- *
- * @param array $data
- * @throws \Exception
- */
- private function setMockParserOutput($data)
- {
- // clear test object handler value to inject parsed content
- $property = new \ReflectionProperty(TestObjectHandler::class, 'testObjectHandler');
- $property->setAccessible(true);
- $property->setValue(null);
-
- $mockDataParser = AspectMock::double(TestDataParser::class, ['readTestData' => $data])->make();
- $instance = AspectMock::double(ObjectManager::class, ['create' => $mockDataParser])
- ->make(); // bypass the private constructor
- AspectMock::double(ObjectManagerFactory::class, ['getObjectManager' => $instance]);
- }
-
/**
* Validate test object when ENABLE_PAUSE is set to true
*
@@ -299,7 +280,7 @@ public function testGetTestObjectWhenEnablePause()
$resolverMock = new MockModuleResolverBuilder();
$resolverMock->setup();
- $this->setMockParserOutput($mockData);
+ ObjectHandlerUtil::mockTestObjectHandlerWitData($mockData);
// run object handler method
$toh = TestObjectHandler::getInstance();
@@ -325,7 +306,7 @@ public function testGetTestObjectWhenEnablePause()
$expectedFailedActionObject2 = new ActionObject(
'pauseWhenFailed',
'pause',
- []
+ [ActionObject::PAUSE_ACTION_INTERNAL_ATTRIBUTE => true]
);
$expectedBeforeHookObject = new TestHookObject(
diff --git a/docs/test/actions.md b/docs/test/actions.md
index f69ee1d66..42ecde053 100644
--- a/docs/test/actions.md
+++ b/docs/test/actions.md
@@ -1251,8 +1251,9 @@ Attribute|Type|Use|Description
`stepKey`|string|required| A unique identifier of the action.
#### Example
+
```xml
-
```
diff --git a/src/Magento/FunctionalTestingFramework/Console/BaseGenerateCommand.php b/src/Magento/FunctionalTestingFramework/Console/BaseGenerateCommand.php
index 525711e31..401153c7b 100644
--- a/src/Magento/FunctionalTestingFramework/Console/BaseGenerateCommand.php
+++ b/src/Magento/FunctionalTestingFramework/Console/BaseGenerateCommand.php
@@ -33,6 +33,7 @@ class BaseGenerateCommand extends Command
const MFTF_NOTICES = "Placeholder text for MFTF notices\n";
const CODECEPT_RUN = 'codecept:run';
const CODECEPT_RUN_FUNCTIONAL = self::CODECEPT_RUN . ' functional ';
+ const CODECEPT_RUN_OPTION_NO_EXIT = ' --no-exit ';
/**
* Enable pause()
diff --git a/src/Magento/FunctionalTestingFramework/Console/RunManifestCommand.php b/src/Magento/FunctionalTestingFramework/Console/RunManifestCommand.php
index 89ae90888..683be1b53 100644
--- a/src/Magento/FunctionalTestingFramework/Console/RunManifestCommand.php
+++ b/src/Magento/FunctionalTestingFramework/Console/RunManifestCommand.php
@@ -81,12 +81,17 @@ protected function execute(InputInterface $input, OutputInterface $output): int
// Delete the Codeception failed file just in case it exists from any previous test runs
$this->deleteFailedFile();
- foreach ($manifestFile as $manifestLine) {
- if (empty($manifestLine)) {
+ for ($line = 0; $line < count($manifestFile); $line++) {
+ if (empty($manifestFile[$line])) {
continue;
}
- $this->runManifestLine($manifestLine, $output);
+ if ($line == count($manifestFile) - 1) {
+ $this->runManifestLine($manifestFile[$line], $output, true);
+ } else {
+ $this->runManifestLine($manifestFile[$line], $output);
+ }
+
$this->aggregateFailed();
}
@@ -103,18 +108,23 @@ protected function execute(InputInterface $input, OutputInterface $output): int
*
* @param string $manifestLine
* @param OutputInterface $output
+ * @param boolean $exit
* @return void
* @throws \Exception
*
* @SuppressWarnings(PHPMD.UnusedLocalVariable) Need this because of the unused $type variable in the closure
*/
- private function runManifestLine(string $manifestLine, OutputInterface $output)
+ private function runManifestLine($manifestLine, $output, $exit = false)
{
if (getenv('ENABLE_PAUSE') === 'true') {
$codeceptionCommand = BaseGenerateCommand::CODECEPT_RUN_FUNCTIONAL
- . '--verbose --steps --debug ' . $manifestLine;
+ . '--verbose --steps --debug ';
+ if (!$exit) {
+ $codeceptionCommand .= BaseGenerateCommand::CODECEPT_RUN_OPTION_NO_EXIT;
+ }
+ $codeceptionCommand .= $manifestLine;
$input = new StringInput($codeceptionCommand);
- $command = $this->getApplication()->find('codecept:run');
+ $command = $this->getApplication()->find(BaseGenerateCommand::CODECEPT_RUN);
$subReturnCode = $command->run($input, $output);
} else {
$codeceptionCommand = realpath(PROJECT_ROOT . "/vendor/bin/codecept")
diff --git a/src/Magento/FunctionalTestingFramework/Console/RunTestCommand.php b/src/Magento/FunctionalTestingFramework/Console/RunTestCommand.php
index 3d99b6fad..6931e8303 100644
--- a/src/Magento/FunctionalTestingFramework/Console/RunTestCommand.php
+++ b/src/Magento/FunctionalTestingFramework/Console/RunTestCommand.php
@@ -135,8 +135,8 @@ private function runTests(array $tests, OutputInterface $output)
TestGenerator::DEFAULT_DIR .
DIRECTORY_SEPARATOR ;
- foreach ($tests as $test) {
- $testName = $test . 'Cest.php';
+ for ($i = 0; $i < count($tests); $i++) {
+ $testName = $tests[$i] . 'Cest.php';
if (!realpath($testsDirectory . $testName)) {
throw new TestFrameworkException(
$testName . " is not available under " . $testsDirectory
@@ -145,6 +145,9 @@ private function runTests(array $tests, OutputInterface $output)
if ($this->pauseEnabled()) {
$fullCommand = $codeceptionCommand . $testsDirectory . $testName . ' --verbose --steps --debug';
+ if ($i != count($tests) - 1) {
+ $fullCommand .= self::CODECEPT_RUN_OPTION_NO_EXIT;
+ }
$this->returnCode = max($this->returnCode, $this->codeceptRunTest($fullCommand, $output));
} else {
$fullCommand = $codeceptionCommand . $testsDirectory . $testName . ' --verbose --steps';
@@ -169,10 +172,18 @@ private function runTestsInSuite(array $suitesConfig, OutputInterface $output)
$codeceptionCommand = realpath(PROJECT_ROOT . '/vendor/bin/codecept')
. ' run functional --verbose --steps ';
}
+
+ $count = count($suitesConfig);
+ $index = 0;
//for tests in suites, run them as a group to run before and after block
foreach (array_keys($suitesConfig) as $suite) {
$fullCommand = $codeceptionCommand . " -g {$suite}";
+
+ $index += 1;
if ($this->pauseEnabled()) {
+ if ($index != $count) {
+ $fullCommand .= self::CODECEPT_RUN_OPTION_NO_EXIT;
+ }
$this->returnCode = max($this->returnCode, $this->codeceptRunTest($fullCommand, $output));
} else {
$this->returnCode = max($this->returnCode, $this->executeTestCommand($fullCommand, $output));
diff --git a/src/Magento/FunctionalTestingFramework/Console/RunTestFailedCommand.php b/src/Magento/FunctionalTestingFramework/Console/RunTestFailedCommand.php
index 87606d53d..6e6954c70 100644
--- a/src/Magento/FunctionalTestingFramework/Console/RunTestFailedCommand.php
+++ b/src/Magento/FunctionalTestingFramework/Console/RunTestFailedCommand.php
@@ -116,13 +116,16 @@ protected function execute(InputInterface $input, OutputInterface $output): int
$testManifestList = $this->readTestManifestFile();
$returnCode = 0;
- foreach ($testManifestList as $testCommand) {
+ for ($i = 0; $i < count($testManifestList); $i++) {
if ($this->pauseEnabled()) {
- $codeceptionCommand = self::CODECEPT_RUN_FUNCTIONAL . $testCommand . ' --debug';
+ $codeceptionCommand = self::CODECEPT_RUN_FUNCTIONAL . $testManifestList[$i] . ' --debug ';
+ if ($i != count($testManifestList) - 1) {
+ $codeceptionCommand .= self::CODECEPT_RUN_OPTION_NO_EXIT;
+ }
$returnCode = $this->codeceptRunTest($codeceptionCommand, $output);
} else {
$codeceptionCommand = realpath(PROJECT_ROOT . '/vendor/bin/codecept') . ' run functional ';
- $codeceptionCommand .= $testCommand;
+ $codeceptionCommand .= $testManifestList[$i];
$process = new Process($codeceptionCommand);
$process->setWorkingDirectory(TESTS_BP);
@@ -142,6 +145,7 @@ function ($type, $buffer) use ($output) {
);
}
}
+
foreach ($this->failedList as $test) {
$this->writeFailedTestToFile($test, $this->testsFailedFile);
}
diff --git a/src/Magento/FunctionalTestingFramework/Console/RunTestGroupCommand.php b/src/Magento/FunctionalTestingFramework/Console/RunTestGroupCommand.php
index 896218efc..f18b9dc30 100644
--- a/src/Magento/FunctionalTestingFramework/Console/RunTestGroupCommand.php
+++ b/src/Magento/FunctionalTestingFramework/Console/RunTestGroupCommand.php
@@ -52,6 +52,7 @@ protected function configure()
* @throws \Exception
*
* @SuppressWarnings(PHPMD.UnusedLocalVariable)
+ * @SuppressWarnings(PHPMD.CyclomaticComplexity)
*/
protected function execute(InputInterface $input, OutputInterface $output): int
{
@@ -102,10 +103,13 @@ protected function execute(InputInterface $input, OutputInterface $output): int
$exitCode = -1;
$returnCodes = [];
- foreach ($groups as $group) {
- $codeceptionCommandString = $commandString . " -g {$group}";
+ for ($i = 0; $i < count($groups); $i++) {
+ $codeceptionCommandString = $commandString . ' -g ' . $groups[$i];
if ($this->pauseEnabled()) {
+ if ($i != count($groups) - 1) {
+ $codeceptionCommandString .= self::CODECEPT_RUN_OPTION_NO_EXIT;
+ }
$returnCodes[] = $this->codeceptRunTest($codeceptionCommandString, $output);
} else {
$process = new Process($codeceptionCommandString);
diff --git a/src/Magento/FunctionalTestingFramework/Module/MagentoWebDriver.php b/src/Magento/FunctionalTestingFramework/Module/MagentoWebDriver.php
index 13d44a358..00e295606 100644
--- a/src/Magento/FunctionalTestingFramework/Module/MagentoWebDriver.php
+++ b/src/Magento/FunctionalTestingFramework/Module/MagentoWebDriver.php
@@ -54,7 +54,9 @@
class MagentoWebDriver extends WebDriver
{
use AttachmentSupport;
- use Pause;
+ use Pause {
+ pause as codeceptPause;
+ }
const MAGENTO_CRON_INTERVAL = 60;
const MAGENTO_CRON_COMMAND = 'cron:run';
@@ -843,7 +845,7 @@ public function _failed(TestInterface $test, $fail)
if ($this->pngReport === null && $this->htmlReport === null) {
$this->saveScreenshot();
if (getenv('ENABLE_PAUSE') === 'true') {
- $this->pause();
+ $this->pause(true);
}
}
@@ -1028,4 +1030,23 @@ public function switchToIFrame($locator = null)
$this->webDriver->switchTo()->frame($els[0]);
}
}
+
+ /**
+ * Invoke Codeption pause()
+ *
+ * @param boolean $pauseOnFail
+ * @return void
+ */
+ public function pause($pauseOnFail = false)
+ {
+ if (!\Codeception\Util\Debug::isEnabled()) {
+ return;
+ }
+
+ if ($pauseOnFail) {
+ print(PHP_EOL . "Failure encountered. Pausing execution..." . PHP_EOL . PHP_EOL);
+ }
+
+ $this->codeceptPause();
+ }
}
diff --git a/src/Magento/FunctionalTestingFramework/Test/Objects/ActionObject.php b/src/Magento/FunctionalTestingFramework/Test/Objects/ActionObject.php
index 8d4e7a945..8e4aa973c 100644
--- a/src/Magento/FunctionalTestingFramework/Test/Objects/ActionObject.php
+++ b/src/Magento/FunctionalTestingFramework/Test/Objects/ActionObject.php
@@ -77,6 +77,7 @@ class ActionObject
const ACTION_TYPE_COMMENT = 'comment';
const ACTION_TYPE_HELPER = 'helper';
const INVISIBLE_STEP_ACTIONS = ['retrieveEntityField', 'getSecret'];
+ const PAUSE_ACTION_INTERNAL_ATTRIBUTE = 'pauseOnFail';
/**
* The unique identifier for the action
diff --git a/src/Magento/FunctionalTestingFramework/Test/Util/TestHookObjectExtractor.php b/src/Magento/FunctionalTestingFramework/Test/Util/TestHookObjectExtractor.php
index a68f09491..e9c163d31 100644
--- a/src/Magento/FunctionalTestingFramework/Test/Util/TestHookObjectExtractor.php
+++ b/src/Magento/FunctionalTestingFramework/Test/Util/TestHookObjectExtractor.php
@@ -66,7 +66,11 @@ public function createDefaultFailedHook($parentName)
{
$defaultSteps['saveScreenshot'] = new ActionObject("saveScreenshot", "saveScreenshot", []);
if (getenv('ENABLE_PAUSE') === 'true') {
- $defaultSteps['pauseWhenFailed'] = new ActionObject('pauseWhenFailed', 'pause', []);
+ $defaultSteps['pauseWhenFailed'] = new ActionObject(
+ 'pauseWhenFailed',
+ 'pause',
+ [ActionObject::PAUSE_ACTION_INTERNAL_ATTRIBUTE => true]
+ );
}
$hook = new TestHookObject(
diff --git a/src/Magento/FunctionalTestingFramework/Util/TestGenerator.php b/src/Magento/FunctionalTestingFramework/Util/TestGenerator.php
index 82fb2c06a..8caffc565 100644
--- a/src/Magento/FunctionalTestingFramework/Util/TestGenerator.php
+++ b/src/Magento/FunctionalTestingFramework/Util/TestGenerator.php
@@ -1441,6 +1441,16 @@ public function generateStepsPhp($actionObjects, $generationScope = TestGenerato
$testSteps .= $dateGenerateCode;
break;
+ case "pause":
+ $pauseAttr = $actionObject->getCustomActionAttributes(
+ ActionObject::PAUSE_ACTION_INTERNAL_ATTRIBUTE
+ );
+ if ($pauseAttr) {
+ $testSteps .= sprintf("\t\t$%s->%s(%s);", $actor, $actionObject->getType(), 'true');
+ } else {
+ $testSteps .= sprintf("\t\t$%s->%s();", $actor, $actionObject->getType());
+ }
+ break;
case "comment":
$input = $input === null ? strtr($value, ['$' => '\$', '{' => '\{', '}' => '\}']) : $input;
// Combining userInput from native XML comment and action to fall-through 'default' case