magento · tomreece · Aug 19, 2020 · Aug 17, 2020 · Aug 18, 2020 · Aug 18, 2020
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,5 +1,41 @@
 Magento Functional Testing Framework Changelog
 ================================================
+3.1.0
+________
+
+### Enhancements
+
+* Customizability
+  * Introduced the new `return` action that allows action groups to return a value. See the [actions page](./docs/test/actions.md#return) for details.
+  * Introduced new MFTF command that invokes `vendor/bin/codecept run`. See the [mftf page](./docs/commands/mftf.md#codeceptrun) for details.
+
+* Usability
+  * Introduced new action `pause`, to invoke codeception interactive pause for debugging during test execution. See the [Interactive Pause](./docs/interactive-pause.md) page for details.
+  * Introduced a new `.env` configuration option `ENABLE_PAUSE`, to enable the new pause feature.
+
+* Maintainability    
+  * Added a new static check that checks for the usage of the `pause` action. See the [command page](./docs/commands/mftf.md#static-checks) for details.
+
+* Modularity   
+  * MFTF now supports the use of actions from multiple modules within suites.
+
+* Traceability
+  * A deprecation notice is now added at test execution time for deprecated metadata usage.
+
+### Fixes
+
+* Fixed issue with suite precondition failure for `createData` with required entity.
+
+### GitHub Issues/Pull requests:
+
+  * [#547](https://github.com/magento/magento2-functional-testing-framework/pull/547) -- Fix invalid behavior of MAGENTO_BACKEND_BASE_URL
+  * [#742](https://github.com/magento/magento2-functional-testing-framework/pull/742) -- Fix Waits In MagentoPwaWebDriver
+  * [#750](https://github.com/magento/magento2-functional-testing-framework/pull/750) -- Docs: Outlining the difference between Allure severity levels
+  * [#683](https://github.com/magento/magento2-functional-testing-framework/pull/683) -- Docs: Renamed sample test name with the correct one
+  * [#691](https://github.com/magento/magento2-functional-testing-framework/pull/691) -- Docs: Branch name updates
+  * [#678](https://github.com/magento/magento2-functional-testing-framework/pull/678) -- Docs: Command added to modify the web server rewrites configuration
+  * [#745](https://github.com/magento/magento2-functional-testing-framework/pull/745) -- Docs: Remove invalid sample test name 
+
 3.0.0
 ---------
 

diff --git a/composer.json b/composer.json
@@ -2,7 +2,7 @@
     "name": "magento/magento2-functional-testing-framework",
     "description": "Magento2 Functional Testing Framework",
     "type": "library",
-    "version": "3.0.0",
+    "version": "3.1.0",
     "license": "AGPL-3.0",
     "keywords": ["magento", "automation", "functional", "testing"],
     "config": {

diff --git a/composer.lock b/composer.lock
diff --git a/dev/tests/unit/Magento/FunctionalTestFramework/Test/Handlers/TestObjectHandlerTest.php b/dev/tests/unit/Magento/FunctionalTestFramework/Test/Handlers/TestObjectHandlerTest.php
@@ -260,25 +260,6 @@ public function testGetAllTestObjectsWithInvalidExtends()
         $toh->getAllObjects();
     }
 
-    /**
-     * Function used to set mock for parser return and force init method to run between tests.
-     *
-     * @param array $data
-     * @throws \Exception
-     */
-    private function setMockParserOutput($data)
-    {
-        // clear test object handler value to inject parsed content
-        $property = new \ReflectionProperty(TestObjectHandler::class, 'testObjectHandler');
-        $property->setAccessible(true);
-        $property->setValue(null);
-
-        $mockDataParser = AspectMock::double(TestDataParser::class, ['readTestData' => $data])->make();
-        $instance = AspectMock::double(ObjectManager::class, ['create' => $mockDataParser])
-            ->make(); // bypass the private constructor
-        AspectMock::double(ObjectManagerFactory::class, ['getObjectManager' => $instance]);
-    }
-
     /**
      * Validate test object when ENABLE_PAUSE is set to true
      *
@@ -299,7 +280,7 @@ public function testGetTestObjectWhenEnablePause()
 
         $resolverMock = new MockModuleResolverBuilder();
         $resolverMock->setup();
-        $this->setMockParserOutput($mockData);
+        ObjectHandlerUtil::mockTestObjectHandlerWitData($mockData);
 
         // run object handler method
         $toh = TestObjectHandler::getInstance();
@@ -325,7 +306,7 @@ public function testGetTestObjectWhenEnablePause()
         $expectedFailedActionObject2 = new ActionObject(
             'pauseWhenFailed',
             'pause',
-            []
+            [ActionObject::PAUSE_ACTION_INTERNAL_ATTRIBUTE => true]
         );
 
         $expectedBeforeHookObject = new TestHookObject(

diff --git a/docs/test/actions.md b/docs/test/actions.md
@@ -1251,8 +1251,9 @@ Attribute|Type|Use|Description
 `stepKey`|string|required| A unique identifier of the action.
 
 #### Example
+
 ```xml
-<!-- Returns value of $grabInputName to the calling 
+<!-- Returns value of $grabInputName to the calling -->
 <return value="{$grabInputName}" stepKey="returnInputName"/>
 ```
 

diff --git a/src/Magento/FunctionalTestingFramework/Console/BaseGenerateCommand.php b/src/Magento/FunctionalTestingFramework/Console/BaseGenerateCommand.php
@@ -33,6 +33,7 @@ class BaseGenerateCommand extends Command
     const MFTF_NOTICES = "Placeholder text for MFTF notices\n";
     const CODECEPT_RUN = 'codecept:run';
     const CODECEPT_RUN_FUNCTIONAL = self::CODECEPT_RUN . ' functional ';
+    const CODECEPT_RUN_OPTION_NO_EXIT = ' --no-exit ';
 
     /**
      * Enable pause()

diff --git a/src/Magento/FunctionalTestingFramework/Console/RunManifestCommand.php b/src/Magento/FunctionalTestingFramework/Console/RunManifestCommand.php
@@ -81,12 +81,17 @@ protected function execute(InputInterface $input, OutputInterface $output): int
         // Delete the Codeception failed file just in case it exists from any previous test runs
         $this->deleteFailedFile();
 
-        foreach ($manifestFile as $manifestLine) {
-            if (empty($manifestLine)) {
+        for ($line = 0; $line < count($manifestFile); $line++) {
+            if (empty($manifestFile[$line])) {
                 continue;
             }
 
-            $this->runManifestLine($manifestLine, $output);
+            if ($line == count($manifestFile) - 1) {
+                $this->runManifestLine($manifestFile[$line], $output, true);
+            } else {
+                $this->runManifestLine($manifestFile[$line], $output);
+            }
+
             $this->aggregateFailed();
         }
 
@@ -103,18 +108,23 @@ protected function execute(InputInterface $input, OutputInterface $output): int
      *
      * @param string          $manifestLine
      * @param OutputInterface $output
+     * @param boolean         $exit
      * @return void
      * @throws \Exception
      *
      * @SuppressWarnings(PHPMD.UnusedLocalVariable) Need this because of the unused $type variable in the closure
      */
-    private function runManifestLine(string $manifestLine, OutputInterface $output)
+    private function runManifestLine($manifestLine, $output, $exit = false)
     {
         if (getenv('ENABLE_PAUSE') === 'true') {
             $codeceptionCommand = BaseGenerateCommand::CODECEPT_RUN_FUNCTIONAL
-                . '--verbose --steps --debug ' . $manifestLine;
+                . '--verbose --steps --debug ';
+            if (!$exit) {
+                $codeceptionCommand .= BaseGenerateCommand::CODECEPT_RUN_OPTION_NO_EXIT;
+            }
+            $codeceptionCommand .= $manifestLine;
             $input = new StringInput($codeceptionCommand);
-            $command = $this->getApplication()->find('codecept:run');
+            $command = $this->getApplication()->find(BaseGenerateCommand::CODECEPT_RUN);
             $subReturnCode = $command->run($input, $output);
         } else {
             $codeceptionCommand = realpath(PROJECT_ROOT . "/vendor/bin/codecept")

diff --git a/src/Magento/FunctionalTestingFramework/Console/RunTestCommand.php b/src/Magento/FunctionalTestingFramework/Console/RunTestCommand.php
@@ -135,8 +135,8 @@ private function runTests(array $tests, OutputInterface $output)
             TestGenerator::DEFAULT_DIR .
             DIRECTORY_SEPARATOR ;
 
-        foreach ($tests as $test) {
-            $testName = $test . 'Cest.php';
+        for ($i = 0; $i < count($tests); $i++) {
+            $testName = $tests[$i] . 'Cest.php';
             if (!realpath($testsDirectory . $testName)) {
                 throw new TestFrameworkException(
                     $testName . " is not available under " . $testsDirectory
@@ -145,6 +145,9 @@ private function runTests(array $tests, OutputInterface $output)
 
             if ($this->pauseEnabled()) {
                 $fullCommand = $codeceptionCommand . $testsDirectory . $testName . ' --verbose --steps --debug';
+                if ($i != count($tests) - 1) {
+                    $fullCommand .= self::CODECEPT_RUN_OPTION_NO_EXIT;
+                }
                 $this->returnCode = max($this->returnCode, $this->codeceptRunTest($fullCommand, $output));
             } else {
                 $fullCommand = $codeceptionCommand . $testsDirectory . $testName . ' --verbose --steps';
@@ -169,10 +172,18 @@ private function runTestsInSuite(array $suitesConfig, OutputInterface $output)
             $codeceptionCommand = realpath(PROJECT_ROOT . '/vendor/bin/codecept')
                 . ' run functional --verbose --steps ';
         }
+
+        $count = count($suitesConfig);
+        $index = 0;
         //for tests in suites, run them as a group to run before and after block
         foreach (array_keys($suitesConfig) as $suite) {
             $fullCommand = $codeceptionCommand . " -g {$suite}";
+
+            $index += 1;
             if ($this->pauseEnabled()) {
+                if ($index != $count) {
+                    $fullCommand .= self::CODECEPT_RUN_OPTION_NO_EXIT;
+                }
                 $this->returnCode = max($this->returnCode, $this->codeceptRunTest($fullCommand, $output));
             } else {
                 $this->returnCode = max($this->returnCode, $this->executeTestCommand($fullCommand, $output));

diff --git a/src/Magento/FunctionalTestingFramework/Console/RunTestFailedCommand.php b/src/Magento/FunctionalTestingFramework/Console/RunTestFailedCommand.php
@@ -116,13 +116,16 @@ protected function execute(InputInterface $input, OutputInterface $output): int
 
         $testManifestList = $this->readTestManifestFile();
         $returnCode = 0;
-        foreach ($testManifestList as $testCommand) {
+        for ($i = 0; $i < count($testManifestList); $i++) {
             if ($this->pauseEnabled()) {
-                $codeceptionCommand = self::CODECEPT_RUN_FUNCTIONAL . $testCommand . ' --debug';
+                $codeceptionCommand = self::CODECEPT_RUN_FUNCTIONAL . $testManifestList[$i] . ' --debug ';
+                if ($i != count($testManifestList) - 1) {
+                    $codeceptionCommand .= self::CODECEPT_RUN_OPTION_NO_EXIT;
+                }
                 $returnCode = $this->codeceptRunTest($codeceptionCommand, $output);
             } else {
                 $codeceptionCommand = realpath(PROJECT_ROOT . '/vendor/bin/codecept') . ' run functional ';
-                $codeceptionCommand .= $testCommand;
+                $codeceptionCommand .= $testManifestList[$i];
 
                 $process = new Process($codeceptionCommand);
                 $process->setWorkingDirectory(TESTS_BP);
@@ -142,6 +145,7 @@ function ($type, $buffer) use ($output) {
                 );
             }
         }
+
         foreach ($this->failedList as $test) {
             $this->writeFailedTestToFile($test, $this->testsFailedFile);
         }

diff --git a/src/Magento/FunctionalTestingFramework/Console/RunTestGroupCommand.php b/src/Magento/FunctionalTestingFramework/Console/RunTestGroupCommand.php
@@ -52,6 +52,7 @@ protected function configure()
      * @throws \Exception
      *
      * @SuppressWarnings(PHPMD.UnusedLocalVariable)
+     * @SuppressWarnings(PHPMD.CyclomaticComplexity)
      */
     protected function execute(InputInterface $input, OutputInterface $output): int
     {
@@ -102,10 +103,13 @@ protected function execute(InputInterface $input, OutputInterface $output): int
 
         $exitCode = -1;
         $returnCodes = [];
-        foreach ($groups as $group) {
-            $codeceptionCommandString = $commandString . " -g {$group}";
+        for ($i = 0; $i < count($groups); $i++) {
+            $codeceptionCommandString = $commandString . ' -g ' . $groups[$i];
 
             if ($this->pauseEnabled()) {
+                if ($i != count($groups) - 1) {
+                    $codeceptionCommandString .= self::CODECEPT_RUN_OPTION_NO_EXIT;
+                }
                 $returnCodes[] = $this->codeceptRunTest($codeceptionCommandString, $output);
             } else {
                 $process = new Process($codeceptionCommandString);

diff --git a/src/Magento/FunctionalTestingFramework/Module/MagentoWebDriver.php b/src/Magento/FunctionalTestingFramework/Module/MagentoWebDriver.php
@@ -54,7 +54,9 @@
 class MagentoWebDriver extends WebDriver
 {
     use AttachmentSupport;
-    use Pause;
+    use Pause {
+        pause as codeceptPause;
+    }
 
     const MAGENTO_CRON_INTERVAL = 60;
     const MAGENTO_CRON_COMMAND = 'cron:run';
@@ -843,7 +845,7 @@ public function _failed(TestInterface $test, $fail)
         if ($this->pngReport === null && $this->htmlReport === null) {
             $this->saveScreenshot();
             if (getenv('ENABLE_PAUSE') === 'true') {
-                $this->pause();
+                $this->pause(true);
             }
         }
 
@@ -1028,4 +1030,23 @@ public function switchToIFrame($locator = null)
             $this->webDriver->switchTo()->frame($els[0]);
         }
     }
+
+    /**
+     * Invoke Codeption pause()
+     *
+     * @param boolean $pauseOnFail
+     * @return void
+     */
+    public function pause($pauseOnFail = false)
+    {
+        if (!\Codeception\Util\Debug::isEnabled()) {
+            return;
+        }
+
+        if ($pauseOnFail) {
+            print(PHP_EOL . "Failure encountered. Pausing execution..." . PHP_EOL . PHP_EOL);
+        }
+
+        $this->codeceptPause();
+    }
 }
diff --git a/src/Magento/FunctionalTestingFramework/Test/Objects/ActionObject.php b/src/Magento/FunctionalTestingFramework/Test/Objects/ActionObject.php
@@ -77,6 +77,7 @@ class ActionObject
     const ACTION_TYPE_COMMENT = 'comment';
     const ACTION_TYPE_HELPER = 'helper';
     const INVISIBLE_STEP_ACTIONS = ['retrieveEntityField', 'getSecret'];
+    const PAUSE_ACTION_INTERNAL_ATTRIBUTE = 'pauseOnFail';
 
     /**
      * The unique identifier for the action

diff --git a/src/Magento/FunctionalTestingFramework/Test/Util/TestHookObjectExtractor.php b/src/Magento/FunctionalTestingFramework/Test/Util/TestHookObjectExtractor.php
@@ -66,7 +66,11 @@ public function createDefaultFailedHook($parentName)
     {
         $defaultSteps['saveScreenshot'] = new ActionObject("saveScreenshot", "saveScreenshot", []);
         if (getenv('ENABLE_PAUSE') === 'true') {
-            $defaultSteps['pauseWhenFailed'] = new ActionObject('pauseWhenFailed', 'pause', []);
+            $defaultSteps['pauseWhenFailed'] = new ActionObject(
+                'pauseWhenFailed',
+                'pause',
+                [ActionObject::PAUSE_ACTION_INTERNAL_ATTRIBUTE => true]
+            );
         }
 
         $hook = new TestHookObject(

diff --git a/src/Magento/FunctionalTestingFramework/Util/TestGenerator.php b/src/Magento/FunctionalTestingFramework/Util/TestGenerator.php
@@ -1441,6 +1441,16 @@ public function generateStepsPhp($actionObjects, $generationScope = TestGenerato
 
                     $testSteps .= $dateGenerateCode;
                     break;
+                case "pause":
+                    $pauseAttr =  $actionObject->getCustomActionAttributes(
+                        ActionObject::PAUSE_ACTION_INTERNAL_ATTRIBUTE
+                    );
+                    if ($pauseAttr) {
+                        $testSteps .= sprintf("\t\t$%s->%s(%s);", $actor, $actionObject->getType(), 'true');
+                    } else {
+                        $testSteps .= sprintf("\t\t$%s->%s();", $actor, $actionObject->getType());
+                    }
+                    break;
                 case "comment":
                     $input = $input === null ? strtr($value, ['$' => '\$', '{' => '\{', '}' => '\}']) : $input;
                 // Combining userInput from native XML comment and <comment/> action to fall-through 'default' case