From 90320b179ee6a64249c1e830e9f40d9c29304d06 Mon Sep 17 00:00:00 2001
From: Paul Siedler <paul.siedler@netresearch.de>
Date: Thu, 25 Apr 2019 15:16:40 +0200
Subject: [PATCH 01/81] #339: allow direct execution of the bin/magento
 executable instead of calling custom endpoint

---
 .../Module/MagentoWebDriver.php               | 178 ++++++++++++------
 1 file changed, 120 insertions(+), 58 deletions(-)

diff --git a/src/Magento/FunctionalTestingFramework/Module/MagentoWebDriver.php b/src/Magento/FunctionalTestingFramework/Module/MagentoWebDriver.php
index 93e6a5968..91c00c691 100644
--- a/src/Magento/FunctionalTestingFramework/Module/MagentoWebDriver.php
+++ b/src/Magento/FunctionalTestingFramework/Module/MagentoWebDriver.php
@@ -6,20 +6,21 @@
 
 namespace Magento\FunctionalTestingFramework\Module;
 
+use Codeception\Exception\ModuleConfigException;
+use Codeception\Exception\ModuleException;
 use Codeception\Module\WebDriver;
 use Codeception\Test\Descriptor;
 use Codeception\TestInterface;
-use Facebook\WebDriver\Interactions\WebDriverActions;
-use Codeception\Exception\ModuleConfigException;
-use Codeception\Exception\ModuleException;
 use Codeception\Util\Uri;
+use Facebook\WebDriver\Interactions\WebDriverActions;
 use Magento\FunctionalTestingFramework\DataGenerator\Handlers\CredentialStore;
 use Magento\FunctionalTestingFramework\DataGenerator\Persist\Curl\WebapiExecutor;
-use Magento\FunctionalTestingFramework\Util\Protocol\CurlTransport;
-use Magento\FunctionalTestingFramework\Util\Protocol\CurlInterface;
+use Magento\FunctionalTestingFramework\Exceptions\TestFrameworkException;
 use Magento\FunctionalTestingFramework\Util\ConfigSanitizerUtil;
+use Magento\FunctionalTestingFramework\Util\Protocol\CurlInterface;
+use Magento\FunctionalTestingFramework\Util\Protocol\CurlTransport;
+use Symfony\Component\Process\Process;
 use Yandex\Allure\Adapter\Support\AttachmentSupport;
-use Magento\FunctionalTestingFramework\Exceptions\TestFrameworkException;
 
 /**
  * MagentoWebDriver module provides common Magento web actions through Selenium WebDriver.
@@ -47,6 +48,7 @@ class MagentoWebDriver extends WebDriver
 
     /**
      * List of known magento loading masks by selector
+     *
      * @var array
      */
     public static $loadingMasksLocators = [
@@ -54,7 +56,7 @@ class MagentoWebDriver extends WebDriver
         '//div[contains(@class, "admin_data-grid-loading-mask")]',
         '//div[contains(@class, "admin__data-grid-loading-mask")]',
         '//div[contains(@class, "admin__form-loading-mask")]',
-        '//div[@data-role="spinner"]'
+        '//div[@data-role="spinner"]',
     ];
 
     /**
@@ -67,7 +69,7 @@ class MagentoWebDriver extends WebDriver
         'backend_name',
         'username',
         'password',
-        'browser'
+        'browser',
     ];
 
     /**
@@ -114,6 +116,7 @@ class MagentoWebDriver extends WebDriver
 
     /**
      * Sanitizes config, then initializes using parent.
+     *
      * @return void
      */
     public function _initialize()
@@ -137,6 +140,7 @@ public function _resetConfig()
 
     /**
      * Remap parent::_after, called in TestContextExtension
+     *
      * @param TestInterface $test
      * @return void
      */
@@ -147,9 +151,10 @@ public function _runAfter(TestInterface $test)
 
     /**
      * Override parent::_after to do nothing.
-     * @return void
+     *
      * @param TestInterface $test
      * @SuppressWarnings(PHPMD)
+     * @return void
      */
     public function _after(TestInterface $test)
     {
@@ -159,9 +164,9 @@ public function _after(TestInterface $test)
     /**
      * Returns URL of a host.
      *
-     * @api
      * @return mixed
      * @throws ModuleConfigException
+     * @api
      */
     public function _getUrl()
     {
@@ -171,6 +176,7 @@ public function _getUrl()
                 "Module connection failure. The URL for client can't bre retrieved"
             );
         }
+
         return $this->config['url'];
     }
 
@@ -178,8 +184,8 @@ public function _getUrl()
      * Uri of currently opened page.
      *
      * @return string
-     * @api
      * @throws ModuleException
+     * @api
      */
     public function _getCurrentUri()
     {
@@ -187,6 +193,7 @@ public function _getCurrentUri()
         if ($url == 'about:blank') {
             throw new ModuleException($this, 'Current url is blank, no page was opened');
         }
+
         return Uri::retrieveUri($url);
     }
 
@@ -243,6 +250,7 @@ public function grabFromCurrentUrl($regex = null)
         if (!isset($matches[1])) {
             $this->fail("Nothing to grab. A regex parameter with a capture group is required. Ex: '/(foo)(bar)/'");
         }
+
         return $matches[1];
     }
 
@@ -297,17 +305,17 @@ public function closeAdminNotification()
      * Search for and Select multiple options from a Magento Multi-Select drop down menu.
      * e.g. The drop down menu you use to assign Products to Categories.
      *
-     * @param string  $select
-     * @param array   $options
+     * @param string $select
+     * @param array $options
      * @param boolean $requireAction
-     * @throws \Exception
      * @return void
+     * @throws \Exception
      */
     public function searchAndMultiSelectOption($select, array $options, $requireAction = false)
     {
-        $selectDropdown     = $select . ' .action-select.admin__action-multiselect';
-        $selectSearchText   = $select
-            . ' .admin__action-multiselect-search-wrap>input[data-role="advanced-select-text"]';
+        $selectDropdown = $select . ' .action-select.admin__action-multiselect';
+        $selectSearchText = $select
+                            . ' .admin__action-multiselect-search-wrap>input[data-role="advanced-select-text"]';
         $selectSearchResult = $select . ' .admin__action-multiselect-label>span';
 
         $this->waitForPageLoad();
@@ -326,11 +334,11 @@ public function searchAndMultiSelectOption($select, array $options, $requireActi
     /**
      * Select multiple options from a drop down using a filter and text field to narrow results.
      *
-     * @param string   $selectSearchTextField
-     * @param string   $selectSearchResult
+     * @param string $selectSearchTextField
+     * @param string $selectSearchResult
      * @param string[] $options
-     * @throws \Exception
      * @return void
+     * @throws \Exception
      */
     public function selectMultipleOptions($selectSearchTextField, $selectSearchResult, array $options)
     {
@@ -367,8 +375,8 @@ public function waitForAjaxLoad($timeout = null)
      * Wait for all JavaScript to finish executing.
      *
      * @param integer $timeout
-     * @throws \Exception
      * @return void
+     * @throws \Exception
      */
     public function waitForPageLoad($timeout = null)
     {
@@ -383,8 +391,8 @@ public function waitForPageLoad($timeout = null)
      * Wait for all visible loading masks to disappear. Gets all elements by mask selector, then loops over them.
      *
      * @param integer $timeout
-     * @throws \Exception
      * @return void
+     * @throws \Exception
      */
     public function waitForLoadingMaskToDisappear($timeout = null)
     {
@@ -401,7 +409,7 @@ public function waitForLoadingMaskToDisappear($timeout = null)
     }
 
     /**
-     * @param float  $money
+     * @param float $money
      * @param string $locale
      * @return array
      */
@@ -412,6 +420,7 @@ public function formatMoney(float $money, $locale = 'en_US.UTF-8')
         $this->mResetLocale();
         $prefix = substr($money, 0, 1);
         $number = substr($money, 1);
+
         return ['prefix' => $prefix, 'number' => $number];
     }
 
@@ -424,12 +433,13 @@ public function formatMoney(float $money, $locale = 'en_US.UTF-8')
     public function parseFloat($floatString)
     {
         $floatString = str_replace(',', '', $floatString);
+
         return floatval($floatString);
     }
 
     /**
      * @param integer $category
-     * @param string  $locale
+     * @param string $locale
      * @return void
      */
     public function mSetLocale(int $category, $locale)
@@ -445,6 +455,7 @@ public function mSetLocale(int $category, $locale)
 
     /**
      * Reset Locale setting.
+     *
      * @return void
      */
     public function mResetLocale()
@@ -459,6 +470,7 @@ public function mResetLocale()
 
     /**
      * Scroll to the Top of the Page.
+     *
      * @return void
      */
     public function scrollToTopOfPage()
@@ -467,44 +479,28 @@ public function scrollToTopOfPage()
     }
 
     /**
-     * Takes given $command and executes it against exposed MTF CLI entry point. Returns response from server.
+     * Takes given $command and executes it against bin/magento or custom exposed entrypoint. Returns command output.
+     *
      * @param string $command
      * @param string $arguments
-     * @throws TestFrameworkException
      * @return string
+     * @throws TestFrameworkException
      */
     public function magentoCLI($command, $arguments = null)
     {
-        // Remove index.php if it's present in url
-        $baseUrl = rtrim(
-            str_replace('index.php', '', rtrim($this->config['url'], '/')),
-            '/'
-        );
-        $apiURL = $baseUrl . '/' . ltrim(getenv('MAGENTO_CLI_COMMAND_PATH'), '/');
-
-        $restExecutor = new WebapiExecutor();
-        $executor = new CurlTransport();
-        $executor->write(
-            $apiURL,
-            [
-                'token' => $restExecutor->getAuthToken(),
-                getenv('MAGENTO_CLI_COMMAND_PARAMETER') => $command,
-                'arguments' => $arguments
-            ],
-            CurlInterface::POST,
-            []
-        );
-        $response = $executor->read();
-        $restExecutor->close();
-        $executor->close();
-        return $response;
+        try {
+            return $this->shellExecMagentoCLI($command, $arguments);
+        } catch (\Exception $exception) {
+            return $this->curlExecMagentoCLI($command, $arguments);
+        }
     }
 
     /**
      * Runs DELETE request to delete a Magento entity against the url given.
+     *
      * @param string $url
-     * @throws TestFrameworkException
      * @return string
+     * @throws TestFrameworkException
      */
     public function deleteEntityByUrl($url)
     {
@@ -512,17 +508,18 @@ public function deleteEntityByUrl($url)
         $executor->write($url, [], CurlInterface::DELETE, []);
         $response = $executor->read();
         $executor->close();
+
         return $response;
     }
 
     /**
      * Conditional click for an area that should be visible
      *
-     * @param string  $selector
-     * @param string  $dependentSelector
+     * @param string $selector
+     * @param string $dependentSelector
      * @param boolean $visible
-     * @throws \Exception
      * @return void
+     * @throws \Exception
      */
     public function conditionalClick($selector, $dependentSelector, $visible)
     {
@@ -577,6 +574,7 @@ public function assertElementContainsAttribute($selector, $attribute, $value)
 
     /**
      * Sets current test to the given test, and resets test failure artifacts to null
+     *
      * @param TestInterface $test
      * @return void
      */
@@ -591,8 +589,9 @@ public function _before(TestInterface $test)
 
     /**
      * Override for codeception's default dragAndDrop to include offset options.
-     * @param string  $source
-     * @param string  $target
+     *
+     * @param string $source
+     * @param string $target
      * @param integer $xOffset
      * @param integer $yOffset
      * @return void
@@ -641,7 +640,7 @@ public function fillSecretField($field, $value)
      * following parent execution of test failure processing.
      *
      * @param TestInterface $test
-     * @param \Exception    $fail
+     * @param \Exception $fail
      * @return void
      */
     public function _failed(TestInterface $test, $fail)
@@ -666,6 +665,7 @@ public function _failed(TestInterface $test, $fail)
 
     /**
      * Function which saves a screenshot of the current stat of the browser
+     *
      * @return void
      */
     public function saveScreenshot()
@@ -685,8 +685,8 @@ public function saveScreenshot()
      * Go to a page and wait for ajax requests to finish
      *
      * @param string $page
-     * @throws \Exception
      * @return void
+     * @throws \Exception
      */
     public function amOnPage($page)
     {
@@ -698,8 +698,8 @@ public function amOnPage($page)
      * Turn Readiness check on or off
      *
      * @param boolean $check
-     * @throws \Exception
      * @return void
+     * @throws \Exception
      */
     public function skipReadinessCheck($check)
     {
@@ -742,6 +742,7 @@ private function getJsErrors()
                 $errors .= "\n" . $jsError;
             }
         }
+
         return $errors;
     }
 
@@ -754,4 +755,65 @@ public function dontSeeJsError()
     {
         $this->assertEmpty($this->jsErrors, $this->getJsErrors());
     }
+
+    /**
+     * Takes given $command and executes it against bin/magento executable. Returns stdout output from the command.
+     *
+     * @param string $command
+     * @param string $arguments
+     *
+     * @throws \RuntimeException
+     * @return string
+     */
+    private function shellExecMagentoCLI($command, $arguments): string
+    {
+        $php = PHP_BINDIR ? PHP_BINDIR . DIRECTORY_SEPARATOR. 'php' : 'php';
+        $binMagento = realpath(MAGENTO_BP . DIRECTORY_SEPARATOR . 'bin' . DIRECTORY_SEPARATOR . 'magento');
+        $command = $php . ' -f ' . $binMagento . ' ' . $command . ' ' . $arguments;
+        $process = new Process(escapeshellcmd($command), MAGENTO_BP);
+        $process->setIdleTimeout(60);
+        $process->setTimeout(0);
+        $exitCode = $process->run();
+        if ($exitCode !== 0) {
+            throw new \RuntimeException($process->getErrorOutput());
+        }
+
+        return $process->getOutput();
+    }
+
+    /**
+     * Takes given $command and executes it against exposed MTF CLI entry point. Returns response from server.
+     *
+     * @param string $command
+     * @param string $arguments
+     * @return string
+     * @throws TestFrameworkException
+     */
+    private function curlExecMagentoCLI($command, $arguments): string
+    {
+        // Remove index.php if it's present in url
+        $baseUrl = rtrim(
+            str_replace('index.php', '', rtrim($this->config['url'], '/')),
+            '/'
+        );
+        $apiURL = $baseUrl . '/' . ltrim(getenv('MAGENTO_CLI_COMMAND_PATH'), '/');
+
+        $restExecutor = new WebapiExecutor();
+        $executor = new CurlTransport();
+        $executor->write(
+            $apiURL,
+            [
+                'token' => $restExecutor->getAuthToken(),
+                getenv('MAGENTO_CLI_COMMAND_PARAMETER') => $command,
+                'arguments' => $arguments,
+            ],
+            CurlInterface::POST,
+            []
+        );
+        $response = $executor->read();
+        $restExecutor->close();
+        $executor->close();
+
+        return $response;
+    }
 }

From f1322810a1698c42fa35dd19b808741615d06f96 Mon Sep 17 00:00:00 2001
From: Paul Siedler <paul.siedler@netresearch.de>
Date: Tue, 30 Apr 2019 13:33:08 +0200
Subject: [PATCH 02/81] Fix phpcs issues

---
 .../Module/MagentoWebDriver.php               | 22 +++++++++----------
 1 file changed, 11 insertions(+), 11 deletions(-)

diff --git a/src/Magento/FunctionalTestingFramework/Module/MagentoWebDriver.php b/src/Magento/FunctionalTestingFramework/Module/MagentoWebDriver.php
index 91c00c691..1632757a6 100644
--- a/src/Magento/FunctionalTestingFramework/Module/MagentoWebDriver.php
+++ b/src/Magento/FunctionalTestingFramework/Module/MagentoWebDriver.php
@@ -305,8 +305,8 @@ public function closeAdminNotification()
      * Search for and Select multiple options from a Magento Multi-Select drop down menu.
      * e.g. The drop down menu you use to assign Products to Categories.
      *
-     * @param string $select
-     * @param array $options
+     * @param string  $select
+     * @param array   $options
      * @param boolean $requireAction
      * @return void
      * @throws \Exception
@@ -334,8 +334,8 @@ public function searchAndMultiSelectOption($select, array $options, $requireActi
     /**
      * Select multiple options from a drop down using a filter and text field to narrow results.
      *
-     * @param string $selectSearchTextField
-     * @param string $selectSearchResult
+     * @param string   $selectSearchTextField
+     * @param string   $selectSearchResult
      * @param string[] $options
      * @return void
      * @throws \Exception
@@ -409,7 +409,7 @@ public function waitForLoadingMaskToDisappear($timeout = null)
     }
 
     /**
-     * @param float $money
+     * @param float  $money
      * @param string $locale
      * @return array
      */
@@ -439,7 +439,7 @@ public function parseFloat($floatString)
 
     /**
      * @param integer $category
-     * @param string $locale
+     * @param string  $locale
      * @return void
      */
     public function mSetLocale(int $category, $locale)
@@ -515,8 +515,8 @@ public function deleteEntityByUrl($url)
     /**
      * Conditional click for an area that should be visible
      *
-     * @param string $selector
-     * @param string $dependentSelector
+     * @param string  $selector
+     * @param string  $dependentSelector
      * @param boolean $visible
      * @return void
      * @throws \Exception
@@ -590,8 +590,8 @@ public function _before(TestInterface $test)
     /**
      * Override for codeception's default dragAndDrop to include offset options.
      *
-     * @param string $source
-     * @param string $target
+     * @param string  $source
+     * @param string  $target
      * @param integer $xOffset
      * @param integer $yOffset
      * @return void
@@ -640,7 +640,7 @@ public function fillSecretField($field, $value)
      * following parent execution of test failure processing.
      *
      * @param TestInterface $test
-     * @param \Exception $fail
+     * @param \Exception    $fail
      * @return void
      */
     public function _failed(TestInterface $test, $fail)

From 669ca5330fdb17dc6f1541c742f6e75eba84f9da Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jens=20K=C3=B6cke?= <jens@koecke.net>
Date: Wed, 28 Aug 2019 13:39:49 +0200
Subject: [PATCH 03/81] Add possibility to include multiple non primitive types
 in an array

If an array is defined for an entity it is currently not possible to
include different types of non primitive types in the same array.
---
 .../Persist/OperationDataArrayResolver.php    | 22 ++++++++++++++++++-
 .../Util/OperationElementExtractor.php        | 13 ++++++++---
 2 files changed, 31 insertions(+), 4 deletions(-)

diff --git a/src/Magento/FunctionalTestingFramework/DataGenerator/Persist/OperationDataArrayResolver.php b/src/Magento/FunctionalTestingFramework/DataGenerator/Persist/OperationDataArrayResolver.php
index 6d1ebc3fc..bd641c2ff 100644
--- a/src/Magento/FunctionalTestingFramework/DataGenerator/Persist/OperationDataArrayResolver.php
+++ b/src/Magento/FunctionalTestingFramework/DataGenerator/Persist/OperationDataArrayResolver.php
@@ -109,6 +109,26 @@ public function resolveOperationDataArray($entityObject, $operationMetadata, $op
                     $operationElementType,
                     $operationDataArray
                 );
+            } else if (is_array($operationElementType)) {
+                foreach ($operationElementType as $currentElementType) {
+                    if (in_array($currentElementType, self::PRIMITIVE_TYPES)) {
+                        $this->resolvePrimitiveReferenceElement(
+                            $entityObject,
+                            $operationElement,
+                            $currentElementType,
+                            $operationDataArray
+                        );
+                    } else {
+                        $this->resolveNonPrimitiveReferenceElement(
+                            $entityObject,
+                            $operation,
+                            $fromArray,
+                            $currentElementType,
+                            $operationElement,
+                            $operationDataArray
+                        );
+                    }
+                }
             } else {
                 $this->resolveNonPrimitiveReferenceElement(
                     $entityObject,
@@ -236,7 +256,7 @@ private function resolveNonPrimitiveElement($entityName, $operationElement, $ope
         $linkedEntityObj = $this->resolveLinkedEntityObject($entityName);
 
         // in array case
-        if (!empty($operationElement->getNestedOperationElement($operationElement->getValue()))
+        if (!is_array($operationElement->getValue()) && !empty($operationElement->getNestedOperationElement($operationElement->getValue()))
             && $operationElement->getType() == OperationDefinitionObjectHandler::ENTITY_OPERATION_ARRAY
         ) {
             $operationSubArray = $this->resolveOperationDataArray(
diff --git a/src/Magento/FunctionalTestingFramework/DataGenerator/Util/OperationElementExtractor.php b/src/Magento/FunctionalTestingFramework/DataGenerator/Util/OperationElementExtractor.php
index 35d188f5b..7467e312b 100644
--- a/src/Magento/FunctionalTestingFramework/DataGenerator/Util/OperationElementExtractor.php
+++ b/src/Magento/FunctionalTestingFramework/DataGenerator/Util/OperationElementExtractor.php
@@ -112,9 +112,16 @@ private function extractOperationField(&$operationElements, $operationFieldArray
     private function extractOperationArray(&$operationArrayData, $operationArrayArray)
     {
         foreach ($operationArrayArray as $operationFieldType) {
-            $operationElementValue =
-                $operationFieldType[OperationDefinitionObjectHandler::ENTITY_OPERATION_ARRAY_VALUE][0]
-                [OperationElementExtractor::OPERATION_OBJECT_ARRAY_VALUE] ?? null;
+            $operationElementValue = [];
+            if (isset($operationFieldType[OperationDefinitionObjectHandler::ENTITY_OPERATION_ARRAY_VALUE])) {
+                foreach ($operationFieldType[OperationDefinitionObjectHandler::ENTITY_OPERATION_ARRAY_VALUE] as $operationFieldValue) {
+                    $operationElementValue[] = $operationFieldValue[OperationElementExtractor::OPERATION_OBJECT_ARRAY_VALUE] ?? null;
+                }
+            }
+
+            if (count($operationElementValue) === 1) {
+                $operationElementValue = array_pop($operationElementValue);
+            }
 
             $nestedOperationElements = [];
             if (array_key_exists(OperationElementExtractor::OPERATION_OBJECT_OBJ_NAME, $operationFieldType)) {

From 938a6aba218504cbcb8a712753afa95286652194 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jens=20K=C3=B6cke?= <jens@koecke.net>
Date: Wed, 28 Aug 2019 14:02:26 +0200
Subject: [PATCH 04/81] Correct formatting to adhere to psr-2 coding standards

---
 .../DataGenerator/Persist/OperationDataArrayResolver.php     | 5 +++--
 .../DataGenerator/Util/OperationElementExtractor.php         | 4 +++-
 2 files changed, 6 insertions(+), 3 deletions(-)

diff --git a/src/Magento/FunctionalTestingFramework/DataGenerator/Persist/OperationDataArrayResolver.php b/src/Magento/FunctionalTestingFramework/DataGenerator/Persist/OperationDataArrayResolver.php
index bd641c2ff..c8e582728 100644
--- a/src/Magento/FunctionalTestingFramework/DataGenerator/Persist/OperationDataArrayResolver.php
+++ b/src/Magento/FunctionalTestingFramework/DataGenerator/Persist/OperationDataArrayResolver.php
@@ -109,7 +109,7 @@ public function resolveOperationDataArray($entityObject, $operationMetadata, $op
                     $operationElementType,
                     $operationDataArray
                 );
-            } else if (is_array($operationElementType)) {
+            } elseif (is_array($operationElementType)) {
                 foreach ($operationElementType as $currentElementType) {
                     if (in_array($currentElementType, self::PRIMITIVE_TYPES)) {
                         $this->resolvePrimitiveReferenceElement(
@@ -256,7 +256,8 @@ private function resolveNonPrimitiveElement($entityName, $operationElement, $ope
         $linkedEntityObj = $this->resolveLinkedEntityObject($entityName);
 
         // in array case
-        if (!is_array($operationElement->getValue()) && !empty($operationElement->getNestedOperationElement($operationElement->getValue()))
+        if (!is_array($operationElement->getValue())
+            && !empty($operationElement->getNestedOperationElement($operationElement->getValue()))
             && $operationElement->getType() == OperationDefinitionObjectHandler::ENTITY_OPERATION_ARRAY
         ) {
             $operationSubArray = $this->resolveOperationDataArray(
diff --git a/src/Magento/FunctionalTestingFramework/DataGenerator/Util/OperationElementExtractor.php b/src/Magento/FunctionalTestingFramework/DataGenerator/Util/OperationElementExtractor.php
index 7467e312b..8c127046a 100644
--- a/src/Magento/FunctionalTestingFramework/DataGenerator/Util/OperationElementExtractor.php
+++ b/src/Magento/FunctionalTestingFramework/DataGenerator/Util/OperationElementExtractor.php
@@ -114,7 +114,9 @@ private function extractOperationArray(&$operationArrayData, $operationArrayArra
         foreach ($operationArrayArray as $operationFieldType) {
             $operationElementValue = [];
             if (isset($operationFieldType[OperationDefinitionObjectHandler::ENTITY_OPERATION_ARRAY_VALUE])) {
-                foreach ($operationFieldType[OperationDefinitionObjectHandler::ENTITY_OPERATION_ARRAY_VALUE] as $operationFieldValue) {
+                foreach ($operationFieldType[OperationDefinitionObjectHandler::ENTITY_OPERATION_ARRAY_VALUE]
+                         as $operationFieldValue
+                ) {
                     $operationElementValue[] = $operationFieldValue[OperationElementExtractor::OPERATION_OBJECT_ARRAY_VALUE] ?? null;
                 }
             }

From b3f9db9f55e07f99ae7be1f1021a759e8be75308 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jens=20K=C3=B6cke?= <jens@koecke.net>
Date: Wed, 28 Aug 2019 14:12:51 +0200
Subject: [PATCH 05/81] Correct formatting to adhere to psr-2 coding standards

---
 .../DataGenerator/Util/OperationElementExtractor.php      | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/Magento/FunctionalTestingFramework/DataGenerator/Util/OperationElementExtractor.php b/src/Magento/FunctionalTestingFramework/DataGenerator/Util/OperationElementExtractor.php
index 8c127046a..c86e27972 100644
--- a/src/Magento/FunctionalTestingFramework/DataGenerator/Util/OperationElementExtractor.php
+++ b/src/Magento/FunctionalTestingFramework/DataGenerator/Util/OperationElementExtractor.php
@@ -114,10 +114,10 @@ private function extractOperationArray(&$operationArrayData, $operationArrayArra
         foreach ($operationArrayArray as $operationFieldType) {
             $operationElementValue = [];
             if (isset($operationFieldType[OperationDefinitionObjectHandler::ENTITY_OPERATION_ARRAY_VALUE])) {
-                foreach ($operationFieldType[OperationDefinitionObjectHandler::ENTITY_OPERATION_ARRAY_VALUE]
-                         as $operationFieldValue
-                ) {
-                    $operationElementValue[] = $operationFieldValue[OperationElementExtractor::OPERATION_OBJECT_ARRAY_VALUE] ?? null;
+                foreach ($operationFieldType[OperationDefinitionObjectHandler::ENTITY_OPERATION_ARRAY_VALUE] as
+                    $operationFieldValue) {
+                    $operationElementValue[] =
+                        $operationFieldValue[OperationElementExtractor::OPERATION_OBJECT_ARRAY_VALUE] ?? null;
                 }
             }
 

From 070654daed9144ad7aaa38a981be8d7dddbad8a0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jens=20K=C3=B6cke?= <jens@koecke.net>
Date: Tue, 1 Oct 2019 15:06:01 +0200
Subject: [PATCH 06/81] Add unit test for diveres data array

---
 .../OperationDataArrayResolverTest.php        | 118 ++++++++++++++++++
 1 file changed, 118 insertions(+)

diff --git a/dev/tests/unit/Magento/FunctionalTestFramework/DataGenerator/Persist/OperationDataArrayResolverTest.php b/dev/tests/unit/Magento/FunctionalTestFramework/DataGenerator/Persist/OperationDataArrayResolverTest.php
index 9e425e020..60cf4bf0f 100644
--- a/dev/tests/unit/Magento/FunctionalTestFramework/DataGenerator/Persist/OperationDataArrayResolverTest.php
+++ b/dev/tests/unit/Magento/FunctionalTestFramework/DataGenerator/Persist/OperationDataArrayResolverTest.php
@@ -9,6 +9,7 @@
 use Magento\FunctionalTestingFramework\DataGenerator\Handlers\DataObjectHandler;
 use Magento\FunctionalTestingFramework\DataGenerator\Handlers\OperationDefinitionObjectHandler;
 use Magento\FunctionalTestingFramework\DataGenerator\Persist\OperationDataArrayResolver;
+use Magento\FunctionalTestingFramework\Util\Iterator\AbstractIterator;
 use Magento\FunctionalTestingFramework\Util\MagentoTestCase;
 use tests\unit\Util\EntityDataObjectBuilder;
 use tests\unit\Util\OperationDefinitionBuilder;
@@ -355,6 +356,123 @@ public function testNestedMetadataArrayOfValue()
         $this->assertEquals(self::NESTED_METADATA_ARRAY_RESULT, $result);
     }
 
+    public function testNestedMetadataArrayOfDiverseObjects() {
+
+        $entityDataObjBuilder = new EntityDataObjectBuilder();
+        $parentDataObject = $entityDataObjBuilder
+            ->withName("parentObject")
+            ->withType("parentType")
+            ->withLinkedEntities(['child1Object' => 'childType1','child2Object' => 'childType2'])
+            ->build();
+
+
+        $child1DataObject = $entityDataObjBuilder
+            ->withName('child1Object')
+            ->withType('childType1')
+            ->withDataFields(['city' => 'Testcity','zip' => 12345])
+            ->build();
+
+        $child2DataObject = $entityDataObjBuilder
+            ->withName('child2Object')
+            ->withType('childType2')
+            ->withDataFields(['city' => 'Testcity 2','zip' => 54321,'state' => 'Teststate'])
+            ->build();
+
+        $mockDOHInstance = AspectMock::double(DataObjectHandler::class,
+            [
+                'getObject' => function ($name) use ($child1DataObject, $child2DataObject) {
+            switch ($name) {
+                case 'child1Object':
+                    return $child1DataObject;
+                case 'child2Object':
+                    return $child2DataObject;
+            }
+                }
+            ])->make();
+        AspectMock::double(DataObjectHandler::class,[
+            'getInstance' => $mockDOHInstance
+        ]);
+
+        $operationDefinitionBuilder = new OperationDefinitionBuilder();
+        $child1OperationDefinition = $operationDefinitionBuilder
+            ->withName('createchildType1')
+            ->withOperation('create')
+            ->withType('childType1')
+            ->withMetadata([
+                'city' => 'string',
+                'zip' => 'integer'
+            ])->build();
+
+        $child2OperationDefinition = $operationDefinitionBuilder
+            ->withName('createchildType2')
+            ->withOperation('create')
+            ->withType('childType2')
+            ->withMetadata([
+                'city' => 'string',
+                'zip' => 'integer',
+                'state' => 'string'
+            ])->build();
+
+        $mockODOHInstance = AspectMock::double(
+            OperationDefinitionObjectHandler::class,
+            [
+                'getObject' => function($name) use ($child1OperationDefinition, $child2OperationDefinition) {
+                    switch ($name) {
+                        case 'createchildType1':
+                            return $child1OperationDefinition;
+                        case 'createchildType2':
+                            return $child2OperationDefinition;
+                    }
+                }
+            ]
+            )->make();
+        AspectMock::double(OperationDefinitionObjectHandler::class,
+            [
+                'getInstance' => $mockODOHInstance
+            ]);
+
+        $arrayObElementBuilder = new OperationElementBuilder();
+        $arrayElement = $arrayObElementBuilder
+            ->withKey('address')
+            ->withType(['childType1','childType2'])
+            ->withFields([])
+            ->withElementType(OperationDefinitionObjectHandler::ENTITY_OPERATION_ARRAY)
+            //->withNestedElements(['childType1' => $child1Element, 'childType2' => $child2Element])
+            ->build();
+
+        $parentOpElementBuilder = new OperationElementBuilder();
+        $parentElement = $parentOpElementBuilder
+            ->withKey('parentType')
+            ->withType('parentType')
+            ->addElements(['address' => $arrayElement])
+            ->build();
+
+        $operationResolver = new OperationDataArrayResolver();
+        $result = $operationResolver->resolveOperationDataArray($parentDataObject,[$parentElement],'create',false);
+
+        $expectedResult = [
+            'parentType' => [
+                'address' => [
+                    [
+                        'city' => 'Testcity',
+                        'zip' => '12345'
+                    ],
+                    [
+                        'city' => 'Testcity 2',
+                        'zip' => '54321',
+                        'state' => 'Teststate'
+                    ]
+                ],
+                'name' => 'Hopper',
+                'gpa' => '3.5678',
+                'phone' => '5555555',
+                'isPrimary' => '1'
+            ]
+        ];
+
+        $this->assertEquals($expectedResult, $result);
+    }
+
     /**
      * After class functionality
      * @return void

From 491802cf11310732a4046f2d52ea63bf31873fc9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jens=20K=C3=B6cke?= <jens@koecke.net>
Date: Tue, 1 Oct 2019 15:19:13 +0200
Subject: [PATCH 07/81] Adhere to code style guide

---
 .../OperationDataArrayResolverTest.php        | 36 ++++++++++---------
 1 file changed, 20 insertions(+), 16 deletions(-)

diff --git a/dev/tests/unit/Magento/FunctionalTestFramework/DataGenerator/Persist/OperationDataArrayResolverTest.php b/dev/tests/unit/Magento/FunctionalTestFramework/DataGenerator/Persist/OperationDataArrayResolverTest.php
index 60cf4bf0f..fb6dcd865 100644
--- a/dev/tests/unit/Magento/FunctionalTestFramework/DataGenerator/Persist/OperationDataArrayResolverTest.php
+++ b/dev/tests/unit/Magento/FunctionalTestFramework/DataGenerator/Persist/OperationDataArrayResolverTest.php
@@ -356,7 +356,8 @@ public function testNestedMetadataArrayOfValue()
         $this->assertEquals(self::NESTED_METADATA_ARRAY_RESULT, $result);
     }
 
-    public function testNestedMetadataArrayOfDiverseObjects() {
+    public function testNestedMetadataArrayOfDiverseObjects()
+    {
 
         $entityDataObjBuilder = new EntityDataObjectBuilder();
         $parentDataObject = $entityDataObjBuilder
@@ -365,7 +366,6 @@ public function testNestedMetadataArrayOfDiverseObjects() {
             ->withLinkedEntities(['child1Object' => 'childType1','child2Object' => 'childType2'])
             ->build();
 
-
         $child1DataObject = $entityDataObjBuilder
             ->withName('child1Object')
             ->withType('childType1')
@@ -378,18 +378,20 @@ public function testNestedMetadataArrayOfDiverseObjects() {
             ->withDataFields(['city' => 'Testcity 2','zip' => 54321,'state' => 'Teststate'])
             ->build();
 
-        $mockDOHInstance = AspectMock::double(DataObjectHandler::class,
+        $mockDOHInstance = AspectMock::double(
+            DataObjectHandler::class,
             [
                 'getObject' => function ($name) use ($child1DataObject, $child2DataObject) {
-            switch ($name) {
-                case 'child1Object':
-                    return $child1DataObject;
-                case 'child2Object':
-                    return $child2DataObject;
-            }
+                    switch ($name) {
+                        case 'child1Object':
+                            return $child1DataObject;
+                        case 'child2Object':
+                            return $child2DataObject;
+                    }
                 }
-            ])->make();
-        AspectMock::double(DataObjectHandler::class,[
+            ]
+        )->make();
+        AspectMock::double(DataObjectHandler::class, [
             'getInstance' => $mockDOHInstance
         ]);
 
@@ -416,7 +418,7 @@ public function testNestedMetadataArrayOfDiverseObjects() {
         $mockODOHInstance = AspectMock::double(
             OperationDefinitionObjectHandler::class,
             [
-                'getObject' => function($name) use ($child1OperationDefinition, $child2OperationDefinition) {
+                'getObject' => function ($name) use ($child1OperationDefinition, $child2OperationDefinition) {
                     switch ($name) {
                         case 'createchildType1':
                             return $child1OperationDefinition;
@@ -425,11 +427,13 @@ public function testNestedMetadataArrayOfDiverseObjects() {
                     }
                 }
             ]
-            )->make();
-        AspectMock::double(OperationDefinitionObjectHandler::class,
+        )->make();
+        AspectMock::double(
+            OperationDefinitionObjectHandler::class,
             [
                 'getInstance' => $mockODOHInstance
-            ]);
+            ]
+        );
 
         $arrayObElementBuilder = new OperationElementBuilder();
         $arrayElement = $arrayObElementBuilder
@@ -448,7 +452,7 @@ public function testNestedMetadataArrayOfDiverseObjects() {
             ->build();
 
         $operationResolver = new OperationDataArrayResolver();
-        $result = $operationResolver->resolveOperationDataArray($parentDataObject,[$parentElement],'create',false);
+        $result = $operationResolver->resolveOperationDataArray($parentDataObject, [$parentElement], 'create', false);
 
         $expectedResult = [
             'parentType' => [

From 36a243a40b378d6d81022b41cb19cfefb74a0100 Mon Sep 17 00:00:00 2001
From: Kevin Kozan <kkozan@adobe.com>
Date: Wed, 16 Oct 2019 14:08:09 -0500
Subject: [PATCH 08/81] MQE-1644: Add ability to see JS log in Allure

- Initial logging addition.
---
 .../Extension/ErrorLogger.php                 | 36 ++++++++++++++-----
 .../Extension/TestContextExtension.php        |  8 +++++
 2 files changed, 36 insertions(+), 8 deletions(-)

diff --git a/src/Magento/FunctionalTestingFramework/Extension/ErrorLogger.php b/src/Magento/FunctionalTestingFramework/Extension/ErrorLogger.php
index b0621df1b..07a6d2779 100644
--- a/src/Magento/FunctionalTestingFramework/Extension/ErrorLogger.php
+++ b/src/Magento/FunctionalTestingFramework/Extension/ErrorLogger.php
@@ -12,6 +12,9 @@
  */
 class ErrorLogger
 {
+    const LOG_TYPE_BROWSER = "browser";
+    const ERROR_TYPE_JAVASCRIPT = "javascript";
+
     /**
      * Error Logger Instance
      * @var ErrorLogger
@@ -49,16 +52,33 @@ private function __construct()
     public function logErrors($module, $stepEvent)
     {
         //Types available should be "server", "browser", "driver". Only care about browser at the moment.
-        if (in_array("browser", $module->webDriver->manage()->getAvailableLogTypes())) {
-            $browserLogEntries = $module->webDriver->manage()->getLog("browser");
-            foreach ($browserLogEntries as $entry) {
-                if (array_key_exists("source", $entry) && $entry["source"] === "javascript") {
-                    $this->logError("javascript", $stepEvent, $entry);
-                    //Set javascript error in MagentoWebDriver internal array
-                    $module->setJsError("ERROR({$entry["level"]}) - " . $entry["message"]);
-                }
+        if (in_array(self::LOG_TYPE_BROWSER, $module->webDriver->manage()->getAvailableLogTypes())) {
+            $browserLogEntries = $module->webDriver->manage()->getLog(self::LOG_TYPE_BROWSER);
+            $jsErrors = $this->getLogsOfType($browserLogEntries, self::ERROR_TYPE_JAVASCRIPT);
+            foreach ($jsErrors as $entry) {
+                $this->logError(self::ERROR_TYPE_JAVASCRIPT, $stepEvent, $entry);
+                //Set javascript error in MagentoWebDriver internal array
+                $module->setJsError("ERROR({$entry["level"]}) - " . $entry["message"]);
+            }
+        }
+    }
+
+    /**
+     * Loops through given logs and returns entries of the given type.
+     *
+     * @param array $log
+     * @param string $type
+     * @return array
+     */
+    public function getLogsOfType($log, $type)
+    {
+        $errors = [];
+        foreach ($log as $entry) {
+            if (array_key_exists("source", $entry) && $entry["source"] === $type) {
+                $errors[] = $entry;
             }
         }
+        return $errors;
     }
 
     /**
diff --git a/src/Magento/FunctionalTestingFramework/Extension/TestContextExtension.php b/src/Magento/FunctionalTestingFramework/Extension/TestContextExtension.php
index 467074097..93faf9e82 100644
--- a/src/Magento/FunctionalTestingFramework/Extension/TestContextExtension.php
+++ b/src/Magento/FunctionalTestingFramework/Extension/TestContextExtension.php
@@ -7,6 +7,7 @@
 namespace Magento\FunctionalTestingFramework\Extension;
 
 use Codeception\Events;
+use Magento\FunctionalTestingFramework\Allure\AllureHelper;
 use Magento\FunctionalTestingFramework\DataGenerator\Handlers\PersistedObjectHandler;
 
 /**
@@ -173,6 +174,13 @@ public function beforeStep(\Codeception\Event\StepEvent $e)
      */
     public function afterStep(\Codeception\Event\StepEvent $e)
     {
+        if (getenv('ENABLE_JS_LOG')) {
+            $browserLogEntries = $this->getDriver()->webDriver->manage()->getLog("browser");
+            $jsErrors = ErrorLogger::getInstance()->getLogsOfType($browserLogEntries, ErrorLogger::ERROR_TYPE_JAVASCRIPT);
+            if (!empty($jsErrors)) {
+                AllureHelper::addAttachmentToCurrentStep($jsErrors);
+            }
+        }
         ErrorLogger::getInstance()->logErrors($this->getDriver(), $e);
     }
 

From 997a9ef190f696903d1ecb3206fd31185f5dc886 Mon Sep 17 00:00:00 2001
From: Kevin Kozan <kkozan@adobe.com>
Date: Thu, 17 Oct 2019 09:40:22 -0500
Subject: [PATCH 09/81] MQE-1644: Add ability to see JS log in Allure

- Addec blacklist
- Renamed variables
---
 etc/config/.env.example                       |  5 +++++
 .../Extension/ErrorLogger.php                 | 20 ++++++++++++++++++-
 .../Extension/TestContextExtension.php        | 13 +++++++-----
 3 files changed, 32 insertions(+), 6 deletions(-)

diff --git a/etc/config/.env.example b/etc/config/.env.example
index f25cb3de7..bb1d1ad63 100644
--- a/etc/config/.env.example
+++ b/etc/config/.env.example
@@ -54,4 +54,9 @@ MODULE_WHITELIST=Magento_Framework,ConfigurableProductWishlist,ConfigurableProdu
 
 #*** Default timeout for wait actions
 #WAIT_TIMEOUT=10
+
+#*** Uncomment and set to enable browser log entries on actions in Allure. Blacklist is used to filter logs of a specific "source"
+#ENABLE_BROWSER_LOG=true
+#BROWSER_LOG_BLACKLIST=other
+
 #*** End of .env ***#
diff --git a/src/Magento/FunctionalTestingFramework/Extension/ErrorLogger.php b/src/Magento/FunctionalTestingFramework/Extension/ErrorLogger.php
index 07a6d2779..b76a9c1f4 100644
--- a/src/Magento/FunctionalTestingFramework/Extension/ErrorLogger.php
+++ b/src/Magento/FunctionalTestingFramework/Extension/ErrorLogger.php
@@ -64,7 +64,7 @@ public function logErrors($module, $stepEvent)
     }
 
     /**
-     * Loops through given logs and returns entries of the given type.
+     * Loops through given log and returns entries of the given type.
      *
      * @param array $log
      * @param string $type
@@ -81,6 +81,24 @@ public function getLogsOfType($log, $type)
         return $errors;
     }
 
+    /**
+     * Loops through given log and filters entries of the given type.
+     *
+     * @param array $log
+     * @param string $type
+     * @return array
+     */
+    public function filterLogsOfType($log, $type)
+    {
+        $errors = [];
+        foreach ($log as $entry) {
+            if (array_key_exists("source", $entry) && $entry["source"] !== $type) {
+                $errors[] = $entry;
+            }
+        }
+        return $errors;
+    }
+
     /**
      * Logs errors to console/report.
      * @param string                       $type
diff --git a/src/Magento/FunctionalTestingFramework/Extension/TestContextExtension.php b/src/Magento/FunctionalTestingFramework/Extension/TestContextExtension.php
index 93faf9e82..639d65133 100644
--- a/src/Magento/FunctionalTestingFramework/Extension/TestContextExtension.php
+++ b/src/Magento/FunctionalTestingFramework/Extension/TestContextExtension.php
@@ -174,11 +174,14 @@ public function beforeStep(\Codeception\Event\StepEvent $e)
      */
     public function afterStep(\Codeception\Event\StepEvent $e)
     {
-        if (getenv('ENABLE_JS_LOG')) {
-            $browserLogEntries = $this->getDriver()->webDriver->manage()->getLog("browser");
-            $jsErrors = ErrorLogger::getInstance()->getLogsOfType($browserLogEntries, ErrorLogger::ERROR_TYPE_JAVASCRIPT);
-            if (!empty($jsErrors)) {
-                AllureHelper::addAttachmentToCurrentStep($jsErrors);
+        if (getenv('ENABLE_BROWSER_LOG')) {
+            $browserLog = $this->getDriver()->webDriver->manage()->getLog("browser");
+            foreach (explode(',', getenv('BROWSER_LOG_BLACKLIST')) as $source) {
+                $browserLog = ErrorLogger::getInstance()->filterLogsOfType($browserLog, $source);
+            }
+
+            if (!empty($browserLog)) {
+                AllureHelper::addAttachmentToCurrentStep(json_encode($browserLog, JSON_PRETTY_PRINT), "Browser Log");
             }
         }
         ErrorLogger::getInstance()->logErrors($this->getDriver(), $e);

From dc0bf75cc902212b7eb1572c6757f55134881995 Mon Sep 17 00:00:00 2001
From: Kevin Kozan <kkozan@adobe.com>
Date: Thu, 17 Oct 2019 10:39:34 -0500
Subject: [PATCH 10/81] MQE-1644: Add ability to see JS log in Allure

- Refactor ErrorLogger into BrowserLogUtil, no longer a singleton
---
 .../{ErrorLogger.php => BrowserLogUtil.php}   | 43 ++++---------------
 .../Extension/TestContextExtension.php        |  4 +-
 2 files changed, 10 insertions(+), 37 deletions(-)
 rename src/Magento/FunctionalTestingFramework/Extension/{ErrorLogger.php => BrowserLogUtil.php} (71%)

diff --git a/src/Magento/FunctionalTestingFramework/Extension/ErrorLogger.php b/src/Magento/FunctionalTestingFramework/Extension/BrowserLogUtil.php
similarity index 71%
rename from src/Magento/FunctionalTestingFramework/Extension/ErrorLogger.php
rename to src/Magento/FunctionalTestingFramework/Extension/BrowserLogUtil.php
index b76a9c1f4..330c21ebf 100644
--- a/src/Magento/FunctionalTestingFramework/Extension/ErrorLogger.php
+++ b/src/Magento/FunctionalTestingFramework/Extension/BrowserLogUtil.php
@@ -7,41 +7,14 @@
 namespace Magento\FunctionalTestingFramework\Extension;
 
 /**
- * Class ErrorLogger
+ * Class BrowserLogUtil
  * @package Magento\FunctionalTestingFramework\Extension
  */
-class ErrorLogger
+class BrowserLogUtil
 {
     const LOG_TYPE_BROWSER = "browser";
     const ERROR_TYPE_JAVASCRIPT = "javascript";
 
-    /**
-     * Error Logger Instance
-     * @var ErrorLogger
-     */
-    private static $errorLogger;
-
-    /**
-     * Singleton method to return ErrorLogger.
-     * @return ErrorLogger
-     */
-    public static function getInstance()
-    {
-        if (!self::$errorLogger) {
-            self::$errorLogger = new ErrorLogger();
-        }
-
-        return self::$errorLogger;
-    }
-
-    /**
-     * ErrorLogger constructor.
-     */
-    private function __construct()
-    {
-        // private constructor
-    }
-
     /**
      * Loops through stepEvent for browser log entries
      *
@@ -49,14 +22,14 @@ private function __construct()
      * @param \Codeception\Event\StepEvent                                $stepEvent
      * @return void
      */
-    public function logErrors($module, $stepEvent)
+    public static function logErrors($module, $stepEvent)
     {
         //Types available should be "server", "browser", "driver". Only care about browser at the moment.
         if (in_array(self::LOG_TYPE_BROWSER, $module->webDriver->manage()->getAvailableLogTypes())) {
             $browserLogEntries = $module->webDriver->manage()->getLog(self::LOG_TYPE_BROWSER);
-            $jsErrors = $this->getLogsOfType($browserLogEntries, self::ERROR_TYPE_JAVASCRIPT);
+            $jsErrors = self::getLogsOfType($browserLogEntries, self::ERROR_TYPE_JAVASCRIPT);
             foreach ($jsErrors as $entry) {
-                $this->logError(self::ERROR_TYPE_JAVASCRIPT, $stepEvent, $entry);
+                self::logError(self::ERROR_TYPE_JAVASCRIPT, $stepEvent, $entry);
                 //Set javascript error in MagentoWebDriver internal array
                 $module->setJsError("ERROR({$entry["level"]}) - " . $entry["message"]);
             }
@@ -70,7 +43,7 @@ public function logErrors($module, $stepEvent)
      * @param string $type
      * @return array
      */
-    public function getLogsOfType($log, $type)
+    public static function getLogsOfType($log, $type)
     {
         $errors = [];
         foreach ($log as $entry) {
@@ -88,7 +61,7 @@ public function getLogsOfType($log, $type)
      * @param string $type
      * @return array
      */
-    public function filterLogsOfType($log, $type)
+    public static function filterLogsOfType($log, $type)
     {
         $errors = [];
         foreach ($log as $entry) {
@@ -106,7 +79,7 @@ public function filterLogsOfType($log, $type)
      * @param array                        $entry
      * @return void
      */
-    private function logError($type, $stepEvent, $entry)
+    private static function logError($type, $stepEvent, $entry)
     {
         //TODO Add to overall log
         $stepEvent->getTest()->getScenario()->comment("{$type} ERROR({$entry["level"]}) - " . $entry["message"]);
diff --git a/src/Magento/FunctionalTestingFramework/Extension/TestContextExtension.php b/src/Magento/FunctionalTestingFramework/Extension/TestContextExtension.php
index 639d65133..10b7621b4 100644
--- a/src/Magento/FunctionalTestingFramework/Extension/TestContextExtension.php
+++ b/src/Magento/FunctionalTestingFramework/Extension/TestContextExtension.php
@@ -177,14 +177,14 @@ public function afterStep(\Codeception\Event\StepEvent $e)
         if (getenv('ENABLE_BROWSER_LOG')) {
             $browserLog = $this->getDriver()->webDriver->manage()->getLog("browser");
             foreach (explode(',', getenv('BROWSER_LOG_BLACKLIST')) as $source) {
-                $browserLog = ErrorLogger::getInstance()->filterLogsOfType($browserLog, $source);
+                $browserLog = BrowserLogUtil::filterLogsOfType($browserLog, $source);
             }
 
             if (!empty($browserLog)) {
                 AllureHelper::addAttachmentToCurrentStep(json_encode($browserLog, JSON_PRETTY_PRINT), "Browser Log");
             }
         }
-        ErrorLogger::getInstance()->logErrors($this->getDriver(), $e);
+        BrowserLogUtil::logErrors($this->getDriver(), $e);
     }
 
     /**

From 31073a38efa5d6d47943004995c99539fdd71908 Mon Sep 17 00:00:00 2001
From: Kevin Kozan <kkozan@adobe.com>
Date: Thu, 17 Oct 2019 11:34:26 -0500
Subject: [PATCH 11/81] MQE-1644: Add ability to see JS log in Allure

- Unit test + static check fixes
---
 .../Extension/BrowserLogUtilTest.php          | 76 +++++++++++++++++++
 .../Extension/BrowserLogUtil.php              |  4 +-
 2 files changed, 78 insertions(+), 2 deletions(-)
 create mode 100644 dev/tests/unit/Magento/FunctionalTestFramework/Extension/BrowserLogUtilTest.php

diff --git a/dev/tests/unit/Magento/FunctionalTestFramework/Extension/BrowserLogUtilTest.php b/dev/tests/unit/Magento/FunctionalTestFramework/Extension/BrowserLogUtilTest.php
new file mode 100644
index 000000000..32cac698e
--- /dev/null
+++ b/dev/tests/unit/Magento/FunctionalTestFramework/Extension/BrowserLogUtilTest.php
@@ -0,0 +1,76 @@
+<?php
+/**
+ * Copyright © Magento, Inc. All rights reserved.
+ * See COPYING.txt for license details.
+ */
+
+namespace tests\unit\Magento\FunctionalTestFramework\Extension;
+
+use Magento\FunctionalTestingFramework\Util\MagentoTestCase;
+use Magento\FunctionalTestingFramework\Extension\BrowserLogUtil;
+
+class BrowserLogUtilTest extends MagentoTestCase
+{
+    public function testGetLogsOfType()
+    {
+        $entryOne = [
+            "level" => "WARNING",
+            "message" => "warningMessage",
+            "source" => "console-api",
+            "timestamp" => 1234567890
+        ];
+        $entryTwo = [
+            "level" => "ERROR",
+            "message" => "errorMessage",
+            "source" => "other",
+            "timestamp" => 1234567890
+        ];
+        $entryThree = [
+            "level" => "LOG",
+            "message" => "logMessage",
+            "source" => "javascript",
+            "timestamp" => 1234567890
+        ];
+        $log = [
+            $entryOne,
+            $entryTwo,
+            $entryThree
+        ];
+
+        $actual = BrowserLogUtil::getLogsOfType($log, 'console-api');
+
+        self::assertEquals($entryOne, $actual[0]);
+    }
+
+    public function testFilterLogsOfType()
+    {
+        $entryOne = [
+            "level" => "WARNING",
+            "message" => "warningMessage",
+            "source" => "console-api",
+            "timestamp" => 1234567890
+        ];
+        $entryTwo = [
+            "level" => "ERROR",
+            "message" => "errorMessage",
+            "source" => "other",
+            "timestamp" => 1234567890
+        ];
+        $entryThree = [
+            "level" => "LOG",
+            "message" => "logMessage",
+            "source" => "javascript",
+            "timestamp" => 1234567890
+        ];
+        $log = [
+            $entryOne,
+            $entryTwo,
+            $entryThree
+        ];
+
+        $actual = BrowserLogUtil::filterLogsOfType($log, 'console-api');
+
+        self::assertEquals($entryTwo, $actual[0]);
+        self::assertEquals($entryThree, $actual[1]);
+    }
+}
diff --git a/src/Magento/FunctionalTestingFramework/Extension/BrowserLogUtil.php b/src/Magento/FunctionalTestingFramework/Extension/BrowserLogUtil.php
index 330c21ebf..a4f5c6d4c 100644
--- a/src/Magento/FunctionalTestingFramework/Extension/BrowserLogUtil.php
+++ b/src/Magento/FunctionalTestingFramework/Extension/BrowserLogUtil.php
@@ -39,7 +39,7 @@ public static function logErrors($module, $stepEvent)
     /**
      * Loops through given log and returns entries of the given type.
      *
-     * @param array $log
+     * @param array  $log
      * @param string $type
      * @return array
      */
@@ -57,7 +57,7 @@ public static function getLogsOfType($log, $type)
     /**
      * Loops through given log and filters entries of the given type.
      *
-     * @param array $log
+     * @param array  $log
      * @param string $type
      * @return array
      */

From d7d14c333efbe97836cde7d2fefbec6f05651249 Mon Sep 17 00:00:00 2001
From: Kevin Kozan <kkozan@adobe.com>
Date: Thu, 17 Oct 2019 11:57:33 -0500
Subject: [PATCH 12/81] MQE-1644: Add ability to see JS log in Allure

- Added documentation
---
 docs/configuration.md | 18 ++++++++++++++++++
 1 file changed, 18 insertions(+)

diff --git a/docs/configuration.md b/docs/configuration.md
index c741e2f60..4af2c4e82 100644
--- a/docs/configuration.md
+++ b/docs/configuration.md
@@ -277,6 +277,24 @@ Example:
 CREDENTIAL_VAULT_SECRET_BASE_PATH=secret
 ```
 
+### ENABLE_BROWSER_LOG
+
+Enables addition of browser logs to Allure steps
+
+```conf
+ENABLE_BROWSER_LOG=true
+```
+
+### BROWSER_LOG_BLACKLIST
+
+Blacklists types of browser log entries from appearing in Allure steps.
+
+Denoted in browser log entry as `"SOURCE": "type"`.
+
+```conf
+BROWSER_LOG_BLACKLIST=other,console-api
+```
+
 <!-- Link definitions -->
 
 [`MAGENTO_CLI_COMMAND_PATH`]: #magento_cli_command_path

From cd94d20d0be5bf500548cf5121d42a13154da780 Mon Sep 17 00:00:00 2001
From: Tom Reece <treece@adobe.com>
Date: Fri, 18 Oct 2019 11:13:23 -0500
Subject: [PATCH 13/81] MQE-1749: Review Community PR #433

---
 .../DataGenerator/Persist/OperationDataArrayResolver.php      | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/src/Magento/FunctionalTestingFramework/DataGenerator/Persist/OperationDataArrayResolver.php b/src/Magento/FunctionalTestingFramework/DataGenerator/Persist/OperationDataArrayResolver.php
index 7ce748318..60b6e73c2 100644
--- a/src/Magento/FunctionalTestingFramework/DataGenerator/Persist/OperationDataArrayResolver.php
+++ b/src/Magento/FunctionalTestingFramework/DataGenerator/Persist/OperationDataArrayResolver.php
@@ -67,6 +67,10 @@ public function __construct($dependentEntities = null)
      * @param boolean          $fromArray
      * @return array
      * @throws \Exception
+     *
+     * @SuppressWarnings(PHPMD.CyclomaticComplexity)
+     * I suppressed this warning because I was in a hurry to deliver a community PR. That PR modified this function and
+     * introduced a new conditional, bumping the complexity to 11.
      */
     public function resolveOperationDataArray($entityObject, $operationMetadata, $operation, $fromArray = false)
     {

From b7ee3f89eeb665e88a0c818f669944ff0f679bf2 Mon Sep 17 00:00:00 2001
From: Tom Reece <treece@adobe.com>
Date: Fri, 18 Oct 2019 13:28:38 -0500
Subject: [PATCH 14/81] Work in progress before vacation

---
 .../Console/RunTestGroupCommand.php           | 110 +++++++++++++++++-
 1 file changed, 109 insertions(+), 1 deletion(-)

diff --git a/src/Magento/FunctionalTestingFramework/Console/RunTestGroupCommand.php b/src/Magento/FunctionalTestingFramework/Console/RunTestGroupCommand.php
index 7f954c8fe..86b349f53 100644
--- a/src/Magento/FunctionalTestingFramework/Console/RunTestGroupCommand.php
+++ b/src/Magento/FunctionalTestingFramework/Console/RunTestGroupCommand.php
@@ -119,7 +119,7 @@ function ($type, $buffer) use ($output) {
      * @return string
      * @throws \Magento\FunctionalTestingFramework\Exceptions\XmlException
      */
-    private function getGroupAndSuiteConfiguration(array $groups)
+    private function OLDgetGroupAndSuiteConfiguration(array $groups)
     {
         $testConfiguration['tests'] = [];
         $testConfiguration['suites'] = null;
@@ -139,4 +139,112 @@ private function getGroupAndSuiteConfiguration(array $groups)
         $testConfigurationJson = json_encode($testConfiguration);
         return $testConfigurationJson;
     }
+
+    /** first attempt at an implementation, needs tested */
+    private function first_attempt_getGroupAndSuiteConfiguration(array $groups)
+    {
+        $testConfiguration['tests'] = [];
+        $testConfiguration['suites'] = null;
+        $availableSuites = SuiteObjectHandler::getInstance()->getAllObjects();
+
+        // iterate through all group names passed into the command
+        foreach ($groups as $group) {
+            if (array_key_exists($group, $availableSuites)) {
+                // group is actually a suite, so add it to the suites array
+                $testConfiguration['suites'][$group] = [];
+            } else {
+                // group is a group, so find and add all tests from that group to the tests array
+                $testConfiguration['tests'] = array_merge(
+                    $testConfiguration['tests'],
+                    array_keys(TestObjectHandler::getInstance()->getTestsByGroup($group))
+                );
+            }
+        }
+
+        // find all tests that are in suites and build pairs
+        $testsInSuites = SuiteObjectHandler::getInstance()->getAllTestReferences();
+        $suiteToTestPair = [];
+        foreach ($testConfiguration['tests'] as $test) {
+            if (array_key_exists($test, $testsInSuites)) {
+                $suites = $testsInSuites[$test];
+                foreach ($suites as $suite) {
+                    $suiteToTestPair[] = "$suite:$test";
+                }
+            }
+        }
+
+        // add tests to suites array
+        $diff = [];
+        foreach ($suiteToTestPair as $pair) {
+            list($suite, $test) = explode(":", $pair);
+            $testConfiguration['suites'][$suite][] = $test;
+            $diff[] = $test;
+        }
+
+        // remove tests in suites from the tests array
+        $testConfiguration['tests'] = array_diff($testConfiguration['tests'], $diff);
+
+        // encode and return the result
+        $testConfigurationJson = json_encode($testConfiguration);
+        return $testConfigurationJson;
+    }
+
+    /** second attempt at a cleaner implementation, needs work */
+    private function getGroupAndSuiteConfiguration(array $groupOrSuiteNames)
+    {
+        $result['tests'] = [];
+        $result['suites'] = null;
+
+        $groups = [];
+        $suites = [];
+
+        $allSuites = SuiteObjectHandler::getInstance()->getAllObjects();
+        $testsInSuites = SuiteObjectHandler::getInstance()->getAllTestReferences();
+
+        foreach ($groupOrSuiteNames as $groupOrSuiteName) {
+            if (array_key_exists($groupOrSuiteName, $allSuites)) {
+                $suites[] = $groupOrSuiteName;
+            } else {
+                $groups[] = $groupOrSuiteName;
+            }
+        }
+
+        foreach ($suites as $suite) {
+            $result['suites'][$suite] = [];
+        }
+
+        foreach ($groups as $group) {
+            $testsInGroup = TestObjectHandler::getInstance()->getTestsByGroup($group);
+
+            $testsInGroupAndNotInAnySuite = array_diff(
+                array_keys($testsInGroup),
+                array_keys($testsInSuites)
+            );
+
+            $testsInGroupAndInAnySuite = array_diff(
+                array_keys($testsInGroup),
+                $testsInGroupAndNotInAnySuite
+            );
+
+            foreach ($testsInGroupAndInAnySuite as $testInGroupAndInAnySuite) {
+                $cat = $testsInSuites[$testInGroupAndInAnySuite][0];
+                $dog[$cat][] = $testInGroupAndInAnySuite;
+
+                /*
+                 * todo -- I left off here. Code works so far.
+                 * I need to take this $dog array and put into the $result['suites'] array
+                 * and then test it thoroughly
+                 */
+
+            }
+
+            $result['tests'] = array_merge(
+                $result['tests'],
+                $testsInGroupAndNotInAnySuite
+            );
+        }
+
+        $json = json_encode($result);
+        return $json;
+    }
 }

From 39d987a4e08d393cb994c362672f50900aa0e385 Mon Sep 17 00:00:00 2001
From: Kevin Kozan <kkozan@adobe.com>
Date: Tue, 22 Oct 2019 09:00:30 -0500
Subject: [PATCH 15/81] MQE-1644: Add ability to see JS log in Allure

- Moved redundant logic
---
 .../Extension/BrowserLogUtil.php              | 23 ++++++++-----------
 .../Extension/TestContextExtension.php        |  5 ++--
 2 files changed, 12 insertions(+), 16 deletions(-)

diff --git a/src/Magento/FunctionalTestingFramework/Extension/BrowserLogUtil.php b/src/Magento/FunctionalTestingFramework/Extension/BrowserLogUtil.php
index a4f5c6d4c..ca9d2be9e 100644
--- a/src/Magento/FunctionalTestingFramework/Extension/BrowserLogUtil.php
+++ b/src/Magento/FunctionalTestingFramework/Extension/BrowserLogUtil.php
@@ -16,23 +16,20 @@ class BrowserLogUtil
     const ERROR_TYPE_JAVASCRIPT = "javascript";
 
     /**
-     * Loops through stepEvent for browser log entries
+     * Loops throw errors in log and logs them to allure. Uses Module to set the error itself
      *
-     * @param \Magento\FunctionalTestingFramework\Module\MagentoWebDriver $module
-     * @param \Codeception\Event\StepEvent                                $stepEvent
+     * @param array                         $log
+     * @param \Codeception\Module\WebDriver $module
+     * @param \Codeception\Event\StepEvent  $stepEvent
      * @return void
      */
-    public static function logErrors($module, $stepEvent)
+    public static function logErrors($log, $module, $stepEvent)
     {
-        //Types available should be "server", "browser", "driver". Only care about browser at the moment.
-        if (in_array(self::LOG_TYPE_BROWSER, $module->webDriver->manage()->getAvailableLogTypes())) {
-            $browserLogEntries = $module->webDriver->manage()->getLog(self::LOG_TYPE_BROWSER);
-            $jsErrors = self::getLogsOfType($browserLogEntries, self::ERROR_TYPE_JAVASCRIPT);
-            foreach ($jsErrors as $entry) {
-                self::logError(self::ERROR_TYPE_JAVASCRIPT, $stepEvent, $entry);
-                //Set javascript error in MagentoWebDriver internal array
-                $module->setJsError("ERROR({$entry["level"]}) - " . $entry["message"]);
-            }
+        $jsErrors = self::getLogsOfType($log, self::ERROR_TYPE_JAVASCRIPT);
+        foreach ($jsErrors as $entry) {
+            self::logError(self::ERROR_TYPE_JAVASCRIPT, $stepEvent, $entry);
+            //Set javascript error in MagentoWebDriver internal array
+            $module->setJsError("ERROR({$entry["level"]}) - " . $entry["message"]);
         }
     }
 
diff --git a/src/Magento/FunctionalTestingFramework/Extension/TestContextExtension.php b/src/Magento/FunctionalTestingFramework/Extension/TestContextExtension.php
index 10b7621b4..9fe7adb7a 100644
--- a/src/Magento/FunctionalTestingFramework/Extension/TestContextExtension.php
+++ b/src/Magento/FunctionalTestingFramework/Extension/TestContextExtension.php
@@ -174,17 +174,16 @@ public function beforeStep(\Codeception\Event\StepEvent $e)
      */
     public function afterStep(\Codeception\Event\StepEvent $e)
     {
+        $browserLog = $this->getDriver()->webDriver->manage()->getLog("browser");
         if (getenv('ENABLE_BROWSER_LOG')) {
-            $browserLog = $this->getDriver()->webDriver->manage()->getLog("browser");
             foreach (explode(',', getenv('BROWSER_LOG_BLACKLIST')) as $source) {
                 $browserLog = BrowserLogUtil::filterLogsOfType($browserLog, $source);
             }
-
             if (!empty($browserLog)) {
                 AllureHelper::addAttachmentToCurrentStep(json_encode($browserLog, JSON_PRETTY_PRINT), "Browser Log");
             }
         }
-        BrowserLogUtil::logErrors($this->getDriver(), $e);
+        BrowserLogUtil::logErrors($browserLog, $this->getDriver(), $e);
     }
 
     /**

From d8c58af84dd91060947b7cf81e5088cd1340ef95 Mon Sep 17 00:00:00 2001
From: Kevin Kozan <kkozan@adobe.com>
Date: Wed, 23 Oct 2019 14:29:55 -0500
Subject: [PATCH 16/81] MQE-1782: MFTF run:group can't run test in a suite

- Added unit test template for test/suite configuration method
---
 .../Console/BaseGenerateCommandTest.php       | 74 +++++++++++++++++++
 1 file changed, 74 insertions(+)
 create mode 100644 dev/tests/unit/Magento/FunctionalTestFramework/Console/BaseGenerateCommandTest.php

diff --git a/dev/tests/unit/Magento/FunctionalTestFramework/Console/BaseGenerateCommandTest.php b/dev/tests/unit/Magento/FunctionalTestFramework/Console/BaseGenerateCommandTest.php
new file mode 100644
index 000000000..d1933c7d8
--- /dev/null
+++ b/dev/tests/unit/Magento/FunctionalTestFramework/Console/BaseGenerateCommandTest.php
@@ -0,0 +1,74 @@
+<?php
+/**
+ * Copyright © Magento, Inc. All rights reserved.
+ * See COPYING.txt for license details.
+ */
+namespace Tests\unit\Magento\FunctionalTestingFramework\Console;
+
+use AspectMock\Test as AspectMock;
+use PHPUnit\Framework\TestCase;
+use Magento\FunctionalTestingFramework\Console\BaseGenerateCommand;
+use Magento\FunctionalTestingFramework\Suite\Objects\SuiteObject;
+use Magento\FunctionalTestingFramework\Suite\Handlers\SuiteObjectHandler;
+use Magento\FunctionalTestingFramework\Test\Objects\TestObject;
+use Magento\FunctionalTestingFramework\Test\Handlers\TestObjectHandler;
+class BaseGenerateCommandTest extends TestCase
+{
+    public function tearDown()
+    {
+        AspectMock::clean();
+    }
+
+    /**
+     * One test in one suite
+     */
+    public function testSimpleTestConfig()
+    {
+        $testOne = new TestObject('Test1', [], [], []);
+        $suiteOne = new SuiteObject('Suite1', ['Test1' => $testOne], [], []);
+
+        $testArray = ['Test1' => $testOne];
+        $suiteArray = ['Suite1' => $suiteOne];
+
+        $this->mockHandlers($testArray, $suiteArray);
+
+        $actual = json_decode($this->callConfig(['Test1']), true);
+        $expected = ['tests' => null, 'suites' => ['Suite1' => ['Test1']]];
+        $this->assertEquals($expected, $actual);
+    }
+
+    /**
+     * Mock handlers to skip parsing
+     * @param array $testArray
+     * @param array $suiteArray
+     * @throws \Exception
+     */
+    public function mockHandlers($testArray, $suiteArray)
+    {
+        AspectMock::double(TestObjectHandler::class,['initTestData' => ''])->make();
+        $handler = TestObjectHandler::getInstance();
+        $property = new \ReflectionProperty(TestObjectHandler::class, 'tests');
+        $property->setAccessible(true);
+        $property->setValue($handler, $testArray);
+
+        AspectMock::double(SuiteObjectHandler::class, ['initSuiteData' => ''])->make();
+        $handler = SuiteObjectHandler::getInstance();
+        $property = new \ReflectionProperty(SuiteObjectHandler::class, 'suiteObjects');
+        $property->setAccessible(true);
+        $property->setValue($handler, $suiteArray);
+    }
+
+    /**
+     * Changes visibility and runs getTestAndSuiteConfiguration
+     * @param array $testArray
+     * @return string
+     */
+    public function callConfig($testArray)
+    {
+        $command = new BaseGenerateCommand();
+        $class = new \ReflectionClass($command);
+        $method = $class->getMethod('getTestAndSuiteConfiguration');
+        $method->setAccessible(true);
+        return $method->invokeArgs($command, [$testArray]);
+    }
+}

From 8e7494d91390ebf5a522205694a0f5262bd2fb89 Mon Sep 17 00:00:00 2001
From: Kevin Kozan <kkozan@adobe.com>
Date: Thu, 24 Oct 2019 08:34:47 -0500
Subject: [PATCH 17/81] MQE-1782: MFTF run:group can't run test in a suite

- Moved group config function to BaseGenerateCommand
- Added more unit tests
---
 .../Console/BaseGenerateCommandTest.php       | 70 ++++++++++++++++++-
 .../Console/BaseGenerateCommand.php           | 67 ++++++++++++++++++
 .../Console/RunTestGroupCommand.php           | 59 ----------------
 3 files changed, 134 insertions(+), 62 deletions(-)

diff --git a/dev/tests/unit/Magento/FunctionalTestFramework/Console/BaseGenerateCommandTest.php b/dev/tests/unit/Magento/FunctionalTestFramework/Console/BaseGenerateCommandTest.php
index d1933c7d8..82a223726 100644
--- a/dev/tests/unit/Magento/FunctionalTestFramework/Console/BaseGenerateCommandTest.php
+++ b/dev/tests/unit/Magento/FunctionalTestFramework/Console/BaseGenerateCommandTest.php
@@ -22,7 +22,7 @@ public function tearDown()
     /**
      * One test in one suite
      */
-    public function testSimpleTestConfig()
+    public function testOneTestOneSuiteConfig()
     {
         $testOne = new TestObject('Test1', [], [], []);
         $suiteOne = new SuiteObject('Suite1', ['Test1' => $testOne], [], []);
@@ -32,11 +32,61 @@ public function testSimpleTestConfig()
 
         $this->mockHandlers($testArray, $suiteArray);
 
-        $actual = json_decode($this->callConfig(['Test1']), true);
+        $actual = json_decode($this->callTestConfig(['Test1']), true);
         $expected = ['tests' => null, 'suites' => ['Suite1' => ['Test1']]];
         $this->assertEquals($expected, $actual);
     }
 
+
+    /**
+     * One test in one suite
+     */
+    public function testOneTestTwoSuitesConfig()
+    {
+        $testOne = new TestObject('Test1', [], [], []);
+        $suiteOne = new SuiteObject('Suite1', ['Test1' => $testOne], [], []);
+        $suiteTwo = new SuiteObject('Suite2', ['Test1' => $testOne], [], []);
+
+        $testArray = ['Test1' => $testOne];
+        $suiteArray = ['Suite1' => $suiteOne, 'Suite2' => $suiteTwo];
+
+        $this->mockHandlers($testArray, $suiteArray);
+
+        $actual = json_decode($this->callTestConfig(['Test1']), true);
+        $expected = ['tests' => null, 'suites' => ['Suite1' => ['Test1'], 'Suite2' => ['Test1']]];
+        $this->assertEquals($expected, $actual);
+    }
+
+    public function testOneTestOneGroup()
+    {
+        $testOne = new TestObject('Test1', [], ['group' => ['Group1']], []);
+
+        $testArray = ['Test1' => $testOne];
+        $suiteArray = [];
+
+        $this->mockHandlers($testArray, $suiteArray);
+
+        $actual = json_decode($this->callGroupConfig(['Group1']), true);
+        $expected = ['tests' => ['Test1'], 'suites' => null];
+        $this->assertEquals($expected, $actual);
+    }
+
+    public function testThreeTestsTwoGroup()
+    {
+        $testOne = new TestObject('Test1', [], ['group' => ['Group1']], []);
+        $testTwo = new TestObject('Test2', [], ['group' => ['Group1']], []);
+        $testThree = new TestObject('Test3', [], ['group' => ['Group2']], []);
+
+        $testArray = ['Test1' => $testOne, 'Test2' => $testTwo, 'Test3' => $testThree];
+        $suiteArray = [];
+
+        $this->mockHandlers($testArray, $suiteArray);
+
+        $actual = json_decode($this->callGroupConfig(['Group1', 'Group2']), true);
+        $expected = ['tests' => ['Test1', 'Test2', 'Test3'], 'suites' => null];
+        $this->assertEquals($expected, $actual);
+    }
+
     /**
      * Mock handlers to skip parsing
      * @param array $testArray
@@ -63,7 +113,7 @@ public function mockHandlers($testArray, $suiteArray)
      * @param array $testArray
      * @return string
      */
-    public function callConfig($testArray)
+    public function callTestConfig($testArray)
     {
         $command = new BaseGenerateCommand();
         $class = new \ReflectionClass($command);
@@ -71,4 +121,18 @@ public function callConfig($testArray)
         $method->setAccessible(true);
         return $method->invokeArgs($command, [$testArray]);
     }
+
+    /**
+     * Changes visibility and runs getGroupAndSuiteConfiguration
+     * @param array $groupArray
+     * @return string
+     */
+    public function callGroupConfig($groupArray)
+    {
+        $command = new BaseGenerateCommand();
+        $class = new \ReflectionClass($command);
+        $method = $class->getMethod('getGroupAndSuiteConfiguration');
+        $method->setAccessible(true);
+        return $method->invokeArgs($command, [$groupArray]);
+    }
 }
diff --git a/src/Magento/FunctionalTestingFramework/Console/BaseGenerateCommand.php b/src/Magento/FunctionalTestingFramework/Console/BaseGenerateCommand.php
index 87b203d66..0c7fff674 100644
--- a/src/Magento/FunctionalTestingFramework/Console/BaseGenerateCommand.php
+++ b/src/Magento/FunctionalTestingFramework/Console/BaseGenerateCommand.php
@@ -8,6 +8,7 @@
 
 namespace Magento\FunctionalTestingFramework\Console;
 
+use Magento\FunctionalTestingFramework\Test\Handlers\TestObjectHandler;
 use Symfony\Component\Console\Command\Command;
 use Symfony\Component\Console\Input\InputOption;
 use Symfony\Component\Console\Output\OutputInterface;
@@ -103,4 +104,70 @@ protected function getTestAndSuiteConfiguration(array $tests)
         $testConfigurationJson = json_encode($testConfiguration);
         return $testConfigurationJson;
     }
+
+    /** second attempt at a cleaner implementation, needs work */
+    protected function getGroupAndSuiteConfiguration(array $groupOrSuiteNames)
+    {
+        $result['tests'] = [];
+        $result['suites'] = [];
+
+        $groups = [];
+        $suites = [];
+
+        $allSuites = SuiteObjectHandler::getInstance()->getAllObjects();
+        $testsInSuites = SuiteObjectHandler::getInstance()->getAllTestReferences();
+
+        foreach ($groupOrSuiteNames as $groupOrSuiteName) {
+            if (array_key_exists($groupOrSuiteName, $allSuites)) {
+                $suites[] = $groupOrSuiteName;
+            } else {
+                $groups[] = $groupOrSuiteName;
+            }
+        }
+
+        foreach ($suites as $suite) {
+            $result['suites'][$suite] = [];
+        }
+
+        foreach ($groups as $group) {
+            $testsInGroup = TestObjectHandler::getInstance()->getTestsByGroup($group);
+
+            $testsInGroupAndNotInAnySuite = array_diff(
+                array_keys($testsInGroup),
+                array_keys($testsInSuites)
+            );
+
+            $testsInGroupAndInAnySuite = array_diff(
+                array_keys($testsInGroup),
+                $testsInGroupAndNotInAnySuite
+            );
+
+            foreach ($testsInGroupAndInAnySuite as $testInGroupAndInAnySuite) {
+                $cat = $testsInSuites[$testInGroupAndInAnySuite][0];
+                $dog[$cat][] = $testInGroupAndInAnySuite;
+
+                /*
+                 * todo -- I left off here. Code works so far.
+                 * I need to take this $dog array and put into the $result['suites'] array
+                 * and then test it thoroughly
+                 */
+
+            }
+
+            $result['tests'] = array_merge(
+                $result['tests'],
+                $testsInGroupAndNotInAnySuite
+            );
+        }
+
+        if (empty($result['tests'])) {
+            $result['tests'] = null;
+        }
+        if (empty($result['suites'])) {
+            $result['suites'] = null;
+        }
+
+        $json = json_encode($result);
+        return $json;
+    }
 }
diff --git a/src/Magento/FunctionalTestingFramework/Console/RunTestGroupCommand.php b/src/Magento/FunctionalTestingFramework/Console/RunTestGroupCommand.php
index 86b349f53..d31bf7873 100644
--- a/src/Magento/FunctionalTestingFramework/Console/RunTestGroupCommand.php
+++ b/src/Magento/FunctionalTestingFramework/Console/RunTestGroupCommand.php
@@ -188,63 +188,4 @@ private function first_attempt_getGroupAndSuiteConfiguration(array $groups)
         $testConfigurationJson = json_encode($testConfiguration);
         return $testConfigurationJson;
     }
-
-    /** second attempt at a cleaner implementation, needs work */
-    private function getGroupAndSuiteConfiguration(array $groupOrSuiteNames)
-    {
-        $result['tests'] = [];
-        $result['suites'] = null;
-
-        $groups = [];
-        $suites = [];
-
-        $allSuites = SuiteObjectHandler::getInstance()->getAllObjects();
-        $testsInSuites = SuiteObjectHandler::getInstance()->getAllTestReferences();
-
-        foreach ($groupOrSuiteNames as $groupOrSuiteName) {
-            if (array_key_exists($groupOrSuiteName, $allSuites)) {
-                $suites[] = $groupOrSuiteName;
-            } else {
-                $groups[] = $groupOrSuiteName;
-            }
-        }
-
-        foreach ($suites as $suite) {
-            $result['suites'][$suite] = [];
-        }
-
-        foreach ($groups as $group) {
-            $testsInGroup = TestObjectHandler::getInstance()->getTestsByGroup($group);
-
-            $testsInGroupAndNotInAnySuite = array_diff(
-                array_keys($testsInGroup),
-                array_keys($testsInSuites)
-            );
-
-            $testsInGroupAndInAnySuite = array_diff(
-                array_keys($testsInGroup),
-                $testsInGroupAndNotInAnySuite
-            );
-
-            foreach ($testsInGroupAndInAnySuite as $testInGroupAndInAnySuite) {
-                $cat = $testsInSuites[$testInGroupAndInAnySuite][0];
-                $dog[$cat][] = $testInGroupAndInAnySuite;
-
-                /*
-                 * todo -- I left off here. Code works so far.
-                 * I need to take this $dog array and put into the $result['suites'] array
-                 * and then test it thoroughly
-                 */
-
-            }
-
-            $result['tests'] = array_merge(
-                $result['tests'],
-                $testsInGroupAndNotInAnySuite
-            );
-        }
-
-        $json = json_encode($result);
-        return $json;
-    }
 }

From 09d8bfd5bd10ad305c0dd7e4168e2f786c8c0b0a Mon Sep 17 00:00:00 2001
From: Kevin Kozan <kkozan@adobe.com>
Date: Fri, 25 Oct 2019 09:23:42 -0500
Subject: [PATCH 18/81] MQE-1782: MFTF run:group can't run test in a suite

- Removed old or unused code from implementation
- Added unit tests
---
 .../Console/BaseGenerateCommandTest.php       | 58 ++++++++++++--
 .../Console/BaseGenerateCommand.php           | 19 ++---
 .../Console/RunTestGroupCommand.php           | 77 -------------------
 3 files changed, 58 insertions(+), 96 deletions(-)

diff --git a/dev/tests/unit/Magento/FunctionalTestFramework/Console/BaseGenerateCommandTest.php b/dev/tests/unit/Magento/FunctionalTestFramework/Console/BaseGenerateCommandTest.php
index 82a223726..2d4df8096 100644
--- a/dev/tests/unit/Magento/FunctionalTestFramework/Console/BaseGenerateCommandTest.php
+++ b/dev/tests/unit/Magento/FunctionalTestFramework/Console/BaseGenerateCommandTest.php
@@ -12,6 +12,7 @@
 use Magento\FunctionalTestingFramework\Suite\Handlers\SuiteObjectHandler;
 use Magento\FunctionalTestingFramework\Test\Objects\TestObject;
 use Magento\FunctionalTestingFramework\Test\Handlers\TestObjectHandler;
+
 class BaseGenerateCommandTest extends TestCase
 {
     public function tearDown()
@@ -19,9 +20,6 @@ public function tearDown()
         AspectMock::clean();
     }
 
-    /**
-     * One test in one suite
-     */
     public function testOneTestOneSuiteConfig()
     {
         $testOne = new TestObject('Test1', [], [], []);
@@ -37,10 +35,6 @@ public function testOneTestOneSuiteConfig()
         $this->assertEquals($expected, $actual);
     }
 
-
-    /**
-     * One test in one suite
-     */
     public function testOneTestTwoSuitesConfig()
     {
         $testOne = new TestObject('Test1', [], [], []);
@@ -87,6 +81,54 @@ public function testThreeTestsTwoGroup()
         $this->assertEquals($expected, $actual);
     }
 
+    public function testOneTestOneSuiteOneGroupConfig()
+    {
+        $testOne = new TestObject('Test1', [], ['group' => ['Group1']], []);
+        $suiteOne = new SuiteObject('Suite1', ['Test1' => $testOne], [], []);
+
+        $testArray = ['Test1' => $testOne];
+        $suiteArray = ['Suite1' => $suiteOne];
+
+        $this->mockHandlers($testArray, $suiteArray);
+
+        $actual = json_decode($this->callGroupConfig(['Group1']), true);
+        $expected = ['tests' => null, 'suites' => ['Suite1' => ['Test1']]];
+        $this->assertEquals($expected, $actual);
+    }
+
+    public function testTwoTestOneSuiteTwoGroupConfig()
+    {
+        $testOne = new TestObject('Test1', [], ['group' => ['Group1']], []);
+        $testTwo = new TestObject('Test2', [], ['group' => ['Group2']], []);
+        $suiteOne = new SuiteObject('Suite1', ['Test1' => $testOne, 'Test2' => $testTwo], [], []);
+
+        $testArray = ['Test1' => $testOne, 'Test2' => $testTwo];
+        $suiteArray = ['Suite1' => $suiteOne];
+
+        $this->mockHandlers($testArray, $suiteArray);
+
+        $actual = json_decode($this->callGroupConfig(['Group1', 'Group2']), true);
+        $expected = ['tests' => null, 'suites' => ['Suite1' => ['Test1', 'Test2']]];
+        $this->assertEquals($expected, $actual);
+    }
+
+    public function testTwoTestTwoSuiteOneGroupConfig()
+    {
+        $testOne = new TestObject('Test1', [], ['group' => ['Group1']], []);
+        $testTwo = new TestObject('Test2', [], ['group' => ['Group1']], []);
+        $suiteOne = new SuiteObject('Suite1', ['Test1' => $testOne], [], []);
+        $suiteTwo = new SuiteObject('Suite2', ['Test2' => $testTwo], [], []);
+
+        $testArray = ['Test1' => $testOne, 'Test2' => $testTwo];
+        $suiteArray = ['Suite1' => $suiteOne, 'Suite2' => $suiteTwo];
+
+        $this->mockHandlers($testArray, $suiteArray);
+
+        $actual = json_decode($this->callGroupConfig(['Group1']), true);
+        $expected = ['tests' => null, 'suites' => ['Suite1' => ['Test1'], 'Suite2' => ['Test2']]];
+        $this->assertEquals($expected, $actual);
+    }
+
     /**
      * Mock handlers to skip parsing
      * @param array $testArray
@@ -95,7 +137,7 @@ public function testThreeTestsTwoGroup()
      */
     public function mockHandlers($testArray, $suiteArray)
     {
-        AspectMock::double(TestObjectHandler::class,['initTestData' => ''])->make();
+        AspectMock::double(TestObjectHandler::class, ['initTestData' => ''])->make();
         $handler = TestObjectHandler::getInstance();
         $property = new \ReflectionProperty(TestObjectHandler::class, 'tests');
         $property->setAccessible(true);
diff --git a/src/Magento/FunctionalTestingFramework/Console/BaseGenerateCommand.php b/src/Magento/FunctionalTestingFramework/Console/BaseGenerateCommand.php
index 0c7fff674..d99839d7c 100644
--- a/src/Magento/FunctionalTestingFramework/Console/BaseGenerateCommand.php
+++ b/src/Magento/FunctionalTestingFramework/Console/BaseGenerateCommand.php
@@ -76,7 +76,6 @@ protected function removeGeneratedDirectory(OutputInterface $output, bool $verbo
      * @return false|string
      * @throws \Magento\FunctionalTestingFramework\Exceptions\XmlException
      */
-
     protected function getTestAndSuiteConfiguration(array $tests)
     {
         $testConfiguration['tests'] = null;
@@ -105,7 +104,12 @@ protected function getTestAndSuiteConfiguration(array $tests)
         return $testConfigurationJson;
     }
 
-    /** second attempt at a cleaner implementation, needs work */
+    /**
+     * Returns an array of test configuration to be used as an argument for generation of tests
+     * This function uses group or suite names for generation
+     * @return false|string
+     * @throws \Magento\FunctionalTestingFramework\Exceptions\XmlException
+     */
     protected function getGroupAndSuiteConfiguration(array $groupOrSuiteNames)
     {
         $result['tests'] = [];
@@ -143,15 +147,8 @@ protected function getGroupAndSuiteConfiguration(array $groupOrSuiteNames)
             );
 
             foreach ($testsInGroupAndInAnySuite as $testInGroupAndInAnySuite) {
-                $cat = $testsInSuites[$testInGroupAndInAnySuite][0];
-                $dog[$cat][] = $testInGroupAndInAnySuite;
-
-                /*
-                 * todo -- I left off here. Code works so far.
-                 * I need to take this $dog array and put into the $result['suites'] array
-                 * and then test it thoroughly
-                 */
-
+                $suiteName = $testsInSuites[$testInGroupAndInAnySuite][0];
+                $result['suites'][$suiteName][] = $testInGroupAndInAnySuite;
             }
 
             $result['tests'] = array_merge(
diff --git a/src/Magento/FunctionalTestingFramework/Console/RunTestGroupCommand.php b/src/Magento/FunctionalTestingFramework/Console/RunTestGroupCommand.php
index d31bf7873..98d121d40 100644
--- a/src/Magento/FunctionalTestingFramework/Console/RunTestGroupCommand.php
+++ b/src/Magento/FunctionalTestingFramework/Console/RunTestGroupCommand.php
@@ -111,81 +111,4 @@ function ($type, $buffer) use ($output) {
             }
         );
     }
-
-    /**
-     * Returns a json string to be used as an argument for generation of a group or suite
-     *
-     * @param array $groups
-     * @return string
-     * @throws \Magento\FunctionalTestingFramework\Exceptions\XmlException
-     */
-    private function OLDgetGroupAndSuiteConfiguration(array $groups)
-    {
-        $testConfiguration['tests'] = [];
-        $testConfiguration['suites'] = null;
-        $availableSuites = SuiteObjectHandler::getInstance()->getAllObjects();
-
-        foreach ($groups as $group) {
-            if (array_key_exists($group, $availableSuites)) {
-                $testConfiguration['suites'][$group] = [];
-            }
-
-            $testConfiguration['tests'] = array_merge(
-                $testConfiguration['tests'],
-                array_keys(TestObjectHandler::getInstance()->getTestsByGroup($group))
-            );
-        }
-
-        $testConfigurationJson = json_encode($testConfiguration);
-        return $testConfigurationJson;
-    }
-
-    /** first attempt at an implementation, needs tested */
-    private function first_attempt_getGroupAndSuiteConfiguration(array $groups)
-    {
-        $testConfiguration['tests'] = [];
-        $testConfiguration['suites'] = null;
-        $availableSuites = SuiteObjectHandler::getInstance()->getAllObjects();
-
-        // iterate through all group names passed into the command
-        foreach ($groups as $group) {
-            if (array_key_exists($group, $availableSuites)) {
-                // group is actually a suite, so add it to the suites array
-                $testConfiguration['suites'][$group] = [];
-            } else {
-                // group is a group, so find and add all tests from that group to the tests array
-                $testConfiguration['tests'] = array_merge(
-                    $testConfiguration['tests'],
-                    array_keys(TestObjectHandler::getInstance()->getTestsByGroup($group))
-                );
-            }
-        }
-
-        // find all tests that are in suites and build pairs
-        $testsInSuites = SuiteObjectHandler::getInstance()->getAllTestReferences();
-        $suiteToTestPair = [];
-        foreach ($testConfiguration['tests'] as $test) {
-            if (array_key_exists($test, $testsInSuites)) {
-                $suites = $testsInSuites[$test];
-                foreach ($suites as $suite) {
-                    $suiteToTestPair[] = "$suite:$test";
-                }
-            }
-        }
-
-        // add tests to suites array
-        $diff = [];
-        foreach ($suiteToTestPair as $pair) {
-            list($suite, $test) = explode(":", $pair);
-            $testConfiguration['suites'][$suite][] = $test;
-            $diff[] = $test;
-        }
-
-        // remove tests in suites from the tests array
-        $testConfiguration['tests'] = array_diff($testConfiguration['tests'], $diff);
-
-        // encode and return the result
-        $testConfigurationJson = json_encode($testConfiguration);
-        return $testConfigurationJson;
-    }
 }

From 2a9df738db3cec1bce7fd18624b7b4312ccf1bfa Mon Sep 17 00:00:00 2001
From: Kevin Kozan <kkozan@adobe.com>
Date: Mon, 28 Oct 2019 13:44:54 -0500
Subject: [PATCH 19/81] MQE-1782: MFTF run:group can't run test in a suite

- Fixed mixed group/suite usecase
- Added Unit test
---
 .../Console/BaseGenerateCommandTest.php       | 22 +++++++++++++++++++
 .../Console/BaseGenerateCommand.php           |  4 ++++
 2 files changed, 26 insertions(+)

diff --git a/dev/tests/unit/Magento/FunctionalTestFramework/Console/BaseGenerateCommandTest.php b/dev/tests/unit/Magento/FunctionalTestFramework/Console/BaseGenerateCommandTest.php
index 2d4df8096..f46becaa5 100644
--- a/dev/tests/unit/Magento/FunctionalTestFramework/Console/BaseGenerateCommandTest.php
+++ b/dev/tests/unit/Magento/FunctionalTestFramework/Console/BaseGenerateCommandTest.php
@@ -129,6 +129,28 @@ public function testTwoTestTwoSuiteOneGroupConfig()
         $this->assertEquals($expected, $actual);
     }
 
+    /**
+     * Test specific usecase of a test that is in a group with the group being called along with the suite
+     * i.e. run:group Group1 Suite1
+     * @throws \Exception
+     */
+    public function testThreeTestOneSuiteOneGroupMix()
+    {
+        $testOne = new TestObject('Test1', [], [], []);
+        $testTwo = new TestObject('Test2', [], [], []);
+        $testThree = new TestObject('Test3', [], ['group' => ['Group1']], []);
+        $suiteOne = new SuiteObject('Suite1', ['Test1' => $testOne, 'Test2' => $testTwo, 'Test3' => $testThree], [], []);
+
+        $testArray = ['Test1' => $testOne, 'Test2' => $testTwo, 'Test3' => $testThree];
+        $suiteArray = ['Suite1' => $suiteOne];
+
+        $this->mockHandlers($testArray, $suiteArray);
+
+        $actual = json_decode($this->callGroupConfig(['Group1', 'Suite1']), true);
+        $expected = ['tests' => null, 'suites' => ['Suite1' => []]];
+        $this->assertEquals($expected, $actual);
+    }
+
     /**
      * Mock handlers to skip parsing
      * @param array $testArray
diff --git a/src/Magento/FunctionalTestingFramework/Console/BaseGenerateCommand.php b/src/Magento/FunctionalTestingFramework/Console/BaseGenerateCommand.php
index d99839d7c..244610877 100644
--- a/src/Magento/FunctionalTestingFramework/Console/BaseGenerateCommand.php
+++ b/src/Magento/FunctionalTestingFramework/Console/BaseGenerateCommand.php
@@ -148,6 +148,10 @@ protected function getGroupAndSuiteConfiguration(array $groupOrSuiteNames)
 
             foreach ($testsInGroupAndInAnySuite as $testInGroupAndInAnySuite) {
                 $suiteName = $testsInSuites[$testInGroupAndInAnySuite][0];
+                if (isset($result['suites'][$suiteName])) {
+                    // Suite is already being called to run in its entirety, do not filter list
+                    continue;
+                }
                 $result['suites'][$suiteName][] = $testInGroupAndInAnySuite;
             }
 

From 9cb452badb4c4353b80eb877bd0320903a5ea0e1 Mon Sep 17 00:00:00 2001
From: Kevin Kozan <kkozan@adobe.com>
Date: Mon, 28 Oct 2019 14:23:50 -0500
Subject: [PATCH 20/81] MQE-1782: MFTF run:group can't run test in a suite

- changed from isset to array_search
- fixed static check
---
 .../Console/BaseGenerateCommandTest.php                    | 7 ++++++-
 .../Console/BaseGenerateCommand.php                        | 2 +-
 2 files changed, 7 insertions(+), 2 deletions(-)

diff --git a/dev/tests/unit/Magento/FunctionalTestFramework/Console/BaseGenerateCommandTest.php b/dev/tests/unit/Magento/FunctionalTestFramework/Console/BaseGenerateCommandTest.php
index f46becaa5..1b1686ce2 100644
--- a/dev/tests/unit/Magento/FunctionalTestFramework/Console/BaseGenerateCommandTest.php
+++ b/dev/tests/unit/Magento/FunctionalTestFramework/Console/BaseGenerateCommandTest.php
@@ -139,7 +139,12 @@ public function testThreeTestOneSuiteOneGroupMix()
         $testOne = new TestObject('Test1', [], [], []);
         $testTwo = new TestObject('Test2', [], [], []);
         $testThree = new TestObject('Test3', [], ['group' => ['Group1']], []);
-        $suiteOne = new SuiteObject('Suite1', ['Test1' => $testOne, 'Test2' => $testTwo, 'Test3' => $testThree], [], []);
+        $suiteOne = new SuiteObject(
+            'Suite1',
+            ['Test1' => $testOne, 'Test2' => $testTwo, 'Test3' => $testThree],
+            [],
+            []
+        );
 
         $testArray = ['Test1' => $testOne, 'Test2' => $testTwo, 'Test3' => $testThree];
         $suiteArray = ['Suite1' => $suiteOne];
diff --git a/src/Magento/FunctionalTestingFramework/Console/BaseGenerateCommand.php b/src/Magento/FunctionalTestingFramework/Console/BaseGenerateCommand.php
index 244610877..5afaf17c5 100644
--- a/src/Magento/FunctionalTestingFramework/Console/BaseGenerateCommand.php
+++ b/src/Magento/FunctionalTestingFramework/Console/BaseGenerateCommand.php
@@ -148,7 +148,7 @@ protected function getGroupAndSuiteConfiguration(array $groupOrSuiteNames)
 
             foreach ($testsInGroupAndInAnySuite as $testInGroupAndInAnySuite) {
                 $suiteName = $testsInSuites[$testInGroupAndInAnySuite][0];
-                if (isset($result['suites'][$suiteName])) {
+                if (array_search($suiteName, $suites) !== false) {
                     // Suite is already being called to run in its entirety, do not filter list
                     continue;
                 }

From 5497d2123f4ed358add5aa4c1c0670fb8dafe2c1 Mon Sep 17 00:00:00 2001
From: Tom Reece <treece@adobe.com>
Date: Wed, 30 Oct 2019 13:40:14 -0500
Subject: [PATCH 21/81] MQE-1845: Convert Writing tests with actiongroups

---
 docs/guides/action-groups.md | 82 ++++++++++++++++++++++++++++++++++++
 1 file changed, 82 insertions(+)
 create mode 100644 docs/guides/action-groups.md

diff --git a/docs/guides/action-groups.md b/docs/guides/action-groups.md
new file mode 100644
index 000000000..69527d7cc
--- /dev/null
+++ b/docs/guides/action-groups.md
@@ -0,0 +1,82 @@
+# Action Group Best Practices
+
+We should strive to write tests using only action groups. Fortunately we have built up a large set of action groups to get  started. We can make use of them and extend them for our own specific needs. In some cases, we may never even need to write action groups of our own. We may be able to simply chain together calls to existing action groups to implement our new test case.
+
+## Why use Action Groups?
+
+Action groups simplify maintainability by reducing duplication. Because they are re-usable building blocks, odds are that they are already made use of by existing tests in the Magento codebase. This proves their stability through real-world use. Take for example, the action group named `LoginAsAdmin`:
+
+```xml
+<actionGroup name="LoginAsAdmin">
+    <annotations>
+        <description>Login to Backend Admin using provided User Data. PLEASE NOTE: This Action Group does NOT validate that you are Logged In.</description>
+    </annotations>
+    <arguments>
+        <argument name="adminUser" type="entity" defaultValue="DefaultAdminUser"/>
+    </arguments>
+
+    <amOnPage url="{{AdminLoginPage.url}}" stepKey="navigateToAdmin"/>
+    <fillField selector="{{AdminLoginFormSection.username}}" userInput="{{adminUser.username}}" stepKey="fillUsername"/>
+    <fillField selector="{{AdminLoginFormSection.password}}" userInput="{{adminUser.password}}" stepKey="fillPassword"/>
+    <click selector="{{AdminLoginFormSection.signIn}}" stepKey="clickLogin"/>
+    <closeAdminNotification stepKey="closeAdminNotification"/>
+</actionGroup>
+``` 
+
+As you may be able to guess, logging in to the admin panel is one of the most used action groups. It is used around 1,500 times at the time of this writing.
+
+Imagine if this wasn't an action group and instead we were to copy and paste these 5 actions so many times. In that scenario, if we need to make a small change it would require a lot of work. But with the action group, we can make the change in one place.
+
+## How can I extend Action Groups?
+
+Let's continue using `LoginAsAdmin` as our example. I have trimmed away metadata to clearly reveal that this action group performs 5 actions:
+
+```
+<actionGroup name="LoginAsAdmin">
+    ...
+    <amOnPage url="{{AdminLoginPage.url}}" .../>
+    <fillField selector="{{AdminLoginFormSection.username}}" .../>
+    <fillField selector="{{AdminLoginFormSection.password}}" .../>
+    <click selector="{{AdminLoginFormSection.signIn}}" .../>
+    <closeAdminNotification .../>
+</actionGroup>
+```
+
+This works against the standard Magento admin panel login page. But let's imagine we're working on a Magento extension that adds a CAPTCHA field to the login page. If we create and activate this extension and then we try to run all existing tests, we can expect almost everything to fail because now we are unable to log in because we did not completely fill out all of the login form. The CAPTCHA field was left unfilled.
+
+We can overcome this by making use of MFTF's extensibility. All we need to do is to provide a "merge" that modifies the existing `LoginAsAdmin` action group. Our simple merge file will look like this:
+
+```
+<actionGroup name="LoginAsAdmin">
+    <fillField selector="{{CaptchaSection.captchaInput}}" before="signIn" .../>
+</actionGroup>
+```
+
+Because the name of this merge is also `LoginAsAdmin`, the two get merged together and an additional step happens everytime this action group is made use of.
+
+To continue this demonstration, let's imagine someone else is working on a Two Factor Authentication extension and they also provide a merge for the `LoginAsAdmin` action group. Their merge looks similar to what we've already seen. The only difference is that this time we fill a different field:
+
+```
+<actionGroup name="LoginAsAdmin">
+    <fillField selector="{{TwoFactorSection.twoFactorInput}}" before="signIn" .../>
+</actionGroup>
+```
+
+Bringing it all together, our resulting `LoginAsAdmin` action group becomes this:
+
+```
+<actionGroup name="LoginAsAdmin">
+    ...
+    <amOnPage url="{{AdminLoginPage.url}}" .../>
+    <fillField selector="{{AdminLoginFormSection.username}}" .../>
+    <fillField selector="{{AdminLoginFormSection.password}}" .../>
+    <fillField selector="{{CaptchaSection.captchaInput}}" .../>
+    <fillField selector="{{TwoFactorSection.twoFactorInput}}" .../>
+    <click selector="{{AdminLoginFormSection.signIn}}" .../>
+    <closeAdminNotification .../>
+</actionGroup>
+```
+
+Note that no file actually contains these exact contents above, but instead all three files come together to form this action group.
+
+One final thing to be aware of is that this extensibility can be applied in many ways. Obviously we need to use it if we want to affect existing Magento entities like tests, action groups, and data. But something not so obvious is that this can be used within the walls of your own entities in order to make them more maintainable too.

From 7ac812bed2cfe6f4f2570a4c818e1e3b0f4db5f8 Mon Sep 17 00:00:00 2001
From: Ji Lu <jilu1@adobe.com>
Date: Wed, 30 Oct 2019 15:06:09 -0500
Subject: [PATCH 22/81] MQE-1257: MFTF Troubleshoot command

---
 dev/tests/functional/standalone_bootstrap.php |   8 +-
 .../Config/MftfApplicationConfig.php          |   3 +-
 .../Console/CommandList.php                   |  27 +--
 .../Console/TroubleShootCommand.php           | 158 ++++++++++++++++++
 .../Module/MagentoWebDriver.php               |  33 ++++
 .../Util/ModuleResolver.php                   |   2 +-
 .../FunctionalTestingFramework/_bootstrap.php |   8 +-
 7 files changed, 218 insertions(+), 21 deletions(-)
 create mode 100644 src/Magento/FunctionalTestingFramework/Console/TroubleShootCommand.php

diff --git a/dev/tests/functional/standalone_bootstrap.php b/dev/tests/functional/standalone_bootstrap.php
index 763062d04..de6ef394d 100755
--- a/dev/tests/functional/standalone_bootstrap.php
+++ b/dev/tests/functional/standalone_bootstrap.php
@@ -15,10 +15,12 @@
 
 require_once realpath(PROJECT_ROOT . '/vendor/autoload.php');
 
+$envFilePath = dirname(dirname(__DIR__)) . DIRECTORY_SEPARATOR;
+defined('ENV_FILE_PATH') || define('ENV_FILE_PATH', $envFilePath);
+
 //Load constants from .env file
-$envFilePath = dirname(dirname(__DIR__));
-if (file_exists($envFilePath . DIRECTORY_SEPARATOR . '.env')) {
-    $env = new \Dotenv\Loader($envFilePath . DIRECTORY_SEPARATOR . '.env');
+if (file_exists(ENV_FILE_PATH . '.env')) {
+    $env = new \Dotenv\Loader(ENV_FILE_PATH . '.env');
     $env->load();
 
     foreach ($_ENV as $key => $var) {
diff --git a/src/Magento/FunctionalTestingFramework/Config/MftfApplicationConfig.php b/src/Magento/FunctionalTestingFramework/Config/MftfApplicationConfig.php
index 80db27de0..f429110d7 100644
--- a/src/Magento/FunctionalTestingFramework/Config/MftfApplicationConfig.php
+++ b/src/Magento/FunctionalTestingFramework/Config/MftfApplicationConfig.php
@@ -15,7 +15,8 @@ class MftfApplicationConfig
     const GENERATION_PHASE = "generation";
     const EXECUTION_PHASE = "execution";
     const UNIT_TEST_PHASE = "testing";
-    const MFTF_PHASES = [self::GENERATION_PHASE, self::EXECUTION_PHASE, self::UNIT_TEST_PHASE];
+    const DIAGNOSTIC_PHASE = "diagnostic";
+    const MFTF_PHASES = [self::GENERATION_PHASE, self::EXECUTION_PHASE, self::UNIT_TEST_PHASE, self::DIAGNOSTIC_PHASE];
 
     /**
      * Mftf debug levels
diff --git a/src/Magento/FunctionalTestingFramework/Console/CommandList.php b/src/Magento/FunctionalTestingFramework/Console/CommandList.php
index 34d221840..be62227ac 100644
--- a/src/Magento/FunctionalTestingFramework/Console/CommandList.php
+++ b/src/Magento/FunctionalTestingFramework/Console/CommandList.php
@@ -29,19 +29,20 @@ class CommandList implements CommandListInterface
     public function __construct(array $commands = [])
     {
         $this->commands = [
-             'build:project' => new BuildProjectCommand(),
-             'reset' => new CleanProjectCommand(),
-             'generate:urn-catalog' => new GenerateDevUrnCommand(),
-             'generate:suite' => new GenerateSuiteCommand(),
-             'generate:tests' => new GenerateTestsCommand(),
-             'run:test' => new RunTestCommand(),
-             'run:group' => new RunTestGroupCommand(),
-             'run:failed' => new RunTestFailedCommand(),
-             'run:manifest' => new RunManifestCommand(),
-             'setup:env' => new SetupEnvCommand(),
-             'upgrade:tests' => new UpgradeTestsCommand(),
-             'generate:docs' => new GenerateDocsCommand(),
-             'static-checks' => new StaticChecksCommand()
+            'build:project' => new BuildProjectCommand(),
+            'generate:docs' => new GenerateDocsCommand(),
+            'generate:suite' => new GenerateSuiteCommand(),
+            'generate:tests' => new GenerateTestsCommand(),
+            'generate:urn-catalog' => new GenerateDevUrnCommand(),
+            'reset' => new CleanProjectCommand(),
+            'run:failed' => new RunTestFailedCommand(),
+            'run:group' => new RunTestGroupCommand(),
+            'run:manifest' => new RunManifestCommand(),
+            'run:test' => new RunTestCommand(),
+            'setup:env' => new SetupEnvCommand(),
+            'static-checks' => new StaticChecksCommand(),
+            'troubleshoot' => new TroubleShootCommand(),
+            'upgrade:tests' => new UpgradeTestsCommand(),
         ] + $commands;
     }
 
diff --git a/src/Magento/FunctionalTestingFramework/Console/TroubleShootCommand.php b/src/Magento/FunctionalTestingFramework/Console/TroubleShootCommand.php
new file mode 100644
index 000000000..124473bcd
--- /dev/null
+++ b/src/Magento/FunctionalTestingFramework/Console/TroubleShootCommand.php
@@ -0,0 +1,158 @@
+<?php
+/**
+ * Copyright © Magento, Inc. All rights reserved.
+ * See COPYING.txt for license details.
+ */
+declare(strict_types = 1);
+
+namespace Magento\FunctionalTestingFramework\Console;
+
+use Codeception\Configuration;
+use Symfony\Component\EventDispatcher\EventDispatcher;
+use Codeception\SuiteManager;
+use Magento\FunctionalTestingFramework\Config\MftfApplicationConfig;
+use Magento\FunctionalTestingFramework\Exceptions\TestFrameworkException;
+use Symfony\Component\Console\Command\Command;
+use Symfony\Component\Console\Input\InputInterface;
+use Symfony\Component\Console\Output\OutputInterface;
+use Magento\FunctionalTestingFramework\Util\ModuleResolver;
+
+class TroubleShootCommand extends Command
+{
+    const CODECEPTION_AUTOLOAD_FILE = PROJECT_ROOT . '/vendor/codeception/codeception/autoload.php';
+    const MFTF_CODECEPTION_CONFIG_FILE = ENV_FILE_PATH . 'codeception.yml';
+    const SUITE = 'functional';
+    const REQUIRED_PHP_EXTS = ['CURL', 'mbstring', 'bcmath', 'zip', 'dom', 'gd', 'intl'];
+
+    /**
+     * Command Output
+     *
+     * @var OutputInterface
+     */
+    private $output;
+
+    /**
+     * Configures the current command.
+     *
+     * @return void
+     */
+    protected function configure()
+    {
+        $this->setName('troubleshoot')
+            ->setDescription(
+                'This command checks environment readiness for generating and running MFTF tests.'
+            );
+    }
+
+    /**
+     * Executes the current command.
+     *
+     * @param InputInterface  $input
+     * @param OutputInterface $output
+     * @return void
+     * @throws TestFrameworkException
+     */
+    protected function execute(InputInterface $input, OutputInterface $output)
+    {
+        $cmdStatus = true;
+
+        // Config application
+        $verbose = $output->isVerbose();
+        $this->output = $output;
+        MftfApplicationConfig::create(
+            false,
+            MftfApplicationConfig::DIAGNOSTIC_PHASE,
+            $verbose,
+            MftfApplicationConfig::LEVEL_DEVELOPER,
+            false
+        );
+
+        // Check required PHP extensions
+        foreach (self::REQUIRED_PHP_EXTS as $ext) {
+            $status = $this->checkPhpExtIsAvailable($ext);
+            $cmdStatus = $cmdStatus && !$status ? false : $cmdStatus;
+        }
+
+        // Check authentication to Magento Admin
+        $status = $this->checkAuthenticationToMagentoAdmin();
+        $cmdStatus = $cmdStatus && !$status ? false : $cmdStatus;
+
+        // Check connectivity and authentication to Magento Admin
+        $status = $this->checkConnectivityToSeleniumServer();
+        $cmdStatus = $cmdStatus && !$status ? false : $cmdStatus;
+
+        if ($cmdStatus) {
+            exit(0);
+        } else {
+            exit(1);
+        }
+    }
+
+    /**
+     * Check php extention is installed and available
+     *
+     * @param string $ext
+     * @return boolean
+     */
+    private function checkPhpExtIsAvailable($ext)
+    {
+        $result = false;
+        $this->output->writeln("\nChecking PHP extenstion \"{$ext}\" ...");
+        if (extension_loaded(strtolower($ext))) {
+            $this->output->writeln('Successful');
+            $result = true;
+        } else {
+            $this->output->writeln(
+                "MFTF requires \"{$ext}\" extension installed to make tests run\n"
+                . "Please make sure that the PHP you run has \"{$ext}\" installed and enabled."
+            );
+        }
+        return $result;
+    }
+
+    /**
+     * Check authentication to Magento Admin
+     *
+     * @return boolean
+     */
+    private function checkAuthenticationToMagentoAdmin()
+    {
+        $result = false;
+        try {
+            $this->output->writeln("\nChecking authentication to Magento Admin ...");
+            ModuleResolver::getInstance()->getAdminToken();
+            $this->output->writeln('Successful');
+            $result = true;
+        } catch (TestFrameworkException $e) {
+            $this->output->writeln($e->getMessage());
+        }
+        return $result;
+    }
+
+    /**
+     * Check Connectivity to Selenium Server
+     *
+     * @return boolean
+     */
+    private function checkConnectivityToSeleniumServer()
+    {
+        $result = false;
+
+        // Check connectivity to Selenium through Codeception
+        $this->output->writeln("\nChecking connectivity to Selenium Server ...");
+        require_once realpath(self::CODECEPTION_AUTOLOAD_FILE);
+
+        $config = Configuration::config(realpath(self::MFTF_CODECEPTION_CONFIG_FILE));
+        $settings = Configuration::suiteSettings(self::SUITE, $config);
+        $dispatcher = new EventDispatcher();
+        $suiteManager = new SuiteManager($dispatcher, self::SUITE, $settings);
+        try {
+            $suiteManager->initialize();
+            $this->output->writeln('Successful');
+            $result = true;
+        } catch (TestFrameworkException $e) {
+            $this->output->writeln($e->getMessage());
+        }
+        return $result;
+    }
+}
diff --git a/src/Magento/FunctionalTestingFramework/Module/MagentoWebDriver.php b/src/Magento/FunctionalTestingFramework/Module/MagentoWebDriver.php
index 17a868efd..248e16e44 100644
--- a/src/Magento/FunctionalTestingFramework/Module/MagentoWebDriver.php
+++ b/src/Magento/FunctionalTestingFramework/Module/MagentoWebDriver.php
@@ -23,6 +23,9 @@
 use Symfony\Component\Process\Process;
 use Yandex\Allure\Adapter\Support\AttachmentSupport;
 use Magento\FunctionalTestingFramework\Exceptions\TestFrameworkException;
+use Magento\FunctionalTestingFramework\Config\MftfApplicationConfig;
+use Facebook\WebDriver\Remote\RemoteWebDriver;
+use Facebook\WebDriver\Exception\WebDriverCurlException;
 
 /**
  * MagentoWebDriver module provides common Magento web actions through Selenium WebDriver.
@@ -126,6 +129,11 @@ public function _initialize()
         $this->config = ConfigSanitizerUtil::sanitizeWebDriverConfig($this->config);
         parent::_initialize();
         $this->cleanJsError();
+
+        // Check Selenium Server readiness if it's in diagnostic phase
+        if (MftfApplicationConfig::getConfig()->getPhase() === MftfApplicationConfig::DIAGNOSTIC_PHASE) {
+            $this->checkSeleniumServerReadiness();
+        }
     }
 
     /**
@@ -826,6 +834,31 @@ public function makeScreenshot($name = null)
         AllureHelper::addAttachmentToCurrentStep($screenName, 'Screenshot');
     }
 
+    /**
+     * Check connectivity to running selenium server
+     *
+     * @return void
+     * @throws TestFrameworkException
+     */
+    public function checkSeleniumServerReadiness()
+    {
+        try {
+            RemoteWebDriver::create(
+                $this->wdHost,
+                $this->capabilities,
+                $this->connectionTimeoutInMs,
+                $this->requestTimeoutInMs,
+                $this->httpProxy,
+                $this->httpProxyPort
+            );
+        } catch (WebDriverCurlException $e) {
+            throw new TestFrameworkException(
+                "Can't connect to Webdriver at {$this->wdHost}.\n"
+                . "Please make sure that Selenium Server is running."
+            );
+        }
+    }
+
     /**
      * Takes given $command and executes it against bin/magento executable. Returns stdout output from the command.
      *
diff --git a/src/Magento/FunctionalTestingFramework/Util/ModuleResolver.php b/src/Magento/FunctionalTestingFramework/Util/ModuleResolver.php
index d776603d1..c9c050878 100644
--- a/src/Magento/FunctionalTestingFramework/Util/ModuleResolver.php
+++ b/src/Magento/FunctionalTestingFramework/Util/ModuleResolver.php
@@ -703,7 +703,7 @@ private function printMagentoVersionInfo()
      *
      * @return string|boolean
      */
-    protected function getAdminToken()
+    public function getAdminToken()
     {
         $login = $_ENV['MAGENTO_ADMIN_USERNAME'] ?? null;
         $password = $_ENV['MAGENTO_ADMIN_PASSWORD'] ?? null;
diff --git a/src/Magento/FunctionalTestingFramework/_bootstrap.php b/src/Magento/FunctionalTestingFramework/_bootstrap.php
index 34807d2b9..dd7cfd37d 100644
--- a/src/Magento/FunctionalTestingFramework/_bootstrap.php
+++ b/src/Magento/FunctionalTestingFramework/_bootstrap.php
@@ -15,11 +15,13 @@
     return;
 }
 defined('PROJECT_ROOT') || define('PROJECT_ROOT', $projectRootPath);
-$envFilepath = realpath($projectRootPath . '/dev/tests/acceptance/');
 
+$envFilePath = realpath($projectRootPath . '/dev/tests/acceptance/');
+defined('ENV_FILE_PATH') || define('ENV_FILE_PATH', $envFilePath);
 
-if (file_exists($envFilepath . DIRECTORY_SEPARATOR . '.env')) {
-    $env = new \Dotenv\Loader($envFilepath . DIRECTORY_SEPARATOR . '.env');
+//Load constants from .env file
+if (file_exists(ENV_FILE_PATH . '.env')) {
+    $env = new \Dotenv\Loader(ENV_FILE_PATH . '.env');
     $env->load();
 
     if (array_key_exists('TESTS_MODULE_PATH', $_ENV) xor array_key_exists('TESTS_BP', $_ENV)) {

From 7bdcafd91775c627377482a41bede5f9e80b9e9e Mon Sep 17 00:00:00 2001
From: Ji Lu <jilu1@adobe.com>
Date: Wed, 30 Oct 2019 15:34:47 -0500
Subject: [PATCH 23/81] MQE-1257: MFTF Troubleshoot command

---
 .../Console/TroubleShootCommand.php                        | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/src/Magento/FunctionalTestingFramework/Console/TroubleShootCommand.php b/src/Magento/FunctionalTestingFramework/Console/TroubleShootCommand.php
index 124473bcd..3dc9cb288 100644
--- a/src/Magento/FunctionalTestingFramework/Console/TroubleShootCommand.php
+++ b/src/Magento/FunctionalTestingFramework/Console/TroubleShootCommand.php
@@ -49,8 +49,9 @@ protected function configure()
      *
      * @param InputInterface  $input
      * @param OutputInterface $output
-     * @return void
+     * @return integer
      * @throws TestFrameworkException
+     * @SuppressWarnings(PHPMD.UnusedFormalParameter)
      */
     protected function execute(InputInterface $input, OutputInterface $output)
     {
@@ -82,9 +83,9 @@ protected function execute(InputInterface $input, OutputInterface $output)
         $cmdStatus = $cmdStatus && !$status ? false : $cmdStatus;
 
         if ($cmdStatus) {
-            exit(0);
+            return 0;
         } else {
-            exit(1);
+            return 1;
         }
     }
 

From 4fdf441f93b476dbeeb72a6cb032f50ee50bf06d Mon Sep 17 00:00:00 2001
From: Ji Lu <jilu1@adobe.com>
Date: Wed, 30 Oct 2019 16:09:03 -0500
Subject: [PATCH 24/81] MQE-1257: MFTF Troubleshoot command

---
 .../FunctionalTestingFramework/Module/MagentoWebDriver.php     | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/Magento/FunctionalTestingFramework/Module/MagentoWebDriver.php b/src/Magento/FunctionalTestingFramework/Module/MagentoWebDriver.php
index 248e16e44..2b41af99e 100644
--- a/src/Magento/FunctionalTestingFramework/Module/MagentoWebDriver.php
+++ b/src/Magento/FunctionalTestingFramework/Module/MagentoWebDriver.php
@@ -843,7 +843,7 @@ public function makeScreenshot($name = null)
     public function checkSeleniumServerReadiness()
     {
         try {
-            RemoteWebDriver::create(
+            $driver = RemoteWebDriver::create(
                 $this->wdHost,
                 $this->capabilities,
                 $this->connectionTimeoutInMs,
@@ -851,6 +851,7 @@ public function checkSeleniumServerReadiness()
                 $this->httpProxy,
                 $this->httpProxyPort
             );
+            $driver->close();
         } catch (WebDriverCurlException $e) {
             throw new TestFrameworkException(
                 "Can't connect to Webdriver at {$this->wdHost}.\n"

From aa1629d661b66ea8406a5d385d5e86974fffe1b4 Mon Sep 17 00:00:00 2001
From: Donald Booth <dobooth@adobe.com>
Date: Fri, 1 Nov 2019 11:28:17 -0500
Subject: [PATCH 25/81] Editorial pass

---
 docs/guides/action-groups.md | 30 +++++++++++++++---------------
 1 file changed, 15 insertions(+), 15 deletions(-)

diff --git a/docs/guides/action-groups.md b/docs/guides/action-groups.md
index 69527d7cc..30c531e4e 100644
--- a/docs/guides/action-groups.md
+++ b/docs/guides/action-groups.md
@@ -1,6 +1,6 @@
 # Action Group Best Practices
 
-We should strive to write tests using only action groups. Fortunately we have built up a large set of action groups to get  started. We can make use of them and extend them for our own specific needs. In some cases, we may never even need to write action groups of our own. We may be able to simply chain together calls to existing action groups to implement our new test case.
+We strive to write tests using only action groups. Fortunately, we have built up a large set of action groups to get started. We can make use of them and extend them for our own specific needs. In some cases, we may never even need to write action groups of our own. We may be able to simply chain together calls to existing action groups to implement our new test case.
 
 ## Why use Action Groups?
 
@@ -23,15 +23,15 @@ Action groups simplify maintainability by reducing duplication. Because they are
 </actionGroup>
 ``` 
 
-As you may be able to guess, logging in to the admin panel is one of the most used action groups. It is used around 1,500 times at the time of this writing.
+Logging in to the admin panel is one of the most used action groups. It is used around 1,500 times at the time of this writing.
 
-Imagine if this wasn't an action group and instead we were to copy and paste these 5 actions so many times. In that scenario, if we need to make a small change it would require a lot of work. But with the action group, we can make the change in one place.
+Imagine if this was not an action group and instead we were to copy and paste these 5 actions every time. In that scenario, if a small change was needed, it would require a lot of work. But with the action group, we can make the change in one place.
 
-## How can I extend Action Groups?
+## How to extend action groups
 
-Let's continue using `LoginAsAdmin` as our example. I have trimmed away metadata to clearly reveal that this action group performs 5 actions:
+Again using `LoginAsAdmin` as our example, we trim away metadata to clearly reveal that this action group performs 5 actions:
 
-```
+```xml
 <actionGroup name="LoginAsAdmin">
     ...
     <amOnPage url="{{AdminLoginPage.url}}" .../>
@@ -42,21 +42,21 @@ Let's continue using `LoginAsAdmin` as our example. I have trimmed away metadata
 </actionGroup>
 ```
 
-This works against the standard Magento admin panel login page. But let's imagine we're working on a Magento extension that adds a CAPTCHA field to the login page. If we create and activate this extension and then we try to run all existing tests, we can expect almost everything to fail because now we are unable to log in because we did not completely fill out all of the login form. The CAPTCHA field was left unfilled.
+This works against the standard Magento admin panel login page. Bu imagine we are working on a Magento extension that adds a CAPTCHA field to the login page. If we create and activate this extension and then run all existing tests, we can expect almost everything to fail because the CAPTCHA field is left unfilled.
 
-We can overcome this by making use of MFTF's extensibility. All we need to do is to provide a "merge" that modifies the existing `LoginAsAdmin` action group. Our simple merge file will look like this:
+We can overcome this by making use of MFTF's extensibility. All we need to do is to provide a "merge" that modifies the existing `LoginAsAdmin` action group. Our merge file will look like:
 
-```
+```xml
 <actionGroup name="LoginAsAdmin">
     <fillField selector="{{CaptchaSection.captchaInput}}" before="signIn" .../>
 </actionGroup>
 ```
 
-Because the name of this merge is also `LoginAsAdmin`, the two get merged together and an additional step happens everytime this action group is made use of.
+Because the name of this merge is also `LoginAsAdmin`, the two get merged together and an additional step happens everytime this action group is used.
 
-To continue this demonstration, let's imagine someone else is working on a Two Factor Authentication extension and they also provide a merge for the `LoginAsAdmin` action group. Their merge looks similar to what we've already seen. The only difference is that this time we fill a different field:
+To continue this example, imagine someone else is working on a 'Two-Factor Authentication' extension and they also provide a merge for the `LoginAsAdmin` action group. Their merge looks similar to what we have already seen. The only difference is that this time we fill a different field:
 
-```
+```xml
 <actionGroup name="LoginAsAdmin">
     <fillField selector="{{TwoFactorSection.twoFactorInput}}" before="signIn" .../>
 </actionGroup>
@@ -64,7 +64,7 @@ To continue this demonstration, let's imagine someone else is working on a Two F
 
 Bringing it all together, our resulting `LoginAsAdmin` action group becomes this:
 
-```
+```xml
 <actionGroup name="LoginAsAdmin">
     ...
     <amOnPage url="{{AdminLoginPage.url}}" .../>
@@ -77,6 +77,6 @@ Bringing it all together, our resulting `LoginAsAdmin` action group becomes this
 </actionGroup>
 ```
 
-Note that no file actually contains these exact contents above, but instead all three files come together to form this action group.
+No one file contains this exact content as above, but instead all three files come together to form this action group.
 
-One final thing to be aware of is that this extensibility can be applied in many ways. Obviously we need to use it if we want to affect existing Magento entities like tests, action groups, and data. But something not so obvious is that this can be used within the walls of your own entities in order to make them more maintainable too.
+This extensibility can be applied in many ways. We can use it to affect existing Magento entities such as tests, action groups, and data. Not so obvious is that this tehcnique can be used within your own entities to make them more maintainable as well.

From f1299b1b03e2bfe7eb1a6382985b91fb3edba6de Mon Sep 17 00:00:00 2001
From: Ji Lu <jilu1@adobe.com>
Date: Fri, 8 Nov 2019 16:41:32 -0600
Subject: [PATCH 26/81] MQE-1257: MFTF doctor command

---
 .../Config/MftfApplicationConfig.php          |   3 +-
 .../Console/CommandList.php                   |   2 +-
 ...ubleShootCommand.php => DoctorCommand.php} | 122 ++++++++++++------
 .../Exceptions/TestFrameworkException.php     |  18 +++
 .../Module/MagentoWebDriver.php               |  31 -----
 .../Module/MagentoWebDriverDoctor.php         |  87 +++++++++++++
 6 files changed, 188 insertions(+), 75 deletions(-)
 rename src/Magento/FunctionalTestingFramework/Console/{TroubleShootCommand.php => DoctorCommand.php} (56%)
 create mode 100644 src/Magento/FunctionalTestingFramework/Module/MagentoWebDriverDoctor.php

diff --git a/src/Magento/FunctionalTestingFramework/Config/MftfApplicationConfig.php b/src/Magento/FunctionalTestingFramework/Config/MftfApplicationConfig.php
index f429110d7..80db27de0 100644
--- a/src/Magento/FunctionalTestingFramework/Config/MftfApplicationConfig.php
+++ b/src/Magento/FunctionalTestingFramework/Config/MftfApplicationConfig.php
@@ -15,8 +15,7 @@ class MftfApplicationConfig
     const GENERATION_PHASE = "generation";
     const EXECUTION_PHASE = "execution";
     const UNIT_TEST_PHASE = "testing";
-    const DIAGNOSTIC_PHASE = "diagnostic";
-    const MFTF_PHASES = [self::GENERATION_PHASE, self::EXECUTION_PHASE, self::UNIT_TEST_PHASE, self::DIAGNOSTIC_PHASE];
+    const MFTF_PHASES = [self::GENERATION_PHASE, self::EXECUTION_PHASE, self::UNIT_TEST_PHASE];
 
     /**
      * Mftf debug levels
diff --git a/src/Magento/FunctionalTestingFramework/Console/CommandList.php b/src/Magento/FunctionalTestingFramework/Console/CommandList.php
index be62227ac..bf9cbd58e 100644
--- a/src/Magento/FunctionalTestingFramework/Console/CommandList.php
+++ b/src/Magento/FunctionalTestingFramework/Console/CommandList.php
@@ -30,6 +30,7 @@ public function __construct(array $commands = [])
     {
         $this->commands = [
             'build:project' => new BuildProjectCommand(),
+            'doctor' => new DoctorCommand(),
             'generate:docs' => new GenerateDocsCommand(),
             'generate:suite' => new GenerateSuiteCommand(),
             'generate:tests' => new GenerateTestsCommand(),
@@ -41,7 +42,6 @@ public function __construct(array $commands = [])
             'run:test' => new RunTestCommand(),
             'setup:env' => new SetupEnvCommand(),
             'static-checks' => new StaticChecksCommand(),
-            'troubleshoot' => new TroubleShootCommand(),
             'upgrade:tests' => new UpgradeTestsCommand(),
         ] + $commands;
     }
diff --git a/src/Magento/FunctionalTestingFramework/Console/TroubleShootCommand.php b/src/Magento/FunctionalTestingFramework/Console/DoctorCommand.php
similarity index 56%
rename from src/Magento/FunctionalTestingFramework/Console/TroubleShootCommand.php
rename to src/Magento/FunctionalTestingFramework/Console/DoctorCommand.php
index 3dc9cb288..469d4ac47 100644
--- a/src/Magento/FunctionalTestingFramework/Console/TroubleShootCommand.php
+++ b/src/Magento/FunctionalTestingFramework/Console/DoctorCommand.php
@@ -16,13 +16,14 @@
 use Symfony\Component\Console\Input\InputInterface;
 use Symfony\Component\Console\Output\OutputInterface;
 use Magento\FunctionalTestingFramework\Util\ModuleResolver;
+use Magento\FunctionalTestingFramework\Module\MagentoWebDriver;
+use Magento\FunctionalTestingFramework\Module\MagentoWebDriverDoctor;
 
-class TroubleShootCommand extends Command
+class DoctorCommand extends Command
 {
     const CODECEPTION_AUTOLOAD_FILE = PROJECT_ROOT . '/vendor/codeception/codeception/autoload.php';
     const MFTF_CODECEPTION_CONFIG_FILE = ENV_FILE_PATH . 'codeception.yml';
     const SUITE = 'functional';
-    const REQUIRED_PHP_EXTS = ['CURL', 'mbstring', 'bcmath', 'zip', 'dom', 'gd', 'intl'];
 
     /**
      * Command Output
@@ -31,6 +32,13 @@ class TroubleShootCommand extends Command
      */
     private $output;
 
+    /**
+     * Exception Context
+     *
+     * @var array
+     */
+    private $context = [];
+
     /**
      * Configures the current command.
      *
@@ -38,7 +46,7 @@ class TroubleShootCommand extends Command
      */
     protected function configure()
     {
-        $this->setName('troubleshoot')
+        $this->setName('doctor')
             ->setDescription(
                 'This command checks environment readiness for generating and running MFTF tests.'
             );
@@ -62,24 +70,22 @@ protected function execute(InputInterface $input, OutputInterface $output)
         $this->output = $output;
         MftfApplicationConfig::create(
             false,
-            MftfApplicationConfig::DIAGNOSTIC_PHASE,
+            MftfApplicationConfig::GENERATION_PHASE,
             $verbose,
             MftfApplicationConfig::LEVEL_DEVELOPER,
             false
         );
 
-        // Check required PHP extensions
-        foreach (self::REQUIRED_PHP_EXTS as $ext) {
-            $status = $this->checkPhpExtIsAvailable($ext);
-            $cmdStatus = $cmdStatus && !$status ? false : $cmdStatus;
-        }
-
         // Check authentication to Magento Admin
         $status = $this->checkAuthenticationToMagentoAdmin();
         $cmdStatus = $cmdStatus && !$status ? false : $cmdStatus;
 
-        // Check connectivity and authentication to Magento Admin
-        $status = $this->checkConnectivityToSeleniumServer();
+        // Check connection to Selenium
+        $status = $this->checkConnectionToSeleniumServer();
+        $cmdStatus = $cmdStatus && !$status ? false : $cmdStatus;
+
+        // Check access to Magento CLI
+        $status = $this->checkAccessToMagentoCLI();
         $cmdStatus = $cmdStatus && !$status ? false : $cmdStatus;
 
         if ($cmdStatus) {
@@ -90,70 +96,104 @@ protected function execute(InputInterface $input, OutputInterface $output)
     }
 
     /**
-     * Check php extention is installed and available
+     * Check authentication to Magento Admin
      *
-     * @param string $ext
      * @return boolean
      */
-    private function checkPhpExtIsAvailable($ext)
+    private function checkAuthenticationToMagentoAdmin()
     {
         $result = false;
-        $this->output->writeln("\nChecking PHP extenstion \"{$ext}\" ...");
-        if (extension_loaded(strtolower($ext))) {
+        try {
+            $this->output->writeln("\nChecking authentication to Magento Admin ...");
+            ModuleResolver::getInstance()->getAdminToken();
             $this->output->writeln('Successful');
             $result = true;
-        } else {
-            $this->output->writeln(
-                "MFTF requires \"{$ext}\" extension installed to make tests run\n"
-                . "Please make sure that the PHP you run has \"{$ext}\" installed and enabled."
-            );
+        } catch (TestFrameworkException $e) {
+            $this->output->writeln($e->getMessage());
         }
         return $result;
     }
 
     /**
-     * Check authentication to Magento Admin
+     * Check Connection to Selenium Server
      *
      * @return boolean
      */
-    private function checkAuthenticationToMagentoAdmin()
+    private function checkConnectionToSeleniumServer()
     {
-        $result = false;
-        try {
-            $this->output->writeln("\nChecking authentication to Magento Admin ...");
-            ModuleResolver::getInstance()->getAdminToken();
+        // Check connection to Selenium through Codeception
+        $this->output->writeln("\nChecking connection to Selenium Server ...");
+        $this->runMagentoWebDriverDoctor();
+
+        if (isset($this->context[MagentoWebDriverDoctor::EXCEPTION_TYPE_SELENIUM])) {
+            $this->output->write($this->context[MagentoWebDriverDoctor::EXCEPTION_TYPE_SELENIUM] . "\n");
+            return false;
+        } else {
             $this->output->writeln('Successful');
-            $result = true;
-        } catch (TestFrameworkException $e) {
-            $this->output->writeln($e->getMessage());
+            return true;
         }
-        return $result;
     }
 
     /**
-     * Check Connectivity to Selenium Server
+     * Check access to Magento CLI setup
      *
      * @return boolean
      */
-    private function checkConnectivityToSeleniumServer()
+    private function checkAccessToMagentoCLI()
     {
-        $result = false;
+        // Check Magento CLI setup
+        $this->output->writeln("\nChecking access to Magento CLI ...");
+        $this->runMagentoWebDriverDoctor();
+
+        if (isset($this->context[MagentoWebDriverDoctor::EXCEPTION_TYPE_MAGENTO_CLI])) {
+            $this->output->write($this->context[MagentoWebDriverDoctor::EXCEPTION_TYPE_MAGENTO_CLI] . "\n");
+            return false;
+        } else {
+            $this->output->writeln('Successful');
+            return true;
+        }
+    }
+
+    /**
+     * Run diagnose through MagentoWebDriverDoctor
+     *
+     * @return void
+     */
+    private function runMagentoWebDriverDoctor()
+    {
+        if (!empty($this->context)) {
+            return;
+        }
+
+        $magentoWebDriver = '\\' . MagentoWebDriver::class;
+        $magentoWebDriverDoctor = '\\' . MagentoWebDriverDoctor::class;
 
-        // Check connectivity to Selenium through Codeception
-        $this->output->writeln("\nChecking connectivity to Selenium Server ...");
         require_once realpath(self::CODECEPTION_AUTOLOAD_FILE);
 
         $config = Configuration::config(realpath(self::MFTF_CODECEPTION_CONFIG_FILE));
         $settings = Configuration::suiteSettings(self::SUITE, $config);
+
+        // Enable MagentoWebDriverDoctor
+        $settings['modules']['enabled'][] = $magentoWebDriverDoctor;
+        $settings['modules']['config'][$magentoWebDriverDoctor] =
+            $settings['modules']['config'][$magentoWebDriver];
+
+        // Disable MagentoWebDriver to avoid conflicts
+        foreach ($settings['modules']['enabled'] as $index => $module) {
+            if ($module == $magentoWebDriver) {
+                unset($settings['modules']['enabled'][$index]);
+                break;
+            }
+        }
+        unset($settings['modules']['config'][$magentoWebDriver]);
+
         $dispatcher = new EventDispatcher();
         $suiteManager = new SuiteManager($dispatcher, self::SUITE, $settings);
         try {
             $suiteManager->initialize();
-            $this->output->writeln('Successful');
-            $result = true;
+            $this->context = ['Successful'];
         } catch (TestFrameworkException $e) {
-            $this->output->writeln($e->getMessage());
+            $this->context = $e->getContext();
         }
-        return $result;
     }
 }
diff --git a/src/Magento/FunctionalTestingFramework/Exceptions/TestFrameworkException.php b/src/Magento/FunctionalTestingFramework/Exceptions/TestFrameworkException.php
index 5e9e594c0..a82eddef0 100644
--- a/src/Magento/FunctionalTestingFramework/Exceptions/TestFrameworkException.php
+++ b/src/Magento/FunctionalTestingFramework/Exceptions/TestFrameworkException.php
@@ -13,6 +13,13 @@
  */
 class TestFrameworkException extends \Exception
 {
+    /**
+     * Exception context
+     *
+     * @var array
+     */
+    protected $context;
+
     /**
      * TestFrameworkException constructor.
      * @param string $message
@@ -27,6 +34,17 @@ public function __construct($message, $context = [])
             $context
         );
 
+        $this->context = $context;
         parent::__construct($message);
     }
+
+    /**
+     * Return exception context
+     *
+     * @return array
+     */
+    public function getContext()
+    {
+        return $this->context;
+    }
 }
diff --git a/src/Magento/FunctionalTestingFramework/Module/MagentoWebDriver.php b/src/Magento/FunctionalTestingFramework/Module/MagentoWebDriver.php
index 2b41af99e..215438420 100644
--- a/src/Magento/FunctionalTestingFramework/Module/MagentoWebDriver.php
+++ b/src/Magento/FunctionalTestingFramework/Module/MagentoWebDriver.php
@@ -129,11 +129,6 @@ public function _initialize()
         $this->config = ConfigSanitizerUtil::sanitizeWebDriverConfig($this->config);
         parent::_initialize();
         $this->cleanJsError();
-
-        // Check Selenium Server readiness if it's in diagnostic phase
-        if (MftfApplicationConfig::getConfig()->getPhase() === MftfApplicationConfig::DIAGNOSTIC_PHASE) {
-            $this->checkSeleniumServerReadiness();
-        }
     }
 
     /**
@@ -834,32 +829,6 @@ public function makeScreenshot($name = null)
         AllureHelper::addAttachmentToCurrentStep($screenName, 'Screenshot');
     }
 
-    /**
-     * Check connectivity to running selenium server
-     *
-     * @return void
-     * @throws TestFrameworkException
-     */
-    public function checkSeleniumServerReadiness()
-    {
-        try {
-            $driver = RemoteWebDriver::create(
-                $this->wdHost,
-                $this->capabilities,
-                $this->connectionTimeoutInMs,
-                $this->requestTimeoutInMs,
-                $this->httpProxy,
-                $this->httpProxyPort
-            );
-            $driver->close();
-        } catch (WebDriverCurlException $e) {
-            throw new TestFrameworkException(
-                "Can't connect to Webdriver at {$this->wdHost}.\n"
-                . "Please make sure that Selenium Server is running."
-            );
-        }
-    }
-
     /**
      * Takes given $command and executes it against bin/magento executable. Returns stdout output from the command.
      *
diff --git a/src/Magento/FunctionalTestingFramework/Module/MagentoWebDriverDoctor.php b/src/Magento/FunctionalTestingFramework/Module/MagentoWebDriverDoctor.php
new file mode 100644
index 000000000..3acde4a42
--- /dev/null
+++ b/src/Magento/FunctionalTestingFramework/Module/MagentoWebDriverDoctor.php
@@ -0,0 +1,87 @@
+<?php
+/**
+ * Copyright © Magento, Inc. All rights reserved.
+ * See COPYING.txt for license details.
+ */
+
+namespace Magento\FunctionalTestingFramework\Module;
+
+use Magento\FunctionalTestingFramework\Exceptions\TestFrameworkException;
+use Facebook\WebDriver\Remote\RemoteWebDriver;
+
+/**
+ * MagentoWebDriverDoctor module extends MagentoWebDriver module and is a light weighted module to diagnose webdriver
+ * initialization and other setup issues. It uses in memory version of MagentoWebDriver's configuration file
+ */
+class MagentoWebDriverDoctor extends MagentoWebDriver
+{
+    const MAGENTO_CLI_COMMAND = 'list';
+    const EXCEPTION_TYPE_SELENIUM = 'selenium';
+    const EXCEPTION_TYPE_MAGENTO_CLI = 'cli';
+
+    /**
+     * Go through parent initialization routines and in addition diagnose potential environment issues
+     *
+     * @return void
+     * @throws TestFrameworkException
+     */
+    public function _initialize()
+    {
+        parent::_initialize();
+
+        $context = [];
+
+        try {
+            $this->checkSeleniumServerReadiness();
+        } catch (TestFrameworkException $e) {
+            $context[self::EXCEPTION_TYPE_SELENIUM] = $e->getMessage();
+        }
+
+        try {
+            $this->checkMagentoCLI();
+        } catch (TestFrameworkException $e) {
+            $context[self::EXCEPTION_TYPE_MAGENTO_CLI] = $e->getMessage();
+        }
+
+        if (!empty($context)) {
+            throw new TestFrameworkException('MagentoWebDriverDoctor initialization failed', $context);
+        }
+    }
+
+    /**
+     * Check connectivity to running selenium server
+     *
+     * @return void
+     * @throws TestFrameworkException
+     */
+    private function checkSeleniumServerReadiness()
+    {
+        try {
+            $driver = RemoteWebDriver::create(
+                $this->wdHost,
+                $this->capabilities,
+                $this->connectionTimeoutInMs,
+                $this->requestTimeoutInMs,
+                $this->httpProxy,
+                $this->httpProxyPort
+            );
+            $driver->close();
+        } catch (\Exception $e) {
+            throw new TestFrameworkException(
+                "Can't connect to Webdriver at {$this->wdHost}.\n"
+                . "Please make sure that Selenium Server is running."
+            );
+        }
+    }
+
+    /**
+     * Check Magento CLI setup
+     *
+     * @return void
+     * @throws TestFrameworkException
+     */
+    private function checkMagentoCLI()
+    {
+        parent::magentoCLI(self::MAGENTO_CLI_COMMAND);
+    }
+}

From 7c0dafd9cffef9f50b8b662c0b7dafb04d8cd6db Mon Sep 17 00:00:00 2001
From: Kevin Kozan <kkozan@adobe.com>
Date: Mon, 11 Nov 2019 08:58:26 -0600
Subject: [PATCH 27/81] MQE-1712: Get Error HTTP response code: 500 when using
 magentoCLI with command config:sensitive:set

- command.php no longer overwrites potentially useful output with empty output
---
 etc/config/command.php | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/etc/config/command.php b/etc/config/command.php
index 7b45a2595..adc372e43 100644
--- a/etc/config/command.php
+++ b/etc/config/command.php
@@ -31,7 +31,10 @@
                 $process->run();
                 $output = $process->getOutput();
                 if (!$process->isSuccessful()) {
-                    $output = $process->getErrorOutput();
+                    $failureOutput = $process->getErrorOutput();
+                    if (!empty($failureOutput)) {
+                        $output = $failureOutput;
+                    }
                 }
                 if (empty($output)) {
                     $output = "CLI did not return output.";

From bfe90fd3340a0bd051c00fd07ca7c4a7d54ac723 Mon Sep 17 00:00:00 2001
From: filmaj <maj.fil@gmail.com>
Date: Tue, 12 Nov 2019 16:40:55 -0500
Subject: [PATCH 28/81] expanding on credential usage docs to explicitly call
 out how to use credentials differently in File vs Vault Storage
 configurations.

---
 docs/credentials.md | 34 +++++++++++++++++++++-------------
 1 file changed, 21 insertions(+), 13 deletions(-)

diff --git a/docs/credentials.md b/docs/credentials.md
index 3065454b0..8aec0eacd 100644
--- a/docs/credentials.md
+++ b/docs/credentials.md
@@ -1,6 +1,6 @@
 # Credentials
 
-When you test functionality that involves external services such as UPS, FedEx, PayPal, or SignifyD, 
+When you test functionality that involves external services such as UPS, FedEx, PayPal, or SignifyD,
 use the MFTF credentials feature to hide sensitive [data][] like integration tokens and API keys.
 
 Currently the MFTF supports two types of credential storage:
@@ -53,7 +53,7 @@ magento/carriers_usps_password=Lmgxvrq89uPwECeV
 #magento/carriers_dhl_id_us=dhl_test_user
 #magento/carriers_dhl_password_us=Mlgxv3dsagVeG
 ....
-```  
+```
 
 Or add new key & value pairs for your own credentials. The keys use the following format:
 
@@ -64,7 +64,7 @@ Or add new key & value pairs for your own credentials. The keys use the followin
 <div class="bs-callout bs-callout-info" markdown="1">
 The `/` symbol is not supported in a `key_name` other than the one after your vendor or extension name.
 </div>
- 
+
 Otherwise you are free to use any other `key_name` you like, as they are merely the keys to reference from your tests.
 
 ```conf
@@ -74,10 +74,10 @@ vendor/my_awesome_service_token=rRVSVnh3cbDsVG39oTMz4A
 
 ## Configure Vault Storage
 
-Hashicorp vault secures, stores, and tightly controls access to data in modern computing. 
-It provides advanced data protection for your testing credentials. 
+Hashicorp vault secures, stores, and tightly controls access to data in modern computing.
+It provides advanced data protection for your testing credentials.
 
-The MFTF works with both `vault enterprise` and `vault open source` that use `KV Version 2` secret engine. 
+The MFTF works with both `vault enterprise` and `vault open source` that use `KV Version 2` secret engine.
 
 ### Install vault CLI
 
@@ -95,8 +95,8 @@ vault login -method -path
 
 ### Store secrets in vault
 
-The MFTF uses the `KV Version 2` secret engine for secret storage. 
-More information for working with `KV Version 2` can be found in [Vault KV2][Vault KV2]. 
+The MFTF uses the `KV Version 2` secret engine for secret storage.
+More information for working with `KV Version 2` can be found in [Vault KV2][Vault KV2].
 
 #### Secrets path and key convention
 
@@ -125,9 +125,9 @@ vault kv put secret/mftf/magento/carriers_usps_password carriers_usps_password=L
 
 ### Setup MFTF to use vault
 
-Add vault configuration environment variables [`CREDENTIAL_VAULT_ADDRESS`][] and [`CREDENTIAL_VAULT_SECRET_BASE_PATH`][] 
+Add vault configuration environment variables [`CREDENTIAL_VAULT_ADDRESS`][] and [`CREDENTIAL_VAULT_SECRET_BASE_PATH`][]
 from `etc/config/.env.example` in `.env`.
-Set values according to your vault server configuration.  
+Set values according to your vault server configuration.
 
 ```conf
 # Default vault dev server
@@ -137,7 +137,7 @@ CREDENTIAL_VAULT_SECRET_BASE_PATH=secret
 
 ## Configure both File Storage and Vault Storage
 
-It is possible and sometimes useful to setup and use both `.credentials` file and vault for secret storage at the same time. 
+It is possible and sometimes useful to setup and use both `.credentials` file and vault for secret storage at the same time.
 In this case, the MFTF tests are able to read secret data at runtime from both storage options, but the local `.credentials` file will take precedence.
 
 <!-- {% raw %} -->
@@ -150,11 +150,19 @@ Define the value as a reference to the corresponding key in the credentials file
 
 -  `_CREDS` is an environment constant pointing to the `.credentials` file
 -  `my_data_key` is a key in the the `.credentials` file or vault that contains the value to be used in a test step
+  - for File Storage, ensure your key contains the vendor prefix, i.e. `vendor/my_data_key`
+  - for Vault Storage, ensure your key contains your secret basepath + `/mftf/` + your vendor prefix + `/` and finally your key, i.e. `secret/mftf/vendor/my_data_key`
+
+For example, to reference secret data in the [`fillField`][] action, use the `userInput` attribute using a typical File Storage:
+
+```xml
+<fillField stepKey="FillApiToken" selector=".api-token" userInput="{{_CREDS.vendor/my_data_key}}" />
+```
 
-For example, reference secret data in the [`fillField`][] action with the `userInput` attribute.
+... whereas to do the same using Vault Storage:
 
 ```xml
-<fillField stepKey="FillApiToken" selector=".api-token" userInput="{{_CREDS.my_data_key}}" />
+<fillField stepKey="FillApiToken" selector=".api-token" userInput="{{_CREDS.secret/mftf/vendor/my_data_key}}" />
 ```
 
 <!-- {% endraw %} -->

From e894f75e9feb828859eb8d22eca3853158dd2d82 Mon Sep 17 00:00:00 2001
From: Donald Booth <dobooth@adobe.com>
Date: Tue, 12 Nov 2019 16:03:21 -0600
Subject: [PATCH 29/81] Grammar fixup.

---
 docs/credentials.md | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/docs/credentials.md b/docs/credentials.md
index 8aec0eacd..3cc765a45 100644
--- a/docs/credentials.md
+++ b/docs/credentials.md
@@ -55,7 +55,7 @@ magento/carriers_usps_password=Lmgxvrq89uPwECeV
 ....
 ```
 
-Or add new key & value pairs for your own credentials. The keys use the following format:
+Or add new key/value pairs for your own credentials. The keys use the following format:
 
 ```conf
 <vendor>/<key_name>=<key_value>
@@ -150,8 +150,8 @@ Define the value as a reference to the corresponding key in the credentials file
 
 -  `_CREDS` is an environment constant pointing to the `.credentials` file
 -  `my_data_key` is a key in the the `.credentials` file or vault that contains the value to be used in a test step
-  - for File Storage, ensure your key contains the vendor prefix, i.e. `vendor/my_data_key`
-  - for Vault Storage, ensure your key contains your secret basepath + `/mftf/` + your vendor prefix + `/` and finally your key, i.e. `secret/mftf/vendor/my_data_key`
+   - for File Storage, ensure your key contains the vendor prefix, i.e. `vendor/my_data_key`
+   - for Vault Storage, ensure your key contains your secret basepath + `/mftf/` + your vendor prefix + `/` and finally your key, i.e. `secret/mftf/vendor/my_data_key`
 
 For example, to reference secret data in the [`fillField`][] action, use the `userInput` attribute using a typical File Storage:
 
@@ -159,7 +159,7 @@ For example, to reference secret data in the [`fillField`][] action, use the `us
 <fillField stepKey="FillApiToken" selector=".api-token" userInput="{{_CREDS.vendor/my_data_key}}" />
 ```
 
-... whereas to do the same using Vault Storage:
+To do the same using Vault Storage:
 
 ```xml
 <fillField stepKey="FillApiToken" selector=".api-token" userInput="{{_CREDS.secret/mftf/vendor/my_data_key}}" />

From 6dd9655efae9ac00f104d52f1499246d721a6917 Mon Sep 17 00:00:00 2001
From: filmaj <maj.fil@gmail.com>
Date: Wed, 13 Nov 2019 15:28:12 -0500
Subject: [PATCH 30/81] remove vault-based credentials usage details for now

---
 docs/credentials.md | 7 -------
 1 file changed, 7 deletions(-)

diff --git a/docs/credentials.md b/docs/credentials.md
index 3cc765a45..a2850cfe8 100644
--- a/docs/credentials.md
+++ b/docs/credentials.md
@@ -151,7 +151,6 @@ Define the value as a reference to the corresponding key in the credentials file
 -  `_CREDS` is an environment constant pointing to the `.credentials` file
 -  `my_data_key` is a key in the the `.credentials` file or vault that contains the value to be used in a test step
    - for File Storage, ensure your key contains the vendor prefix, i.e. `vendor/my_data_key`
-   - for Vault Storage, ensure your key contains your secret basepath + `/mftf/` + your vendor prefix + `/` and finally your key, i.e. `secret/mftf/vendor/my_data_key`
 
 For example, to reference secret data in the [`fillField`][] action, use the `userInput` attribute using a typical File Storage:
 
@@ -159,12 +158,6 @@ For example, to reference secret data in the [`fillField`][] action, use the `us
 <fillField stepKey="FillApiToken" selector=".api-token" userInput="{{_CREDS.vendor/my_data_key}}" />
 ```
 
-To do the same using Vault Storage:
-
-```xml
-<fillField stepKey="FillApiToken" selector=".api-token" userInput="{{_CREDS.secret/mftf/vendor/my_data_key}}" />
-```
-
 <!-- {% endraw %} -->
 
 ## Implementation details

From 467c1570e80577dfe6fb815dee2396f75828475e Mon Sep 17 00:00:00 2001
From: Soumya Unnikrishnan <sunnikri@adobe.com>
Date: Thu, 14 Nov 2019 16:50:41 -0600
Subject: [PATCH 31/81] MQE-1884: MFTF - <after> failures override other
 failures attaching suppressed exception before _after hook failure to current
 step.

---
 .../Extension/TestContextExtension.php        | 31 ++++++++++++++++++-
 1 file changed, 30 insertions(+), 1 deletion(-)

diff --git a/src/Magento/FunctionalTestingFramework/Extension/TestContextExtension.php b/src/Magento/FunctionalTestingFramework/Extension/TestContextExtension.php
index 9fe7adb7a..6e738418f 100644
--- a/src/Magento/FunctionalTestingFramework/Extension/TestContextExtension.php
+++ b/src/Magento/FunctionalTestingFramework/Extension/TestContextExtension.php
@@ -64,6 +64,8 @@ public function testStart()
     public function testFail(\Codeception\Event\FailEvent $e)
     {
         $cest = $e->getTest();
+        //log suppressed exception in case of _after hook failure
+        $this->logPreviousException($e->getFail());
         $context = $this->extractContext($e->getFail()->getTrace(), $cest->getTestMethod());
         // Do not attempt to run _after if failure was in the _after block
         // Try to run _after but catch exceptions to prevent them from overwriting original failure.
@@ -93,7 +95,9 @@ function () use ($cest) {
         if (!empty($errors)) {
             foreach ($errors as $error) {
                 if ($error->failedTest()->getTestMethod() == $cest->getName()) {
-                    $stack = $errors[0]->thrownException()->getTrace();
+                    //log suppressed exception in case of _after hook failure
+                    $this->logPreviousException($error->thrownException());
+                    $stack = $error->thrownException()->getTrace();
                     $context = $this->extractContext($stack, $cest->getTestMethod());
                     // Do not attempt to run _after if failure was in the _after block
                     // Try to run _after but catch exceptions to prevent them from overwriting original failure.
@@ -150,6 +154,31 @@ public function extractContext($trace, $class)
         return null;
     }
 
+    /**
+     * Attach suppressed exception thrown before _after hook to the current step.
+     * @param  \Exception $exception
+     * @return mixed
+     */
+    public function logPreviousException(\Exception $exception)
+    {
+        $change = function(){
+            if ($this instanceof \PHPUnit\Framework\ExceptionWrapper ) {
+                return $this->previous;
+            }
+            else {
+                return $this->getPrevious();
+            }
+        };
+        $firstException = $change->call($exception);
+        $bind = function() use ($firstException){
+            $exception = $firstException;
+        };
+        $bind->call($exception);
+        if ($firstException !== null) {
+            AllureHelper::addAttachmentToCurrentStep($firstException, 'Exception');
+        }
+    }
+
     /**
      * Codeception event listener function, triggered before step.
      * Check if it's a new page.

From 722f3123d1852d9f794f647ac03e6fe29dcb3b46 Mon Sep 17 00:00:00 2001
From: Soumya Unnikrishnan <sunnikri@adobe.com>
Date: Thu, 14 Nov 2019 16:59:58 -0600
Subject: [PATCH 32/81] MQE-1884: MFTF - <after> failures override other
 failures fixed static checks

---
 .../Extension/TestContextExtension.php                   | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/src/Magento/FunctionalTestingFramework/Extension/TestContextExtension.php b/src/Magento/FunctionalTestingFramework/Extension/TestContextExtension.php
index 6e738418f..be7e2ca39 100644
--- a/src/Magento/FunctionalTestingFramework/Extension/TestContextExtension.php
+++ b/src/Magento/FunctionalTestingFramework/Extension/TestContextExtension.php
@@ -161,16 +161,15 @@ public function extractContext($trace, $class)
      */
     public function logPreviousException(\Exception $exception)
     {
-        $change = function(){
-            if ($this instanceof \PHPUnit\Framework\ExceptionWrapper ) {
+        $change = function () {
+            if ($this instanceof \PHPUnit\Framework\ExceptionWrapper) {
                 return $this->previous;
-            }
-            else {
+            } else {
                 return $this->getPrevious();
             }
         };
         $firstException = $change->call($exception);
-        $bind = function() use ($firstException){
+        $bind = function () use ($firstException) {
             $exception = $firstException;
         };
         $bind->call($exception);

From 960320a4a80889d707b6dbfd12e80a26059bb37a Mon Sep 17 00:00:00 2001
From: Soumya Unnikrishnan <sunnikri@adobe.com>
Date: Fri, 15 Nov 2019 08:29:14 -0600
Subject: [PATCH 33/81] MQE-1884: MFTF - <after> failures override other
 failures fixed static checks

---
 .../Extension/TestContextExtension.php                           | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/Magento/FunctionalTestingFramework/Extension/TestContextExtension.php b/src/Magento/FunctionalTestingFramework/Extension/TestContextExtension.php
index be7e2ca39..276e2e1c2 100644
--- a/src/Magento/FunctionalTestingFramework/Extension/TestContextExtension.php
+++ b/src/Magento/FunctionalTestingFramework/Extension/TestContextExtension.php
@@ -13,6 +13,7 @@
 /**
  * Class TestContextExtension
  * @SuppressWarnings(PHPMD.UnusedPrivateField)
+ * @SuppressWarnings(PHPMD.CouplingBetweenObjects)
  */
 class TestContextExtension extends BaseExtension
 {

From 25c5706cc2df7384de31b6804fc5d76b120a347c Mon Sep 17 00:00:00 2001
From: Ji Lu <jilu1@adobe.com>
Date: Thu, 14 Nov 2019 15:46:24 -0600
Subject: [PATCH 34/81] MQE-1257: MFTF doctor command

---
 docs/commands/mftf.md                         |  14 +++
 .../Console/DoctorCommand.php                 |  64 ++++++-----
 .../Module/MagentoWebDriverDoctor.php         | 108 +++++++++++++++---
 3 files changed, 139 insertions(+), 47 deletions(-)

diff --git a/docs/commands/mftf.md b/docs/commands/mftf.md
index 58be6501e..5e9895958 100644
--- a/docs/commands/mftf.md
+++ b/docs/commands/mftf.md
@@ -109,6 +109,20 @@ You can include options to set configuration parameter values for your environme
 vendor/bin/mftf build:project --MAGENTO_BASE_URL=http://magento.local/ --MAGENTO_BACKEND_NAME=admin214365
 ```
 
+### `doctor`
+
+#### Description
+
+Diagnose MFTF configuration and setup
+
+#### Usage
+
+```bash
+vendor/bin/mftf doctor
+```
+
+#### Options
+
 ### `generate:tests`
 
 #### Description
diff --git a/src/Magento/FunctionalTestingFramework/Console/DoctorCommand.php b/src/Magento/FunctionalTestingFramework/Console/DoctorCommand.php
index 469d4ac47..2caa47437 100644
--- a/src/Magento/FunctionalTestingFramework/Console/DoctorCommand.php
+++ b/src/Magento/FunctionalTestingFramework/Console/DoctorCommand.php
@@ -67,6 +67,7 @@ protected function execute(InputInterface $input, OutputInterface $output)
 
         // Config application
         $verbose = $output->isVerbose();
+        $this->input = $input;
         $this->output = $output;
         MftfApplicationConfig::create(
             false,
@@ -81,11 +82,31 @@ protected function execute(InputInterface $input, OutputInterface $output)
         $cmdStatus = $cmdStatus && !$status ? false : $cmdStatus;
 
         // Check connection to Selenium
-        $status = $this->checkConnectionToSeleniumServer();
+        $status = $this->checkContextOnStep(
+            MagentoWebDriverDoctor::EXCEPTION_CONTEXT_SELENIUM,
+            'Connecting to Selenium Server'
+        );
+        $cmdStatus = $cmdStatus && !$status ? false : $cmdStatus;
+
+        // Check opening Magento Admin in web browser
+        $status = $this->checkContextOnStep(
+            MagentoWebDriverDoctor::EXCEPTION_CONTEXT_ADMIN,
+            'Loading Admin page'
+        );
+        $cmdStatus = $cmdStatus && !$status ? false : $cmdStatus;
+
+        // Check opening Magento Storefront in web browser
+        $status = $this->checkContextOnStep(
+            MagentoWebDriverDoctor::EXCEPTION_CONTEXT_STOREFRONT,
+            'Loading Storefront page'
+        );
         $cmdStatus = $cmdStatus && !$status ? false : $cmdStatus;
 
         // Check access to Magento CLI
-        $status = $this->checkAccessToMagentoCLI();
+        $status = $this->checkContextOnStep(
+            MagentoWebDriverDoctor::EXCEPTION_CONTEXT_CLI,
+            'Running Magento CLI'
+        );
         $cmdStatus = $cmdStatus && !$status ? false : $cmdStatus;
 
         if ($cmdStatus) {
@@ -96,7 +117,7 @@ protected function execute(InputInterface $input, OutputInterface $output)
     }
 
     /**
-     * Check authentication to Magento Admin
+     * Check admin account authentication
      *
      * @return boolean
      */
@@ -104,7 +125,7 @@ private function checkAuthenticationToMagentoAdmin()
     {
         $result = false;
         try {
-            $this->output->writeln("\nChecking authentication to Magento Admin ...");
+            $this->output->writeln("\nAuthenticating admin account by API ...");
             ModuleResolver::getInstance()->getAdminToken();
             $this->output->writeln('Successful');
             $result = true;
@@ -115,38 +136,20 @@ private function checkAuthenticationToMagentoAdmin()
     }
 
     /**
-     * Check Connection to Selenium Server
-     *
-     * @return boolean
-     */
-    private function checkConnectionToSeleniumServer()
-    {
-        // Check connection to Selenium through Codeception
-        $this->output->writeln("\nChecking connection to Selenium Server ...");
-        $this->runMagentoWebDriverDoctor();
-
-        if (isset($this->context[MagentoWebDriverDoctor::EXCEPTION_TYPE_SELENIUM])) {
-            $this->output->write($this->context[MagentoWebDriverDoctor::EXCEPTION_TYPE_SELENIUM] . "\n");
-            return false;
-        } else {
-            $this->output->writeln('Successful');
-            return true;
-        }
-    }
-
-    /**
-     * Check access to Magento CLI setup
+     * Check exception context after runMagentoWebDriverDoctor
      *
+     * @param string $exceptionType
+     * @param string $message
      * @return boolean
+     * @throws TestFrameworkException
      */
-    private function checkAccessToMagentoCLI()
+    private function checkContextOnStep($exceptionType, $message)
     {
-        // Check Magento CLI setup
-        $this->output->writeln("\nChecking access to Magento CLI ...");
+        $this->output->writeln("\n$message ...");
         $this->runMagentoWebDriverDoctor();
 
-        if (isset($this->context[MagentoWebDriverDoctor::EXCEPTION_TYPE_MAGENTO_CLI])) {
-            $this->output->write($this->context[MagentoWebDriverDoctor::EXCEPTION_TYPE_MAGENTO_CLI] . "\n");
+        if (isset($this->context[$exceptionType])) {
+            $this->output->write($this->context[$exceptionType] . "\n");
             return false;
         } else {
             $this->output->writeln('Successful');
@@ -158,6 +161,7 @@ private function checkAccessToMagentoCLI()
      * Run diagnose through MagentoWebDriverDoctor
      *
      * @return void
+     * @throws TestFrameworkException
      */
     private function runMagentoWebDriverDoctor()
     {
diff --git a/src/Magento/FunctionalTestingFramework/Module/MagentoWebDriverDoctor.php b/src/Magento/FunctionalTestingFramework/Module/MagentoWebDriverDoctor.php
index 3acde4a42..35fa37f03 100644
--- a/src/Magento/FunctionalTestingFramework/Module/MagentoWebDriverDoctor.php
+++ b/src/Magento/FunctionalTestingFramework/Module/MagentoWebDriverDoctor.php
@@ -11,13 +11,22 @@
 
 /**
  * MagentoWebDriverDoctor module extends MagentoWebDriver module and is a light weighted module to diagnose webdriver
- * initialization and other setup issues. It uses in memory version of MagentoWebDriver's configuration file
+ * initialization and other setup issues. It uses in memory version of MagentoWebDriver's configuration file.
  */
 class MagentoWebDriverDoctor extends MagentoWebDriver
 {
     const MAGENTO_CLI_COMMAND = 'list';
-    const EXCEPTION_TYPE_SELENIUM = 'selenium';
-    const EXCEPTION_TYPE_MAGENTO_CLI = 'cli';
+    const EXCEPTION_CONTEXT_SELENIUM = 'selenium';
+    const EXCEPTION_CONTEXT_ADMIN = 'admin';
+    const EXCEPTION_CONTEXT_STOREFRONT = 'store';
+    const EXCEPTION_CONTEXT_CLI = 'cli';
+
+    /**
+     * Remote Web Driver
+     *
+     * @var RemoteWebDriver
+     */
+    private $remoteWebDriver = null;
 
     /**
      * Go through parent initialization routines and in addition diagnose potential environment issues
@@ -32,32 +41,52 @@ public function _initialize()
         $context = [];
 
         try {
-            $this->checkSeleniumServerReadiness();
+            $this->connectToSeleniumServer();
         } catch (TestFrameworkException $e) {
-            $context[self::EXCEPTION_TYPE_SELENIUM] = $e->getMessage();
+            $context[self::EXCEPTION_CONTEXT_SELENIUM] = $e->getMessage();
         }
 
         try {
-            $this->checkMagentoCLI();
-        } catch (TestFrameworkException $e) {
-            $context[self::EXCEPTION_TYPE_MAGENTO_CLI] = $e->getMessage();
+            $adminUrl = rtrim(getenv('MAGENTO_BACKEND_BASE_URL'), '/')
+                ?: rtrim(getenv('MAGENTO_BASE_URL'), '/')
+                . '/' . getenv('MAGENTO_BACKEND_NAME') . '/admin';
+            $this->loadPageAtUrl($adminUrl);
+        } catch (\Exception $e) {
+            $context[self::EXCEPTION_CONTEXT_ADMIN] = $e->getMessage();
+        }
+
+        try {
+            $storeUrl = getenv('MAGENTO_BASE_URL');
+            $this->loadPageAtUrl($storeUrl);
+        } catch (\Exception $e) {
+            $context[self::EXCEPTION_CONTEXT_STOREFRONT] = $e->getMessage();
+        }
+
+        try {
+            $this->runMagentoCLI();
+        } catch (\Exception $e) {
+            $context[self::EXCEPTION_CONTEXT_CLI] = $e->getMessage();
+        }
+
+        if (null !== $this->remoteWebDriver) {
+            $this->remoteWebDriver->close();
         }
 
         if (!empty($context)) {
-            throw new TestFrameworkException('MagentoWebDriverDoctor initialization failed', $context);
+            throw new TestFrameworkException('Exception occurred in MagentoWebDriverDoctor', $context);
         }
     }
 
     /**
-     * Check connectivity to running selenium server
+     * Check connecting to running selenium server
      *
      * @return void
      * @throws TestFrameworkException
      */
-    private function checkSeleniumServerReadiness()
+    private function connectToSeleniumServer()
     {
         try {
-            $driver = RemoteWebDriver::create(
+            $this->remoteWebDriver = RemoteWebDriver::create(
                 $this->wdHost,
                 $this->capabilities,
                 $this->connectionTimeoutInMs,
@@ -65,23 +94,68 @@ private function checkSeleniumServerReadiness()
                 $this->httpProxy,
                 $this->httpProxyPort
             );
-            $driver->close();
         } catch (\Exception $e) {
             throw new TestFrameworkException(
-                "Can't connect to Webdriver at {$this->wdHost}.\n"
+                "Failed to connect Selenium WebDriver at: {$this->wdHost}.\n"
                 . "Please make sure that Selenium Server is running."
             );
         }
     }
 
     /**
-     * Check Magento CLI setup
+     * Validate loading a web page at url in the browser controlled by selenium
+     *
+     * @param string $url
+     * @return void
+     * @throws TestFrameworkException
+     */
+    private function loadPageAtUrl($url)
+    {
+        try {
+            // Open the web page at url first
+            $this->remoteWebDriver->get($url);
+
+            // Execute Javascript to retrieve HTTP response code
+            $script = ''
+                . 'var xhr = new XMLHttpRequest();'
+                . "xhr.open('GET', '" . $url . "', false);"
+                . 'xhr.send(null); '
+                . 'return xhr.status';
+            $status = $this->remoteWebDriver->executeScript($script);
+
+            if ($status === 200) {
+                return;
+            }
+        } catch (\Exception $e) {
+        }
+
+        throw new TestFrameworkException(
+            "Failed to load page at url: $url\n"
+            . "Please check network connection for the browser running Selenium."
+        );
+    }
+
+    /**
+     * Check running Magento CLI command
      *
      * @return void
      * @throws TestFrameworkException
      */
-    private function checkMagentoCLI()
+    private function runMagentoCLI()
     {
-        parent::magentoCLI(self::MAGENTO_CLI_COMMAND);
+        try {
+            $regex = '~^.*(?<name>Magento CLI).*[\r\n]+(?<usage>Usage:).*~';
+            $output = parent::magentoCLI(self::MAGENTO_CLI_COMMAND);
+            preg_match($regex, $output, $matches);
+
+            if (isset($matches['name']) && isset($matches['usage'])) {
+                return;
+            }
+        } catch (\Exception $e) {
+            throw new TestFrameworkException(
+                "Failed to run Magento CLI command\n"
+                . "Please reference Magento DevDoc to setup command.php and .htaccess files."
+            );
+        }
     }
 }

From 4908777736d79ab58d04c72035957b38469fb3d6 Mon Sep 17 00:00:00 2001
From: Ji Lu <jilu1@adobe.com>
Date: Fri, 15 Nov 2019 15:45:33 -0600
Subject: [PATCH 35/81] MQE-1257: MFTF doctor command

- config color for console output
---
 .../Console/DoctorCommand.php                 | 25 +++++++++++--------
 1 file changed, 14 insertions(+), 11 deletions(-)

diff --git a/src/Magento/FunctionalTestingFramework/Console/DoctorCommand.php b/src/Magento/FunctionalTestingFramework/Console/DoctorCommand.php
index 2caa47437..d75004fc2 100644
--- a/src/Magento/FunctionalTestingFramework/Console/DoctorCommand.php
+++ b/src/Magento/FunctionalTestingFramework/Console/DoctorCommand.php
@@ -24,6 +24,10 @@ class DoctorCommand extends Command
     const CODECEPTION_AUTOLOAD_FILE = PROJECT_ROOT . '/vendor/codeception/codeception/autoload.php';
     const MFTF_CODECEPTION_CONFIG_FILE = ENV_FILE_PATH . 'codeception.yml';
     const SUITE = 'functional';
+    const COLOR_LIGHT_GREEN = "\e[1;32m";
+    const COLOR_LIGHT_RED = "\e[1;31m";
+    const COLOR_LIGHT_DEFAULT = "\e[1;39m";
+    const COLOR_RESTORE = "\e[0m";
 
     /**
      * Command Output
@@ -60,6 +64,7 @@ protected function configure()
      * @return integer
      * @throws TestFrameworkException
      * @SuppressWarnings(PHPMD.UnusedFormalParameter)
+     * @SuppressWarnings(PHPMD.CyclomaticComplexity)
      */
     protected function execute(InputInterface $input, OutputInterface $output)
     {
@@ -109,11 +114,7 @@ protected function execute(InputInterface $input, OutputInterface $output)
         );
         $cmdStatus = $cmdStatus && !$status ? false : $cmdStatus;
 
-        if ($cmdStatus) {
-            return 0;
-        } else {
-            return 1;
-        }
+        return $cmdStatus ? 0 : 1;
     }
 
     /**
@@ -125,12 +126,14 @@ private function checkAuthenticationToMagentoAdmin()
     {
         $result = false;
         try {
-            $this->output->writeln("\nAuthenticating admin account by API ...");
+            $this->output->writeln(
+                "\n" . self::COLOR_LIGHT_DEFAULT . "Authenticating admin account by API ..." . self::COLOR_RESTORE
+            );
             ModuleResolver::getInstance()->getAdminToken();
-            $this->output->writeln('Successful');
+            $this->output->writeln(self::COLOR_LIGHT_GREEN . 'Successful' . self::COLOR_RESTORE);
             $result = true;
         } catch (TestFrameworkException $e) {
-            $this->output->writeln($e->getMessage());
+            $this->output->writeln(self::COLOR_LIGHT_RED . $e->getMessage() . self::COLOR_RESTORE);
         }
         return $result;
     }
@@ -145,14 +148,14 @@ private function checkAuthenticationToMagentoAdmin()
      */
     private function checkContextOnStep($exceptionType, $message)
     {
-        $this->output->writeln("\n$message ...");
+        $this->output->writeln("\n" . self::COLOR_LIGHT_DEFAULT. $message . self::COLOR_RESTORE);
         $this->runMagentoWebDriverDoctor();
 
         if (isset($this->context[$exceptionType])) {
-            $this->output->write($this->context[$exceptionType] . "\n");
+            $this->output->writeln(self::COLOR_LIGHT_RED . $this->context[$exceptionType] . self::COLOR_RESTORE);
             return false;
         } else {
-            $this->output->writeln('Successful');
+            $this->output->writeln(self::COLOR_LIGHT_GREEN . 'Successful' . self::COLOR_RESTORE);
             return true;
         }
     }

From db8712bf5a46402b893fa950f8a80f52e4769591 Mon Sep 17 00:00:00 2001
From: Soumya Unnikrishnan <sunnikri@adobe.com>
Date: Mon, 18 Nov 2019 11:43:22 -0600
Subject: [PATCH 36/81] MQE-1884: MFTF - <after> failures override other
 failures bumped up codeception version to 2.4.5 to be inline with magento
 version. Removed method runAfterBlock, 2.4.5 runs _after hook implicitly.

---
 composer.json                                 |  2 +-
 composer.lock                                 | 84 ++++++++++++++-----
 .../Extension/TestContextExtension.php        | 44 ----------
 3 files changed, 63 insertions(+), 67 deletions(-)

diff --git a/composer.json b/composer.json
index f980005c1..3e3b0ab42 100755
--- a/composer.json
+++ b/composer.json
@@ -12,7 +12,7 @@
         "php": "7.0.2||7.0.4||~7.0.6||~7.1.0||~7.2.0||~7.3.0",
         "ext-curl": "*",
         "allure-framework/allure-codeception": "~1.3.0",
-        "codeception/codeception": "~2.3.4 || ~2.4.0 ",
+        "codeception/codeception": "~2.4.5",
         "composer/composer": "^1.6",
         "consolidation/robo": "^1.0.0",
         "csharpru/vault-php": "~3.5.3",
diff --git a/composer.lock b/composer.lock
index 275ae9778..5c2ba8c2d 100644
--- a/composer.lock
+++ b/composer.lock
@@ -4,7 +4,7 @@
         "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies",
         "This file is @generated automatically"
     ],
-    "content-hash": "beb8473a3c21b83da864289149fc03b4",
+    "content-hash": "1b0ef9b803188c23577751eba2a3d966",
     "packages": [
         {
             "name": "allure-framework/allure-codeception",
@@ -263,31 +263,28 @@
         },
         {
             "name": "codeception/codeception",
-            "version": "2.3.9",
+            "version": "2.4.5",
             "source": {
                 "type": "git",
                 "url": "https://github.com/Codeception/Codeception.git",
-                "reference": "104f46fa0bde339f1bcc3a375aac21eb36e65a1e"
+                "reference": "5fee32d5c82791548931cbc34806b4de6aa1abfc"
             },
             "dist": {
                 "type": "zip",
-                "url": "https://api.github.com/repos/Codeception/Codeception/zipball/104f46fa0bde339f1bcc3a375aac21eb36e65a1e",
-                "reference": "104f46fa0bde339f1bcc3a375aac21eb36e65a1e",
+                "url": "https://api.github.com/repos/Codeception/Codeception/zipball/5fee32d5c82791548931cbc34806b4de6aa1abfc",
+                "reference": "5fee32d5c82791548931cbc34806b4de6aa1abfc",
                 "shasum": ""
             },
             "require": {
-                "behat/gherkin": "~4.4.0",
-                "codeception/stub": "^1.0",
+                "behat/gherkin": "^4.4.0",
+                "codeception/phpunit-wrapper": "^6.0.9|^7.0.6",
+                "codeception/stub": "^2.0",
                 "ext-json": "*",
                 "ext-mbstring": "*",
                 "facebook/webdriver": ">=1.1.3 <2.0",
                 "guzzlehttp/guzzle": ">=4.1.4 <7.0",
                 "guzzlehttp/psr7": "~1.0",
-                "php": ">=5.4.0 <8.0",
-                "phpunit/php-code-coverage": ">=2.2.4 <6.0",
-                "phpunit/phpunit": ">=4.8.28 <5.0.0 || >=5.6.3 <7.0",
-                "sebastian/comparator": ">1.1 <3.0",
-                "sebastian/diff": ">=1.4 <3.0",
+                "php": ">=5.6.0 <8.0",
                 "symfony/browser-kit": ">=2.7 <5.0",
                 "symfony/console": ">=2.7 <5.0",
                 "symfony/css-selector": ">=2.7 <5.0",
@@ -353,27 +350,70 @@
                 "functional testing",
                 "unit testing"
             ],
-            "time": "2018-02-26T23:29:41+00:00"
+            "time": "2018-08-01T07:21:49+00:00"
         },
         {
-            "name": "codeception/stub",
-            "version": "1.0.4",
+            "name": "codeception/phpunit-wrapper",
+            "version": "6.7.0",
             "source": {
                 "type": "git",
-                "url": "https://github.com/Codeception/Stub.git",
-                "reference": "681b62348837a5ef07d10d8a226f5bc358cc8805"
+                "url": "https://github.com/Codeception/phpunit-wrapper.git",
+                "reference": "93f59e028826464eac086052fa226e58967f6907"
             },
             "dist": {
                 "type": "zip",
-                "url": "https://api.github.com/repos/Codeception/Stub/zipball/681b62348837a5ef07d10d8a226f5bc358cc8805",
-                "reference": "681b62348837a5ef07d10d8a226f5bc358cc8805",
+                "url": "https://api.github.com/repos/Codeception/phpunit-wrapper/zipball/93f59e028826464eac086052fa226e58967f6907",
+                "reference": "93f59e028826464eac086052fa226e58967f6907",
                 "shasum": ""
             },
             "require": {
-                "phpunit/phpunit-mock-objects": ">2.3 <7.0"
+                "phpunit/php-code-coverage": ">=4.0.4 <6.0",
+                "phpunit/phpunit": ">=6.5.13 <7.0",
+                "sebastian/comparator": ">=1.2.4 <3.0",
+                "sebastian/diff": ">=1.4 <4.0"
+            },
+            "replace": {
+                "codeception/phpunit-wrapper": "*"
             },
             "require-dev": {
-                "phpunit/phpunit": ">=4.8 <8.0"
+                "codeception/specify": "*",
+                "vlucas/phpdotenv": "^3.0"
+            },
+            "type": "library",
+            "autoload": {
+                "psr-4": {
+                    "Codeception\\PHPUnit\\": "src\\"
+                }
+            },
+            "notification-url": "https://packagist.org/downloads/",
+            "license": [
+                "MIT"
+            ],
+            "authors": [
+                {
+                    "name": "Davert",
+                    "email": "davert.php@resend.cc"
+                }
+            ],
+            "description": "PHPUnit classes used by Codeception",
+            "time": "2019-08-18T15:43:35+00:00"
+        },
+        {
+            "name": "codeception/stub",
+            "version": "2.1.0",
+            "source": {
+                "type": "git",
+                "url": "https://github.com/Codeception/Stub.git",
+                "reference": "853657f988942f7afb69becf3fd0059f192c705a"
+            },
+            "dist": {
+                "type": "zip",
+                "url": "https://api.github.com/repos/Codeception/Stub/zipball/853657f988942f7afb69becf3fd0059f192c705a",
+                "reference": "853657f988942f7afb69becf3fd0059f192c705a",
+                "shasum": ""
+            },
+            "require": {
+                "codeception/phpunit-wrapper": ">6.0.15 <6.1.0 | ^6.6.1 | ^7.7.1 | ^8.0.3"
             },
             "type": "library",
             "autoload": {
@@ -386,7 +426,7 @@
                 "MIT"
             ],
             "description": "Flexible Stub wrapper for PHPUnit's Mock Builder",
-            "time": "2018-05-17T09:31:08+00:00"
+            "time": "2019-03-02T15:35:10+00:00"
         },
         {
             "name": "composer/ca-bundle",
diff --git a/src/Magento/FunctionalTestingFramework/Extension/TestContextExtension.php b/src/Magento/FunctionalTestingFramework/Extension/TestContextExtension.php
index 276e2e1c2..b0f42590f 100644
--- a/src/Magento/FunctionalTestingFramework/Extension/TestContextExtension.php
+++ b/src/Magento/FunctionalTestingFramework/Extension/TestContextExtension.php
@@ -18,7 +18,6 @@
 class TestContextExtension extends BaseExtension
 {
     const TEST_PHASE_AFTER = "_after";
-    const CODECEPT_AFTER_VERSION = "2.3.9";
     const TEST_FAILED_FILE = 'failed';
 
     /**
@@ -64,15 +63,8 @@ public function testStart()
      */
     public function testFail(\Codeception\Event\FailEvent $e)
     {
-        $cest = $e->getTest();
         //log suppressed exception in case of _after hook failure
         $this->logPreviousException($e->getFail());
-        $context = $this->extractContext($e->getFail()->getTrace(), $cest->getTestMethod());
-        // Do not attempt to run _after if failure was in the _after block
-        // Try to run _after but catch exceptions to prevent them from overwriting original failure.
-        if ($context != TestContextExtension::TEST_PHASE_AFTER) {
-            $this->runAfterBlock($e, $cest);
-        }
     }
 
     /**
@@ -98,13 +90,6 @@ function () use ($cest) {
                 if ($error->failedTest()->getTestMethod() == $cest->getName()) {
                     //log suppressed exception in case of _after hook failure
                     $this->logPreviousException($error->thrownException());
-                    $stack = $error->thrownException()->getTrace();
-                    $context = $this->extractContext($stack, $cest->getTestMethod());
-                    // Do not attempt to run _after if failure was in the _after block
-                    // Try to run _after but catch exceptions to prevent them from overwriting original failure.
-                    if ($context != TestContextExtension::TEST_PHASE_AFTER) {
-                        $this->runAfterBlock($e, $cest);
-                    }
                     continue;
                 }
             }
@@ -113,31 +98,6 @@ function () use ($cest) {
         $this->getDriver()->_runAfter($e->getTest());
     }
 
-    /**
-     * Runs cest's after block, if necessary.
-     * @param \Symfony\Component\EventDispatcher\Event $e
-     * @param \Codeception\TestInterface               $cest
-     * @return void
-     */
-    private function runAfterBlock($e, $cest)
-    {
-        try {
-            $actorClass = $e->getTest()->getMetadata()->getCurrent('actor');
-            $I = new $actorClass($cest->getScenario());
-            if (version_compare(\Codeception\Codecept::VERSION, TestContextExtension::CODECEPT_AFTER_VERSION, "<=")) {
-                call_user_func(\Closure::bind(
-                    function () use ($cest, $I) {
-                        $cest->executeHook($I, 'after');
-                    },
-                    null,
-                    $cest
-                ));
-            }
-        } catch (\Exception $e) {
-            // Do not rethrow Exception
-        }
-    }
-
     /**
      * Extracts hook method from trace, looking specifically for the cest class given.
      * @param array  $trace
@@ -170,10 +130,6 @@ public function logPreviousException(\Exception $exception)
             }
         };
         $firstException = $change->call($exception);
-        $bind = function () use ($firstException) {
-            $exception = $firstException;
-        };
-        $bind->call($exception);
         if ($firstException !== null) {
             AllureHelper::addAttachmentToCurrentStep($firstException, 'Exception');
         }

From 78d396f89c8c0f44d4107efd3cdbf7c3d731aa94 Mon Sep 17 00:00:00 2001
From: Soumya Unnikrishnan <sunnikri@adobe.com>
Date: Mon, 18 Nov 2019 20:58:06 -0600
Subject: [PATCH 37/81] MQE-1884: MFTF - <after> failures override other
 failures Attached stack trace of full stack of exceptions Attachment name now
 has the test hook method in which the exception occured Removed test fail
 method

---
 .../Extension/TestContextExtension.php        | 64 +++++++++++--------
 1 file changed, 38 insertions(+), 26 deletions(-)

diff --git a/src/Magento/FunctionalTestingFramework/Extension/TestContextExtension.php b/src/Magento/FunctionalTestingFramework/Extension/TestContextExtension.php
index b0f42590f..741e3d3b4 100644
--- a/src/Magento/FunctionalTestingFramework/Extension/TestContextExtension.php
+++ b/src/Magento/FunctionalTestingFramework/Extension/TestContextExtension.php
@@ -36,7 +36,6 @@ public function _initialize()
     {
         $events = [
             Events::TEST_START => 'testStart',
-            Events::TEST_FAIL => 'testFail',
             Events::STEP_AFTER => 'afterStep',
             Events::TEST_END => 'testEnd',
             Events::RESULT_PRINT_AFTER => 'saveFailed'
@@ -57,18 +56,7 @@ public function testStart()
     }
 
     /**
-     * Codeception event listener function, triggered on test failure.
-     * @param \Codeception\Event\FailEvent $e
-     * @return void
-     */
-    public function testFail(\Codeception\Event\FailEvent $e)
-    {
-        //log suppressed exception in case of _after hook failure
-        $this->logPreviousException($e->getFail());
-    }
-
-    /**
-     * Codeception event listener function, triggered on test ending (naturally or by error).
+     * Codeception event listener function, triggered on test ending naturally or by errors/failures.
      * @param \Codeception\Event\TestEvent $e
      * @return void
      * @throws \Exception
@@ -77,20 +65,28 @@ public function testEnd(\Codeception\Event\TestEvent $e)
     {
         $cest = $e->getTest();
 
-        //Access private TestResultObject to find stack and if there are any errors (as opposed to failures)
+        //Access private TestResultObject to find stack and if there are any errors/failures
         $testResultObject = call_user_func(\Closure::bind(
             function () use ($cest) {
                 return $cest->getTestResultObject();
             },
             $cest
         ));
-        $errors = $testResultObject->errors();
-        if (!empty($errors)) {
-            foreach ($errors as $error) {
-                if ($error->failedTest()->getTestMethod() == $cest->getName()) {
-                    //log suppressed exception in case of _after hook failure
-                    $this->logPreviousException($error->thrownException());
-                    continue;
+
+        // check for errors in all test hooks and attach in allure
+        if (!empty($testResultObject->errors())) {
+            foreach ($testResultObject->errors() as $error) {
+                if($error->failedTest()->getTestMethod() == $cest->getTestMethod()) {
+                    $this->attachExceptionToAllure($error->thrownException(), $cest->getTestMethod());
+                }
+            }
+        }
+
+        // check for failures in all test hooks and attach in allure
+        if (!empty($testResultObject->failures())) {
+            foreach ($testResultObject->failures() as $failure) {
+                if($failure->failedTest()->getTestMethod() == $cest->getTestMethod()) {
+                    $this->attachExceptionToAllure($failure->thrownException(), $cest->getTestMethod());
                 }
             }
         }
@@ -116,12 +112,27 @@ public function extractContext($trace, $class)
     }
 
     /**
-     * Attach suppressed exception thrown before _after hook to the current step.
+     * Attach stack trace of exceptions thrown in each test hook to allure.
      * @param  \Exception $exception
+     * @param  String     $testMethod
      * @return mixed
      */
-    public function logPreviousException(\Exception $exception)
+    public function attachExceptionToAllure($exception, $testMethod)
     {
+        $exceptionType = null;
+        $trace = null;
+
+        if (is_subclass_of($exception, \PHPUnit\Framework\Exception::class)) {
+            $trace = $exception->getSerializableTrace();
+        } else {
+            $trace = $exception->getTrace();
+        }
+
+        $context = $this->extractContext($trace, $testMethod);
+
+        AllureHelper::addAttachmentToCurrentStep($exception, $context . 'Exception');
+
+        //pop suppressed exceptions and attach to allure
         $change = function () {
             if ($this instanceof \PHPUnit\Framework\ExceptionWrapper) {
                 return $this->previous;
@@ -129,9 +140,10 @@ public function logPreviousException(\Exception $exception)
                 return $this->getPrevious();
             }
         };
-        $firstException = $change->call($exception);
-        if ($firstException !== null) {
-            AllureHelper::addAttachmentToCurrentStep($firstException, 'Exception');
+        $previousException = $change->call($exception);
+
+        if ($previousException !== null) {
+            $this->attachExceptionToAllure($previousException, $testMethod);
         }
     }
 

From 6db9c8d63acc790818ce81b339732358c0d9e464 Mon Sep 17 00:00:00 2001
From: Soumya Unnikrishnan <sunnikri@adobe.com>
Date: Mon, 18 Nov 2019 21:22:31 -0600
Subject: [PATCH 38/81] MQE-1884: MFTF - <after> failures override other
 failures removed redundant var; code cleanup.

---
 .../Extension/TestContextExtension.php                        | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/src/Magento/FunctionalTestingFramework/Extension/TestContextExtension.php b/src/Magento/FunctionalTestingFramework/Extension/TestContextExtension.php
index 741e3d3b4..33f49675c 100644
--- a/src/Magento/FunctionalTestingFramework/Extension/TestContextExtension.php
+++ b/src/Magento/FunctionalTestingFramework/Extension/TestContextExtension.php
@@ -119,9 +119,6 @@ public function extractContext($trace, $class)
      */
     public function attachExceptionToAllure($exception, $testMethod)
     {
-        $exceptionType = null;
-        $trace = null;
-
         if (is_subclass_of($exception, \PHPUnit\Framework\Exception::class)) {
             $trace = $exception->getSerializableTrace();
         } else {
@@ -140,6 +137,7 @@ public function attachExceptionToAllure($exception, $testMethod)
                 return $this->getPrevious();
             }
         };
+
         $previousException = $change->call($exception);
 
         if ($previousException !== null) {

From d3e6a72ccac07b2a5656313e437b600580a46a8c Mon Sep 17 00:00:00 2001
From: Soumya Unnikrishnan <sunnikri@adobe.com>
Date: Mon, 18 Nov 2019 22:22:39 -0600
Subject: [PATCH 39/81] MQE-1884: MFTF - <after> failures override other
 failures fixed static checks

---
 .../Extension/TestContextExtension.php                      | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/Magento/FunctionalTestingFramework/Extension/TestContextExtension.php b/src/Magento/FunctionalTestingFramework/Extension/TestContextExtension.php
index 33f49675c..e8e33a5be 100644
--- a/src/Magento/FunctionalTestingFramework/Extension/TestContextExtension.php
+++ b/src/Magento/FunctionalTestingFramework/Extension/TestContextExtension.php
@@ -76,7 +76,7 @@ function () use ($cest) {
         // check for errors in all test hooks and attach in allure
         if (!empty($testResultObject->errors())) {
             foreach ($testResultObject->errors() as $error) {
-                if($error->failedTest()->getTestMethod() == $cest->getTestMethod()) {
+                if ($error->failedTest()->getTestMethod() == $cest->getTestMethod()) {
                     $this->attachExceptionToAllure($error->thrownException(), $cest->getTestMethod());
                 }
             }
@@ -85,7 +85,7 @@ function () use ($cest) {
         // check for failures in all test hooks and attach in allure
         if (!empty($testResultObject->failures())) {
             foreach ($testResultObject->failures() as $failure) {
-                if($failure->failedTest()->getTestMethod() == $cest->getTestMethod()) {
+                if ($failure->failedTest()->getTestMethod() == $cest->getTestMethod()) {
                     $this->attachExceptionToAllure($failure->thrownException(), $cest->getTestMethod());
                 }
             }
@@ -114,7 +114,7 @@ public function extractContext($trace, $class)
     /**
      * Attach stack trace of exceptions thrown in each test hook to allure.
      * @param  \Exception $exception
-     * @param  String     $testMethod
+     * @param  string     $testMethod
      * @return mixed
      */
     public function attachExceptionToAllure($exception, $testMethod)

From bea39a96e4822f8259633cd63e830b0febb552b9 Mon Sep 17 00:00:00 2001
From: Ji Lu <jilu1@adobe.com>
Date: Tue, 19 Nov 2019 09:25:11 -0600
Subject: [PATCH 40/81] MQE-1257: MFTF doctor command

---
 docs/commands/mftf.md                         |  6 ++-
 .../Console/DoctorCommand.php                 | 34 ++++++++--------
 .../Module/MagentoWebDriverDoctor.php         | 40 +++++++++++--------
 .../FunctionalTestingFramework/_bootstrap.php |  2 +-
 4 files changed, 45 insertions(+), 37 deletions(-)

diff --git a/docs/commands/mftf.md b/docs/commands/mftf.md
index 5e9895958..a71b8a02c 100644
--- a/docs/commands/mftf.md
+++ b/docs/commands/mftf.md
@@ -113,7 +113,11 @@ vendor/bin/mftf build:project --MAGENTO_BASE_URL=http://magento.local/ --MAGENTO
 
 #### Description
 
-Diagnose MFTF configuration and setup
+Diagnose MFTF configuration and setup. Currently this command will check the following:
+- Verify admin credentials are valid. Allowing MFTF authenticates and runs API requests to Magento through cURL
+- Verify that Selenium is up and running and available for MFTF
+- Verify that new session of browser can open Magento admin and store front urls
+- Verify that MFTF can run MagentoCLI commands
 
 #### Usage
 
diff --git a/src/Magento/FunctionalTestingFramework/Console/DoctorCommand.php b/src/Magento/FunctionalTestingFramework/Console/DoctorCommand.php
index d75004fc2..4bd05b836 100644
--- a/src/Magento/FunctionalTestingFramework/Console/DoctorCommand.php
+++ b/src/Magento/FunctionalTestingFramework/Console/DoctorCommand.php
@@ -18,23 +18,20 @@
 use Magento\FunctionalTestingFramework\Util\ModuleResolver;
 use Magento\FunctionalTestingFramework\Module\MagentoWebDriver;
 use Magento\FunctionalTestingFramework\Module\MagentoWebDriverDoctor;
+use Symfony\Component\Console\Style\SymfonyStyle;
 
 class DoctorCommand extends Command
 {
     const CODECEPTION_AUTOLOAD_FILE = PROJECT_ROOT . '/vendor/codeception/codeception/autoload.php';
     const MFTF_CODECEPTION_CONFIG_FILE = ENV_FILE_PATH . 'codeception.yml';
     const SUITE = 'functional';
-    const COLOR_LIGHT_GREEN = "\e[1;32m";
-    const COLOR_LIGHT_RED = "\e[1;31m";
-    const COLOR_LIGHT_DEFAULT = "\e[1;39m";
-    const COLOR_RESTORE = "\e[0m";
 
     /**
-     * Command Output
+     * Console output style
      *
-     * @var OutputInterface
+     * @var SymfonyStyle
      */
-    private $output;
+    private $ioStyle;
 
     /**
      * Exception Context
@@ -63,17 +60,17 @@ protected function configure()
      * @param OutputInterface $output
      * @return integer
      * @throws TestFrameworkException
-     * @SuppressWarnings(PHPMD.UnusedFormalParameter)
      * @SuppressWarnings(PHPMD.CyclomaticComplexity)
      */
     protected function execute(InputInterface $input, OutputInterface $output)
     {
+        // For output style
+        $this->ioStyle = new SymfonyStyle($input, $output);
+
         $cmdStatus = true;
 
         // Config application
         $verbose = $output->isVerbose();
-        $this->input = $input;
-        $this->output = $output;
         MftfApplicationConfig::create(
             false,
             MftfApplicationConfig::GENERATION_PHASE,
@@ -126,14 +123,15 @@ private function checkAuthenticationToMagentoAdmin()
     {
         $result = false;
         try {
-            $this->output->writeln(
-                "\n" . self::COLOR_LIGHT_DEFAULT . "Authenticating admin account by API ..." . self::COLOR_RESTORE
-            );
+            $this->ioStyle->text("Requesting API token for admin user through cURL ...");
             ModuleResolver::getInstance()->getAdminToken();
-            $this->output->writeln(self::COLOR_LIGHT_GREEN . 'Successful' . self::COLOR_RESTORE);
+            $this->ioStyle->success('Successful');
             $result = true;
         } catch (TestFrameworkException $e) {
-            $this->output->writeln(self::COLOR_LIGHT_RED . $e->getMessage() . self::COLOR_RESTORE);
+            $this->ioStyle->error(
+                $e->getMessage()
+                . "\nPlease verify MAGENTO_ADMIN_USERNAME and MAGENTO_ADMIN_PASSWORD in .env."
+            );
         }
         return $result;
     }
@@ -148,14 +146,14 @@ private function checkAuthenticationToMagentoAdmin()
      */
     private function checkContextOnStep($exceptionType, $message)
     {
-        $this->output->writeln("\n" . self::COLOR_LIGHT_DEFAULT. $message . self::COLOR_RESTORE);
+        $this->ioStyle->text($message . ' ...');
         $this->runMagentoWebDriverDoctor();
 
         if (isset($this->context[$exceptionType])) {
-            $this->output->writeln(self::COLOR_LIGHT_RED . $this->context[$exceptionType] . self::COLOR_RESTORE);
+            $this->ioStyle->error($this->context[$exceptionType]);
             return false;
         } else {
-            $this->output->writeln(self::COLOR_LIGHT_GREEN . 'Successful' . self::COLOR_RESTORE);
+            $this->ioStyle->success('Successful');
             return true;
         }
     }
diff --git a/src/Magento/FunctionalTestingFramework/Module/MagentoWebDriverDoctor.php b/src/Magento/FunctionalTestingFramework/Module/MagentoWebDriverDoctor.php
index 35fa37f03..082c2d7c6 100644
--- a/src/Magento/FunctionalTestingFramework/Module/MagentoWebDriverDoctor.php
+++ b/src/Magento/FunctionalTestingFramework/Module/MagentoWebDriverDoctor.php
@@ -94,12 +94,16 @@ private function connectToSeleniumServer()
                 $this->httpProxy,
                 $this->httpProxyPort
             );
+            if (null !== $this->remoteWebDriver) {
+                return;
+            }
         } catch (\Exception $e) {
-            throw new TestFrameworkException(
-                "Failed to connect Selenium WebDriver at: {$this->wdHost}.\n"
-                . "Please make sure that Selenium Server is running."
-            );
         }
+
+        throw new TestFrameworkException(
+            "Failed to connect Selenium WebDriver at: {$this->wdHost}.\n"
+            . "Please make sure that Selenium Server is running."
+        );
     }
 
     /**
@@ -112,19 +116,21 @@ private function connectToSeleniumServer()
     private function loadPageAtUrl($url)
     {
         try {
-            // Open the web page at url first
-            $this->remoteWebDriver->get($url);
-
-            // Execute Javascript to retrieve HTTP response code
-            $script = ''
-                . 'var xhr = new XMLHttpRequest();'
-                . "xhr.open('GET', '" . $url . "', false);"
-                . 'xhr.send(null); '
-                . 'return xhr.status';
-            $status = $this->remoteWebDriver->executeScript($script);
-
-            if ($status === 200) {
-                return;
+            if (null !== $this->remoteWebDriver) {
+                // Open the web page at url first
+                $this->remoteWebDriver->get($url);
+
+                // Execute Javascript to retrieve HTTP response code
+                $script = ''
+                    . 'var xhr = new XMLHttpRequest();'
+                    . "xhr.open('GET', '" . $url . "', false);"
+                    . 'xhr.send(null); '
+                    . 'return xhr.status';
+                $status = $this->remoteWebDriver->executeScript($script);
+
+                if ($status === 200) {
+                    return;
+                }
             }
         } catch (\Exception $e) {
         }
diff --git a/src/Magento/FunctionalTestingFramework/_bootstrap.php b/src/Magento/FunctionalTestingFramework/_bootstrap.php
index dd7cfd37d..e401123b6 100644
--- a/src/Magento/FunctionalTestingFramework/_bootstrap.php
+++ b/src/Magento/FunctionalTestingFramework/_bootstrap.php
@@ -16,7 +16,7 @@
 }
 defined('PROJECT_ROOT') || define('PROJECT_ROOT', $projectRootPath);
 
-$envFilePath = realpath($projectRootPath . '/dev/tests/acceptance/');
+$envFilePath = realpath($projectRootPath . '/dev/tests/acceptance/') . DIRECTORY_SEPARATOR;
 defined('ENV_FILE_PATH') || define('ENV_FILE_PATH', $envFilePath);
 
 //Load constants from .env file

From de359cb99281e0f0cfd667f2aa9a100a2c078c45 Mon Sep 17 00:00:00 2001
From: Ji Lu <jilu1@adobe.com>
Date: Tue, 19 Nov 2019 09:43:40 -0600
Subject: [PATCH 41/81] MQE-1257: MFTF doctor command

---
 .../Module/MagentoWebDriverDoctor.php                           | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/Magento/FunctionalTestingFramework/Module/MagentoWebDriverDoctor.php b/src/Magento/FunctionalTestingFramework/Module/MagentoWebDriverDoctor.php
index 082c2d7c6..98eb8bd4f 100644
--- a/src/Magento/FunctionalTestingFramework/Module/MagentoWebDriverDoctor.php
+++ b/src/Magento/FunctionalTestingFramework/Module/MagentoWebDriverDoctor.php
@@ -137,7 +137,7 @@ private function loadPageAtUrl($url)
 
         throw new TestFrameworkException(
             "Failed to load page at url: $url\n"
-            . "Please check network connection for the browser running Selenium."
+            . "Please check Selenium Browser session have access to Magento instance."
         );
     }
 

From 795f09ef40535b5cc51a81d63798a44f6dc3030c Mon Sep 17 00:00:00 2001
From: Soumya Unnikrishnan <sunnikri@adobe.com>
Date: Tue, 19 Nov 2019 12:11:40 -0600
Subject: [PATCH 42/81] MQE-1884: MFTF - <after> failures override other
 failures Added mapping to contexts for allure attachments.

---
 .../Extension/TestContextExtension.php               | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/src/Magento/FunctionalTestingFramework/Extension/TestContextExtension.php b/src/Magento/FunctionalTestingFramework/Extension/TestContextExtension.php
index e8e33a5be..07153040c 100644
--- a/src/Magento/FunctionalTestingFramework/Extension/TestContextExtension.php
+++ b/src/Magento/FunctionalTestingFramework/Extension/TestContextExtension.php
@@ -18,7 +18,13 @@
 class TestContextExtension extends BaseExtension
 {
     const TEST_PHASE_AFTER = "_after";
+    const TEST_PHASE_BEFORE = "_before";
+
     const TEST_FAILED_FILE = 'failed';
+    const TEST_HOOKS = [
+        self::TEST_PHASE_AFTER => 'AfterHook',
+        self::TEST_PHASE_BEFORE => 'BeforeHook'
+    ];
 
     /**
      * Codeception Events Mapping to methods
@@ -127,6 +133,12 @@ public function attachExceptionToAllure($exception, $testMethod)
 
         $context = $this->extractContext($trace, $testMethod);
 
+        if (isset(self::TEST_HOOKS[$context])) {
+            $context = self::TEST_HOOKS[$context];
+        } else {
+            $context = 'TestMethod';
+        }
+
         AllureHelper::addAttachmentToCurrentStep($exception, $context . 'Exception');
 
         //pop suppressed exceptions and attach to allure

From 3373f7d1b0844a8b09397af469e8be90465c557c Mon Sep 17 00:00:00 2001
From: Kevin Kozan <kkozan@adobe.com>
Date: Wed, 20 Nov 2019 13:13:36 -0600
Subject: [PATCH 43/81] MQE-1765: Introduce API Endpoint and Request Headers to
 Allure artifacts

- Added new Allure event to MFTF codebase to prevent buggy allure behavior
- Changed calls to AllureHelper to use this new event
- Added calls to CurlHandler to add artifacts.
- Unit Test fix
---
 .../Allure/AllureHelperTest.php               |  36 +++++-
 .../Allure/AllureHelper.php                   |   5 +-
 .../Allure/Event/AddUniqueAttachmentEvent.php | 107 ++++++++++++++++++
 .../DataGenerator/Persist/CurlHandler.php     |   2 +
 4 files changed, 146 insertions(+), 4 deletions(-)
 create mode 100644 src/Magento/FunctionalTestingFramework/Allure/Event/AddUniqueAttachmentEvent.php

diff --git a/dev/tests/unit/Magento/FunctionalTestFramework/Allure/AllureHelperTest.php b/dev/tests/unit/Magento/FunctionalTestFramework/Allure/AllureHelperTest.php
index 048c6f7de..0bf62c63e 100644
--- a/dev/tests/unit/Magento/FunctionalTestFramework/Allure/AllureHelperTest.php
+++ b/dev/tests/unit/Magento/FunctionalTestFramework/Allure/AllureHelperTest.php
@@ -6,6 +6,7 @@
 namespace Tests\unit\Magento\FunctionalTestingFramework\Allure;
 
 use Magento\FunctionalTestingFramework\Allure\AllureHelper;
+use Magento\FunctionalTestingFramework\Allure\Event\AddUniqueAttachmentEvent;
 use Yandex\Allure\Adapter\Allure;
 use Yandex\Allure\Adapter\Event\AddAttachmentEvent;
 use Yandex\Allure\Adapter\Event\StepFinishedEvent;
@@ -84,14 +85,45 @@ public function testAddAttachmentToLastStep()
         $this->assertEmpty($thirdStep->getAttachments());
     }
 
+    public function testAddAttachementUniqueName()
+    {
+        $this->mockCopyFile();
+        $expectedData = "string";
+        $expectedCaption = "caption";
+
+        //Prepare Allure lifecycle
+        Allure::lifecycle()->fire(new StepStartedEvent('firstStep'));
+
+        //Call function twice
+        AllureHelper::addAttachmentToCurrentStep($expectedData, $expectedCaption);
+        AllureHelper::addAttachmentToCurrentStep($expectedData, $expectedCaption);
+
+        // Assert file names for both attachments are not the same.
+        $step = Allure::lifecycle()->getStepStorage()->pollLast();
+        $attachmentOne = $step->getAttachments()[0]->getSource();
+        $attachmentTwo = $step->getAttachments()[1]->getSource();
+        $this->assertNotEquals($attachmentOne, $attachmentTwo);
+    }
+
     /**
-     * Mock file system manipulation function
+     * Mock entire attachment writing mechanisms
      * @throws \Exception
      */
     public function mockAttachmentWriteEvent()
     {
-        AspectMock::double(AddAttachmentEvent::class, [
+        AspectMock::double(AddUniqueAttachmentEvent::class, [
             "getAttachmentFileName" => self::MOCK_FILENAME
         ]);
     }
+
+    /**
+     * Mock only file writing mechanism
+     * @throws \Exception
+     */
+    public function mockCopyFile()
+    {
+        AspectMock::double(AddUniqueAttachmentEvent::class, [
+            "copyFile" => true
+        ]);
+    }
 }
diff --git a/src/Magento/FunctionalTestingFramework/Allure/AllureHelper.php b/src/Magento/FunctionalTestingFramework/Allure/AllureHelper.php
index d8f1c63d8..e1d871781 100644
--- a/src/Magento/FunctionalTestingFramework/Allure/AllureHelper.php
+++ b/src/Magento/FunctionalTestingFramework/Allure/AllureHelper.php
@@ -5,6 +5,7 @@
  */
 namespace Magento\FunctionalTestingFramework\Allure;
 
+use Magento\FunctionalTestingFramework\Allure\Event\AddUniqueAttachmentEvent;
 use Yandex\Allure\Adapter\Allure;
 use Yandex\Allure\Adapter\Event\AddAttachmentEvent;
 
@@ -19,7 +20,7 @@ class AllureHelper
      */
     public static function addAttachmentToCurrentStep($data, $caption)
     {
-        Allure::lifecycle()->fire(new AddAttachmentEvent($data, $caption));
+        Allure::lifecycle()->fire(new AddUniqueAttachmentEvent($data, $caption));
     }
 
     /**
@@ -34,7 +35,7 @@ public static function addAttachmentToLastStep($data, $caption)
         $rootStep = Allure::lifecycle()->getStepStorage()->getLast();
         $trueLastStep = array_last($rootStep->getSteps());
 
-        $attachmentEvent = new AddAttachmentEvent($data, $caption);
+        $attachmentEvent = new AddUniqueAttachmentEvent($data, $caption);
         $attachmentEvent->process($trueLastStep);
     }
 }
diff --git a/src/Magento/FunctionalTestingFramework/Allure/Event/AddUniqueAttachmentEvent.php b/src/Magento/FunctionalTestingFramework/Allure/Event/AddUniqueAttachmentEvent.php
new file mode 100644
index 000000000..b6ff032a9
--- /dev/null
+++ b/src/Magento/FunctionalTestingFramework/Allure/Event/AddUniqueAttachmentEvent.php
@@ -0,0 +1,107 @@
+<?php
+
+/**
+ * Copyright © Magento, Inc. All rights reserved.
+ * See COPYING.txt for license details.
+ */
+namespace Magento\FunctionalTestingFramework\Allure\Event;
+
+use Symfony\Component\HttpFoundation\File\MimeType\ExtensionGuesser;
+use Symfony\Component\HttpFoundation\File\MimeType\MimeTypeGuesser;
+use Yandex\Allure\Adapter\AllureException;
+use Yandex\Allure\Adapter\Event\AddAttachmentEvent;
+
+const DEFAULT_FILE_EXTENSION = 'txt';
+const DEFAULT_MIME_TYPE = 'text/plain';
+
+class AddUniqueAttachmentEvent extends AddAttachmentEvent
+{
+    /**
+     * @var string
+     */
+    private $type;
+
+    /**
+     * Near copy of parent function, added uniqid call for filename to prevent buggy allure behavior
+     * @param string $filePathOrContents
+     * @param string $type
+     * @return string
+     * @throws AllureException
+     */
+    public function getAttachmentFileName($filePathOrContents, $type)
+    {
+        $filePath = $filePathOrContents;
+        if (!file_exists($filePath) || !is_file($filePath)) {
+            //Save contents to temporary file
+            $filePath = tempnam(sys_get_temp_dir(), 'allure-attachment');
+            if (!file_put_contents($filePath, $filePathOrContents)) {
+                throw new AllureException("Failed to save attachment contents to $filePath");
+            }
+        }
+
+        if (!isset($type)) {
+            $type = $this->guessFileMimeType($filePath);
+            $this->type = $type;
+        }
+
+        $fileExtension = $this->guessFileExtension($type);
+
+        $fileSha1 = uniqid(sha1_file($filePath));
+        $outputPath = parent::getOutputPath($fileSha1, $fileExtension);
+        if (!$this->copyFile($filePath, $outputPath)) {
+            throw new AllureException("Failed to copy attachment from $filePath to $outputPath.");
+        }
+
+        return $this->getOutputFileName($fileSha1, $fileExtension);
+    }
+
+    /**
+     * Copies file from one path to another. Wrapper for mocking in unit test.
+     * @param string $filePath
+     * @param string $outputPath
+     * @return bool
+     */
+    private function copyFile($filePath, $outputPath)
+    {
+        return copy($filePath, $outputPath);
+    }
+
+    /**
+     * Copy of parent private function
+     * @param string $filePath
+     * @return string
+     */
+    private function guessFileMimeType($filePath)
+    {
+        $type = MimeTypeGuesser::getInstance()->guess($filePath);
+        if (!isset($type)) {
+            return DEFAULT_MIME_TYPE;
+        }
+        return $type;
+    }
+
+    /**
+     * Copy of parent private function
+     * @param string $mimeType
+     * @return string
+     */
+    private function guessFileExtension($mimeType)
+    {
+        $candidate = ExtensionGuesser::getInstance()->guess($mimeType);
+        if (!isset($candidate)) {
+            return DEFAULT_FILE_EXTENSION;
+        }
+        return $candidate;
+    }
+
+    /**
+     * Copy of parent private function
+     * @param string $sha1
+     * @param string $extension
+     * @return string
+     */
+    public function getOutputFileName($sha1, $extension)
+    {
+        return $sha1 . '-attachment.' . $extension;
+    }
+}
diff --git a/src/Magento/FunctionalTestingFramework/DataGenerator/Persist/CurlHandler.php b/src/Magento/FunctionalTestingFramework/DataGenerator/Persist/CurlHandler.php
index 691ce3606..16584e7a3 100644
--- a/src/Magento/FunctionalTestingFramework/DataGenerator/Persist/CurlHandler.php
+++ b/src/Magento/FunctionalTestingFramework/DataGenerator/Persist/CurlHandler.php
@@ -123,6 +123,8 @@ public function executeRequest($dependentEntities)
         $returnRegex = $this->operationDefinition->getReturnRegex();
         $returnIndex = $this->operationDefinition->getReturnIndex();
         $method = $this->operationDefinition->getApiMethod();
+        AllureHelper::addAttachmentToLastStep($apiUrl, 'API Endpoint');
+        AllureHelper::addAttachmentToLastStep(json_encode($headers, JSON_PRETTY_PRINT), 'Request Headers');
 
         $operationDataResolver = new OperationDataArrayResolver($dependentEntities);
         $this->requestData = $operationDataResolver->resolveOperationDataArray(

From 001e5e29ab08f10d3bad4dee82ee3ec9e748b619 Mon Sep 17 00:00:00 2001
From: Ji Lu <jilu1@adobe.com>
Date: Wed, 20 Nov 2019 11:59:23 -0600
Subject: [PATCH 44/81] MQE-1470: add interface to handle file path and url
 format

---
 .../Util/Path/FilePathFormatterTest.php       |  82 ++++++++++++++
 .../Util/Path/UrlFormatterTest.php            |  84 ++++++++++++++
 .../Util/ConfigSanitizerUtil.php              |  71 +-----------
 .../Util/ModuleResolver.php                   |   7 +-
 .../Util/Path/FilePathFormatter.php           |  31 ++++++
 .../Util/Path/FileUrlFormatterInterface.php   |  22 ++++
 .../Util/Path/UrlFormatter.php                | 103 ++++++++++++++++++
 7 files changed, 329 insertions(+), 71 deletions(-)
 create mode 100644 dev/tests/unit/Magento/FunctionalTestFramework/Util/Path/FilePathFormatterTest.php
 create mode 100644 dev/tests/unit/Magento/FunctionalTestFramework/Util/Path/UrlFormatterTest.php
 create mode 100644 src/Magento/FunctionalTestingFramework/Util/Path/FilePathFormatter.php
 create mode 100644 src/Magento/FunctionalTestingFramework/Util/Path/FileUrlFormatterInterface.php
 create mode 100644 src/Magento/FunctionalTestingFramework/Util/Path/UrlFormatter.php

diff --git a/dev/tests/unit/Magento/FunctionalTestFramework/Util/Path/FilePathFormatterTest.php b/dev/tests/unit/Magento/FunctionalTestFramework/Util/Path/FilePathFormatterTest.php
new file mode 100644
index 000000000..c583fd74c
--- /dev/null
+++ b/dev/tests/unit/Magento/FunctionalTestFramework/Util/Path/FilePathFormatterTest.php
@@ -0,0 +1,82 @@
+<?php
+/**
+ * Copyright © Magento, Inc. All rights reserved.
+ * See COPYING.txt for license details.
+ */
+namespace Tests\unit\Magento\FunctionalTestFramework\Util\Path;
+
+use Magento\FunctionalTestingFramework\Exceptions\TestFrameworkException;
+use Magento\FunctionalTestingFramework\Util\MagentoTestCase;
+use Magento\FunctionalTestingFramework\Util\Path\FilePathFormatter;
+
+class FilePathFormatterTest extends MagentoTestCase
+{
+    /**
+     * Test file format
+     *
+     * @dataProvider formatDataProvider
+     * @param string $path
+     * @param boolean $withTrailingSeparator
+     * @param mixed string|boolean $expectedPath
+     * @return void
+     */
+    public function testFormat($path, $withTrailingSeparator, $expectedPath)
+    {
+        if (null !== $expectedPath) {
+            $this->assertEquals($expectedPath, FilePathFormatter::format($path, $withTrailingSeparator));
+        } else {
+            // Assert no exception
+            FilePathFormatter::format($path, $withTrailingSeparator);
+            $this->assertTrue(true);
+        }
+    }
+
+    /**
+     * Test file format with exception
+     *
+     * @dataProvider formatExceptionDataProvider
+     * @param string $path
+     * @param boolean $withTrailingSeparator
+     * @return void
+     */
+    public function testFormatWithException($path, $withTrailingSeparator)
+    {
+        $this->expectException(TestFrameworkException::class);
+        $this->expectExceptionMessage("Invalid or non-existing file: $path\n");
+        FilePathFormatter::format($path, $withTrailingSeparator);
+    }
+
+    /**
+     * Data input
+     *
+     * @return array
+     */
+    public function formatDataProvider()
+    {
+        $path1 = rtrim(TESTS_BP, '/');
+        $path2 = $path1 . DIRECTORY_SEPARATOR;
+        return [
+            [$path1, null, $path1],
+            [$path1, false, $path1],
+            [$path1, true, $path2],
+            [$path2, null, $path1],
+            [$path2, false, $path1],
+            [$path2, true, $path2],
+            [__DIR__. DIRECTORY_SEPARATOR . basename(__FILE__), null, __FILE__],
+            ['', null, null] // Empty string is valid
+        ];
+    }
+
+    /**
+     * Invalid data input
+     *
+     * @return array
+     */
+    public function formatExceptionDataProvider()
+    {
+        return [
+            ['abc', null],
+            ['X://some\dir/@', null],
+        ];
+    }
+}
diff --git a/dev/tests/unit/Magento/FunctionalTestFramework/Util/Path/UrlFormatterTest.php b/dev/tests/unit/Magento/FunctionalTestFramework/Util/Path/UrlFormatterTest.php
new file mode 100644
index 000000000..86d1f0fad
--- /dev/null
+++ b/dev/tests/unit/Magento/FunctionalTestFramework/Util/Path/UrlFormatterTest.php
@@ -0,0 +1,84 @@
+<?php
+/**
+ * Copyright © Magento, Inc. All rights reserved.
+ * See COPYING.txt for license details.
+ */
+namespace Tests\unit\Magento\FunctionalTestFramework\Util\Path;
+
+use Magento\FunctionalTestingFramework\Util\MagentoTestCase;
+use Magento\FunctionalTestingFramework\Util\Path\UrlFormatter;
+use Magento\FunctionalTestingFramework\Exceptions\TestFrameworkException;
+
+class UrlFormatterTest extends MagentoTestCase
+{
+    /**
+     * Test url format
+     *
+     * @dataProvider formatDataProvider
+     * @param string $path
+     * @param boolean $withTrailingSeparator
+     * @param mixed string|boolean $expectedPath
+     * @return void
+     */
+    public function testFormat($path, $withTrailingSeparator, $expectedPath)
+    {
+        $this->assertEquals($expectedPath, UrlFormatter::format($path, $withTrailingSeparator));
+    }
+
+    /**
+     * Test url format with exception
+     *
+     * @dataProvider formatExceptionDataProvider
+     * @param string $path
+     * @param boolean $withTrailingSeparator
+     * @return void
+     */
+    public function testFormatWithException($path, $withTrailingSeparator)
+    {
+        $this->expectException(TestFrameworkException::class);
+        $this->expectExceptionMessage("Invalid url: $path\n");
+        UrlFormatter::format($path, $withTrailingSeparator);
+    }
+
+    /**
+     * Data input
+     *
+     * @return array
+     */
+    public function formatDataProvider()
+    {
+        $url1 = 'http://magento.local/index.php';
+        $url2 = $url1 . '/';
+        $url3 = 'https://www.example.com/index.php/admin';
+        $url4 = $url3 . '/';
+        $url5 = 'www.google.com';
+        $url6 = 'http://www.google.com/';
+        return [
+            [$url1, null, $url1],
+            [$url1, false, $url1],
+            [$url1, true, $url2],
+            [$url2, null, $url1],
+            [$url2, false, $url1],
+            [$url2, true, $url2],
+            [$url3, null, $url3],
+            [$url3, false, $url3],
+            [$url3, true, $url4],
+            [$url4, null, $url3],
+            [$url4, false, $url3],
+            [$url4, true, $url4],
+            [$url5, true, $url6],
+        ];
+    }
+
+    /**
+     * Invalid data input
+     *
+     * @return array
+     */
+    public function formatExceptionDataProvider()
+    {
+        return [
+            ['', null],
+        ];
+    }
+}
diff --git a/src/Magento/FunctionalTestingFramework/Util/ConfigSanitizerUtil.php b/src/Magento/FunctionalTestingFramework/Util/ConfigSanitizerUtil.php
index ce5740493..3b92b48e4 100644
--- a/src/Magento/FunctionalTestingFramework/Util/ConfigSanitizerUtil.php
+++ b/src/Magento/FunctionalTestingFramework/Util/ConfigSanitizerUtil.php
@@ -7,6 +7,8 @@
 namespace Magento\FunctionalTestingFramework\Util;
 
 use Magento\FunctionalTestingFramework\Config\MftfApplicationConfig;
+use Magento\FunctionalTestingFramework\Exceptions\TestFrameworkException;
+use Magento\FunctionalTestingFramework\Util\Path\UrlFormatter;
 
 /**
  * Class ConfigSanitizerUtil
@@ -24,7 +26,7 @@ public static function sanitizeWebDriverConfig($config, $params = ['url', 'selen
         self::validateConfigBasedVars($config);
 
         if (in_array('url', $params)) {
-            $config['url'] = self::sanitizeUrl($config['url']);
+            $config['url'] = UrlFormatter::format($config['url']);
         }
 
         if (in_array('selenium', $params)) {
@@ -80,71 +82,4 @@ private static function validateConfigBasedVars($config)
             }
         }
     }
-
-    /**
-     * Sanitizes and returns given URL.
-     * @param string $url
-     * @return string
-     */
-    public static function sanitizeUrl($url)
-    {
-        if (strlen($url) == 0 && !MftfApplicationConfig::getConfig()->forceGenerateEnabled()) {
-            trigger_error("MAGENTO_BASE_URL must be defined in .env", E_USER_ERROR);
-        }
-
-        if (filter_var($url, FILTER_VALIDATE_URL) === true) {
-            return rtrim($url, "/") . "/";
-        }
-
-        $urlParts = parse_url($url);
-
-        if (!isset($urlParts['scheme'])) {
-            $urlParts['scheme'] = "http";
-        }
-        if (!isset($urlParts['host'])) {
-            $urlParts['host'] = rtrim($urlParts['path'], "/");
-            $urlParts['host'] = str_replace("//", "/", $urlParts['host']);
-            unset($urlParts['path']);
-        }
-
-        if (!isset($urlParts['path'])) {
-            $urlParts['path'] = "/";
-        } else {
-            $urlParts['path'] = rtrim($urlParts['path'], "/") . "/";
-        }
-
-        return str_replace("///", "//", self::buildUrl($urlParts));
-    }
-
-    /**
-     * Returns url from $parts given, used with parse_url output for convenience.
-     * This only exists because of deprecation of http_build_url, which does the exact same thing as the code below.
-     * @param array $parts
-     * @return string
-     */
-    private static function buildUrl(array $parts)
-    {
-        $get = function ($key) use ($parts) {
-            return isset($parts[$key]) ? $parts[$key] : null;
-        };
-
-        $pass      = $get('pass');
-        $user      = $get('user');
-        $userinfo  = $pass !== null ? "$user:$pass" : $user;
-        $port      = $get('port');
-        $scheme    = $get('scheme');
-        $query     = $get('query');
-        $fragment  = $get('fragment');
-        $authority =
-            ($userinfo !== null ? "$userinfo@" : '') .
-            $get('host') .
-            ($port ? ":$port" : '');
-
-        return
-            (strlen($scheme) ? "$scheme:" : '') .
-            (strlen($authority) ? "//$authority" : '') .
-            $get('path') .
-            (strlen($query) ? "?$query" : '') .
-            (strlen($fragment) ? "#$fragment" : '');
-    }
 }
diff --git a/src/Magento/FunctionalTestingFramework/Util/ModuleResolver.php b/src/Magento/FunctionalTestingFramework/Util/ModuleResolver.php
index d776603d1..f5516b0db 100644
--- a/src/Magento/FunctionalTestingFramework/Util/ModuleResolver.php
+++ b/src/Magento/FunctionalTestingFramework/Util/ModuleResolver.php
@@ -9,6 +9,7 @@
 use Magento\FunctionalTestingFramework\Config\MftfApplicationConfig;
 use Magento\FunctionalTestingFramework\Exceptions\TestFrameworkException;
 use Magento\FunctionalTestingFramework\Util\Logger\LoggingUtil;
+use Magento\FunctionalTestingFramework\Util\Path\UrlFormatter;
 use Symfony\Component\HttpFoundation\Response;
 
 /**
@@ -201,7 +202,7 @@ public function getEnabledModules()
 
         $token = $this->getAdminToken();
 
-        $url = ConfigSanitizerUtil::sanitizeUrl(getenv('MAGENTO_BASE_URL')) . $this->moduleUrl;
+        $url = UrlFormatter::format(getenv('MAGENTO_BASE_URL')) . $this->moduleUrl;
 
         $headers = [
             'Authorization: Bearer ' . $token,
@@ -677,7 +678,7 @@ private function printMagentoVersionInfo()
         if (MftfApplicationConfig::getConfig()->forceGenerateEnabled()) {
             return;
         }
-        $url = ConfigSanitizerUtil::sanitizeUrl(getenv('MAGENTO_BASE_URL')) . $this->versionUrl;
+        $url = UrlFormatter::format(getenv('MAGENTO_BASE_URL')) . $this->versionUrl;
         LoggingUtil::getInstance()->getLogger(ModuleResolver::class)->info(
             "Fetching version information.",
             ['url' => $url]
@@ -718,7 +719,7 @@ protected function getAdminToken()
             throw new TestFrameworkException($message, $context);
         }
 
-        $url = ConfigSanitizerUtil::sanitizeUrl($this->getBackendUrl()) . $this->adminTokenUrl;
+        $url = UrlFormatter::format($this->getBackendUrl()) . $this->adminTokenUrl;
         $data = [
             'username' => $login,
             'password' => $password
diff --git a/src/Magento/FunctionalTestingFramework/Util/Path/FilePathFormatter.php b/src/Magento/FunctionalTestingFramework/Util/Path/FilePathFormatter.php
new file mode 100644
index 000000000..1337f1e3c
--- /dev/null
+++ b/src/Magento/FunctionalTestingFramework/Util/Path/FilePathFormatter.php
@@ -0,0 +1,31 @@
+<?php
+/**
+ * Copyright © Magento, Inc. All rights reserved.
+ * See COPYING.txt for license details.
+ */
+
+namespace Magento\FunctionalTestingFramework\Util\Path;
+
+use Magento\FunctionalTestingFramework\Exceptions\TestFrameworkException;
+
+class FilePathFormatter implements FileUrlFormatterInterface
+{
+    /**
+     * Return formatted full file path from input string, or false on error
+     *
+     * @param string  $path
+     * @param boolean $withTrailingSeparator
+     * @return string
+     * @throws TestFrameworkException
+     */
+    public static function format($path, $withTrailingSeparator = false)
+    {
+        $validPath = realpath($path);
+
+        if ($validPath) {
+            return $withTrailingSeparator ? $validPath . DIRECTORY_SEPARATOR : $validPath;
+        }
+
+        throw new TestFrameworkException("Invalid or non-existing file: $path\n");
+    }
+}
diff --git a/src/Magento/FunctionalTestingFramework/Util/Path/FileUrlFormatterInterface.php b/src/Magento/FunctionalTestingFramework/Util/Path/FileUrlFormatterInterface.php
new file mode 100644
index 000000000..3525dd75b
--- /dev/null
+++ b/src/Magento/FunctionalTestingFramework/Util/Path/FileUrlFormatterInterface.php
@@ -0,0 +1,22 @@
+<?php
+/**
+ * Copyright © Magento, Inc. All rights reserved.
+ * See COPYING.txt for license details.
+ */
+
+namespace Magento\FunctionalTestingFramework\Util\Path;
+
+use Magento\FunctionalTestingFramework\Exceptions\TestFrameworkException;
+
+interface FileUrlFormatterInterface
+{
+    /**
+     * Return formatted path (file path, url, etc) from input string, or false on error
+     *
+     * @param string  $input
+     * @param boolean $withTrailingSeparator
+     * @return string
+     * @throws TestFrameworkException
+     */
+    public static function format($input, $withTrailingSeparator = false);
+}
diff --git a/src/Magento/FunctionalTestingFramework/Util/Path/UrlFormatter.php b/src/Magento/FunctionalTestingFramework/Util/Path/UrlFormatter.php
new file mode 100644
index 000000000..3c2a9499e
--- /dev/null
+++ b/src/Magento/FunctionalTestingFramework/Util/Path/UrlFormatter.php
@@ -0,0 +1,103 @@
+<?php
+/**
+ * Copyright © Magento, Inc. All rights reserved.
+ * See COPYING.txt for license details.
+ */
+
+namespace Magento\FunctionalTestingFramework\Util\Path;
+
+use Magento\FunctionalTestingFramework\Exceptions\TestFrameworkException;
+
+class UrlFormatter implements FileUrlFormatterInterface
+{
+    /**
+     * Return formatted url path from input string, or false on error
+     *
+     * @param string  $url
+     * @param boolean $withTrailingSeparator
+     * @return string
+     * @throws TestFrameworkException
+     */
+    public static function format($url, $withTrailingSeparator = true)
+    {
+        $sanitizedUrl = rtrim($url, '/');
+        $sanitizedUrl = filter_var($sanitizedUrl, FILTER_SANITIZE_URL);
+
+        if (false === $sanitizedUrl) {
+            throw new TestFrameworkException("Invalid url: $url\n");
+        }
+
+        $validUrl = filter_var($sanitizedUrl, FILTER_VALIDATE_URL);
+
+        if (false !== $validUrl) {
+            return $withTrailingSeparator ? $validUrl . '/' : $validUrl;
+        }
+
+        $validUrl = filter_var(self::buildUrl($sanitizedUrl), FILTER_VALIDATE_URL);
+
+        if (false !== $validUrl) {
+            return $withTrailingSeparator ? $validUrl . '/' : $validUrl;
+        }
+
+        throw new TestFrameworkException("Invalid url: $url\n");
+    }
+
+    /**
+     * Try to build missing url scheme and host
+     *
+     * @param string $url
+     * @return string
+     */
+    private static function buildUrl($url)
+    {
+        $urlParts = parse_url($url);
+
+        if (!isset($urlParts['scheme'])) {
+            $urlParts['scheme'] = 'http';
+        }
+
+        if (!isset($urlParts['host'])) {
+            $urlParts['host'] = rtrim($urlParts['path'], '/');
+            $urlParts['host'] = str_replace("//", '/', $urlParts['host']);
+            unset($urlParts['path']);
+        }
+
+        if (isset($urlParts['path'])) {
+            $urlParts['path'] = rtrim($urlParts['path'], '/');
+        }
+
+        return str_replace("///", "//", self::merge($urlParts));
+    }
+
+    /**
+     * Returns url from $parts given, used with parse_url output for convenience.
+     * This only exists because of deprecation of http_build_url, which does the exact same thing as the code below.
+     * @param array $parts
+     * @return string
+     */
+    private static function merge(array $parts)
+    {
+        $get = function ($key) use ($parts) {
+            return isset($parts[$key]) ? $parts[$key] : null;
+        };
+
+        $pass      = $get('pass');
+        $user      = $get('user');
+        $userinfo  = $pass !== null ? "$user:$pass" : $user;
+        $port      = $get('port');
+        $scheme    = $get('scheme');
+        $query     = $get('query');
+        $fragment  = $get('fragment');
+        $authority =
+            ($userinfo !== null ? "$userinfo@" : '') .
+            $get('host') .
+            ($port ? ":$port" : '');
+
+        return
+            (strlen($scheme) ? "$scheme:" : '') .
+            (strlen($authority) ? "//$authority" : '') .
+            $get('path') .
+            (strlen($query) ? "?$query" : '') .
+            (strlen($fragment) ? "#$fragment" : '');
+    }
+}

From b5ea617d2f0955c5aed5a599f1188d522aa585ad Mon Sep 17 00:00:00 2001
From: Kevin Kozan <kkozan@adobe.com>
Date: Wed, 20 Nov 2019 13:20:26 -0600
Subject: [PATCH 45/81] MQE-1765: Introduce API Endpoint and Request Headers to
 Allure artifacts

- static check fix
---
 .../Allure/Event/AddUniqueAttachmentEvent.php                   | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/Magento/FunctionalTestingFramework/Allure/Event/AddUniqueAttachmentEvent.php b/src/Magento/FunctionalTestingFramework/Allure/Event/AddUniqueAttachmentEvent.php
index b6ff032a9..fc4ff64c2 100644
--- a/src/Magento/FunctionalTestingFramework/Allure/Event/AddUniqueAttachmentEvent.php
+++ b/src/Magento/FunctionalTestingFramework/Allure/Event/AddUniqueAttachmentEvent.php
@@ -59,7 +59,7 @@ public function getAttachmentFileName($filePathOrContents, $type)
      * Copies file from one path to another. Wrapper for mocking in unit test.
      * @param string $filePath
      * @param string $outputPath
-     * @return bool
+     * @return boolean
      */
     private function copyFile($filePath, $outputPath)
     {

From 44bd4bd2d995b52da8af69b6fffaa15ffb7c4b78 Mon Sep 17 00:00:00 2001
From: Kevin Kozan <kkozan@adobe.com>
Date: Wed, 20 Nov 2019 14:08:00 -0600
Subject: [PATCH 46/81] MQE-1765: Introduce API Endpoint and Request Headers to
 Allure artifact

- PHPUnit test fix
---
 .../FunctionalTestFramework/Allure/AllureHelperTest.php      | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/dev/tests/unit/Magento/FunctionalTestFramework/Allure/AllureHelperTest.php b/dev/tests/unit/Magento/FunctionalTestFramework/Allure/AllureHelperTest.php
index 0bf62c63e..b7eafaead 100644
--- a/dev/tests/unit/Magento/FunctionalTestFramework/Allure/AllureHelperTest.php
+++ b/dev/tests/unit/Magento/FunctionalTestFramework/Allure/AllureHelperTest.php
@@ -25,6 +25,7 @@ class AllureHelperTest extends TestCase
     public function tearDown()
     {
         Allure::setDefaultLifecycle();
+        AspectMock::clean();
     }
 
     /**
@@ -85,6 +86,10 @@ public function testAddAttachmentToLastStep()
         $this->assertEmpty($thirdStep->getAttachments());
     }
 
+    /**
+     * AddAttachment actions should have files with different attachment names
+     * @throws \Yandex\Allure\Adapter\AllureException
+     */
     public function testAddAttachementUniqueName()
     {
         $this->mockCopyFile();

From f039692292ce643ae9382575c8757912f0596e1b Mon Sep 17 00:00:00 2001
From: Soumya Unnikrishnan <sunnikri@adobe.com>
Date: Thu, 21 Nov 2019 09:09:25 -0600
Subject: [PATCH 47/81] MQE-1774: Review community PR #343 Fix for pipeline
 step failures on execution of bin/magento from WebDriver.

---
 .../Module/MagentoWebDriver.php                      | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/src/Magento/FunctionalTestingFramework/Module/MagentoWebDriver.php b/src/Magento/FunctionalTestingFramework/Module/MagentoWebDriver.php
index 215438420..f057cf171 100644
--- a/src/Magento/FunctionalTestingFramework/Module/MagentoWebDriver.php
+++ b/src/Magento/FunctionalTestingFramework/Module/MagentoWebDriver.php
@@ -517,11 +517,13 @@ public function scrollToTopOfPage()
      */
     public function magentoCLI($command, $arguments = null)
     {
-        try {
-            return $this->shellExecMagentoCLI($command, $arguments);
-        } catch (\Exception $exception) {
-            return $this->curlExecMagentoCLI($command, $arguments);
-        }
+        return $this->curlExecMagentoCLI($command, $arguments);
+        //TODO: calling bin/magento from pipeline is timing out, needs investigation (ref: MQE-1774)
+//        try {
+//            return $this->shellExecMagentoCLI($command, $arguments);
+//        } catch (\Exception $exception) {
+//            return $this->curlExecMagentoCLI($command, $arguments);
+//        }
     }
 
     /**

From de5bed6e36162c2dd8bcaf04c6a5d35b690c10c4 Mon Sep 17 00:00:00 2001
From: Soumya Unnikrishnan <sunnikri@adobe.com>
Date: Thu, 21 Nov 2019 09:20:52 -0600
Subject: [PATCH 48/81] MQE-1774: Review community PR #343 Fixed static checks

---
 .../FunctionalTestingFramework/Module/MagentoWebDriver.php       | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/Magento/FunctionalTestingFramework/Module/MagentoWebDriver.php b/src/Magento/FunctionalTestingFramework/Module/MagentoWebDriver.php
index f057cf171..c1ab892d7 100644
--- a/src/Magento/FunctionalTestingFramework/Module/MagentoWebDriver.php
+++ b/src/Magento/FunctionalTestingFramework/Module/MagentoWebDriver.php
@@ -839,6 +839,7 @@ public function makeScreenshot($name = null)
      *
      * @throws \RuntimeException
      * @return string
+     * @SuppressWarnings(PHPMD.UnusedPrivateMethod)
      */
     private function shellExecMagentoCLI($command, $arguments): string
     {

From be71f945893f63e65cd1606d241b87631bbabd21 Mon Sep 17 00:00:00 2001
From: Kevin Kozan <kkozan@adobe.com>
Date: Thu, 21 Nov 2019 10:40:45 -0600
Subject: [PATCH 49/81] MQE-1520: Consolidate WAIT_TIMEOUT in .env and 
 pageload_timeout in config

- changed functional.suite.dist to use wait timeout
- added default setters in bootstrap files (will always be defined)
- removed default in actionObject
---
 dev/tests/functional/standalone_bootstrap.php                | 5 ++++-
 etc/config/functional.suite.dist.yml                         | 2 +-
 .../FunctionalTestingFramework/Test/Objects/ActionObject.php | 3 +--
 src/Magento/FunctionalTestingFramework/_bootstrap.php        | 3 +++
 4 files changed, 9 insertions(+), 4 deletions(-)

diff --git a/dev/tests/functional/standalone_bootstrap.php b/dev/tests/functional/standalone_bootstrap.php
index de6ef394d..ab6372697 100755
--- a/dev/tests/functional/standalone_bootstrap.php
+++ b/dev/tests/functional/standalone_bootstrap.php
@@ -49,7 +49,10 @@
 
     defined('DEFAULT_TIMEZONE') || define('DEFAULT_TIMEZONE', 'America/Los_Angeles');
     $env->setEnvironmentVariable('DEFAULT_TIMEZONE', DEFAULT_TIMEZONE);
-    
+
+    defined('WAIT_TIMEOUT') || define('WAIT_TIMEOUT', 10);
+    $env->setEnvironmentVariable('WAIT_TIMEOUT', 10);
+
     try {
         new DateTimeZone(DEFAULT_TIMEZONE);
     } catch (\Exception $e) {
diff --git a/etc/config/functional.suite.dist.yml b/etc/config/functional.suite.dist.yml
index 12658515b..5487a3c99 100644
--- a/etc/config/functional.suite.dist.yml
+++ b/etc/config/functional.suite.dist.yml
@@ -27,7 +27,7 @@ modules:
             window_size: 1280x1024
             username: "%MAGENTO_ADMIN_USERNAME%"
             password: "%MAGENTO_ADMIN_PASSWORD%"
-            pageload_timeout: 30
+            pageload_timeout: "%WAIT_TIMEOUT%"
             host: "%SELENIUM_HOST%"
             port: "%SELENIUM_PORT%"
             protocol: "%SELENIUM_PROTOCOL%"
diff --git a/src/Magento/FunctionalTestingFramework/Test/Objects/ActionObject.php b/src/Magento/FunctionalTestingFramework/Test/Objects/ActionObject.php
index 9b1cdf1af..8f60e1e90 100644
--- a/src/Magento/FunctionalTestingFramework/Test/Objects/ActionObject.php
+++ b/src/Magento/FunctionalTestingFramework/Test/Objects/ActionObject.php
@@ -71,7 +71,6 @@ class ActionObject
     const ACTION_ATTRIBUTE_VARIABLE_REGEX_PARAMETER = '/\(.+\)/';
     const ACTION_ATTRIBUTE_VARIABLE_REGEX_PATTERN = '/({{[\w]+\.[\w\[\]]+}})|({{[\w]+\.[\w]+\((?(?!}}).)+\)}})/';
     const STRING_PARAMETER_REGEX = "/'[^']+'/";
-    const DEFAULT_WAIT_TIMEOUT = 10;
     const ACTION_ATTRIBUTE_USERINPUT = 'userInput';
     const ACTION_TYPE_COMMENT = 'comment';
 
@@ -167,7 +166,7 @@ public function __construct(
      */
     public static function getDefaultWaitTimeout()
     {
-        return getenv('WAIT_TIMEOUT') ?: self::DEFAULT_WAIT_TIMEOUT;
+        return getenv('WAIT_TIMEOUT');
     }
 
     /**
diff --git a/src/Magento/FunctionalTestingFramework/_bootstrap.php b/src/Magento/FunctionalTestingFramework/_bootstrap.php
index e401123b6..490ff61de 100644
--- a/src/Magento/FunctionalTestingFramework/_bootstrap.php
+++ b/src/Magento/FunctionalTestingFramework/_bootstrap.php
@@ -50,6 +50,9 @@
     defined('DEFAULT_TIMEZONE') || define('DEFAULT_TIMEZONE', 'America/Los_Angeles');
     $env->setEnvironmentVariable('DEFAULT_TIMEZONE', DEFAULT_TIMEZONE);
 
+    defined('WAIT_TIMEOUT') || define('WAIT_TIMEOUT', 10);
+    $env->setEnvironmentVariable('WAIT_TIMEOUT', 10);
+
     try {
         new DateTimeZone(DEFAULT_TIMEZONE);
     } catch (\Exception $e) {

From cb5c63237ef851c2fc1b44c02402841ac6530ab2 Mon Sep 17 00:00:00 2001
From: Ji Lu <jilu1@adobe.com>
Date: Thu, 21 Nov 2019 10:56:44 -0600
Subject: [PATCH 50/81] MQE-1470: add interface to handle file path and url
 format

---
 .../Util/Path/FilePathFormatterTest.php       |  2 +
 .../Util/Path/UrlFormatterTest.php            |  7 +++
 .../Tests/SuiteGenerationTest.php             |  6 +-
 .../Config/FileResolver/Primary.php           |  8 ++-
 .../Config/FileResolver/Root.php              |  5 +-
 .../Config/SchemaLocator.php                  | 11 +++-
 .../Console/BaseGenerateCommand.php           |  5 +-
 .../Console/BuildProjectCommand.php           | 40 +++++++------
 .../Console/CleanProjectCommand.php           | 41 +++++++------
 .../Console/GenerateDevUrnCommand.php         | 23 +++++---
 .../Console/RunManifestCommand.php            | 26 +++++++--
 .../Console/RunTestCommand.php                |  4 +-
 .../Console/RunTestFailedCommand.php          | 57 ++++++++++++-------
 .../Console/SetupEnvCommand.php               |  5 +-
 .../Handlers/CredentialStore.php              |  3 +-
 .../Handlers/SecretStorage/FileStorage.php    |  4 +-
 .../Persist/Curl/AbstractExecutor.php         |  5 +-
 .../Persist/Curl/AdminExecutor.php            | 14 ++++-
 .../Persist/Curl/WebapiExecutor.php           | 28 ++++++---
 .../Module/MagentoWebDriver.php               |  7 ++-
 .../Suite/SuiteGenerator.php                  | 12 +++-
 .../Suite/Util/SuiteObjectExtractor.php       |  7 +--
 .../Util/Env/EnvProcessor.php                 |  6 +-
 .../Util/Logger/LoggingUtil.php               |  5 +-
 .../Util/Manifest/TestManifestFactory.php     |  6 +-
 .../Util/ModuleResolver.php                   | 22 +++++--
 .../Util/Path/FilePathFormatter.php           |  4 +-
 ...erInterface.php => FormatterInterface.php} |  4 +-
 .../Util/Path/UrlFormatter.php                |  6 +-
 .../Util/TestGenerator.php                    |  6 +-
 .../FunctionalTestingFramework/_bootstrap.php |  2 +-
 31 files changed, 260 insertions(+), 121 deletions(-)
 rename src/Magento/FunctionalTestingFramework/Util/Path/{FileUrlFormatterInterface.php => FormatterInterface.php} (92%)

diff --git a/dev/tests/unit/Magento/FunctionalTestFramework/Util/Path/FilePathFormatterTest.php b/dev/tests/unit/Magento/FunctionalTestFramework/Util/Path/FilePathFormatterTest.php
index c583fd74c..bac14113d 100644
--- a/dev/tests/unit/Magento/FunctionalTestFramework/Util/Path/FilePathFormatterTest.php
+++ b/dev/tests/unit/Magento/FunctionalTestFramework/Util/Path/FilePathFormatterTest.php
@@ -19,6 +19,7 @@ class FilePathFormatterTest extends MagentoTestCase
      * @param boolean $withTrailingSeparator
      * @param mixed string|boolean $expectedPath
      * @return void
+     * @throws TestFrameworkException
      */
     public function testFormat($path, $withTrailingSeparator, $expectedPath)
     {
@@ -38,6 +39,7 @@ public function testFormat($path, $withTrailingSeparator, $expectedPath)
      * @param string $path
      * @param boolean $withTrailingSeparator
      * @return void
+     * @throws TestFrameworkException
      */
     public function testFormatWithException($path, $withTrailingSeparator)
     {
diff --git a/dev/tests/unit/Magento/FunctionalTestFramework/Util/Path/UrlFormatterTest.php b/dev/tests/unit/Magento/FunctionalTestFramework/Util/Path/UrlFormatterTest.php
index 86d1f0fad..c66c9558c 100644
--- a/dev/tests/unit/Magento/FunctionalTestFramework/Util/Path/UrlFormatterTest.php
+++ b/dev/tests/unit/Magento/FunctionalTestFramework/Util/Path/UrlFormatterTest.php
@@ -19,6 +19,7 @@ class UrlFormatterTest extends MagentoTestCase
      * @param boolean $withTrailingSeparator
      * @param mixed string|boolean $expectedPath
      * @return void
+     * @throws TestFrameworkException
      */
     public function testFormat($path, $withTrailingSeparator, $expectedPath)
     {
@@ -32,6 +33,7 @@ public function testFormat($path, $withTrailingSeparator, $expectedPath)
      * @param string $path
      * @param boolean $withTrailingSeparator
      * @return void
+     * @throws TestFrameworkException
      */
     public function testFormatWithException($path, $withTrailingSeparator)
     {
@@ -53,6 +55,9 @@ public function formatDataProvider()
         $url4 = $url3 . '/';
         $url5 = 'www.google.com';
         $url6 = 'http://www.google.com/';
+        $url7 = 'http://127.0.0.1:8200';
+        $url8 = 'wwøw.goåoøgle.coøm';
+        $url9 = 'http://www.google.com';
         return [
             [$url1, null, $url1],
             [$url1, false, $url1],
@@ -67,6 +72,8 @@ public function formatDataProvider()
             [$url4, false, $url3],
             [$url4, true, $url4],
             [$url5, true, $url6],
+            [$url7, false, $url7],
+            [$url8, false, $url9],
         ];
     }
 
diff --git a/dev/tests/verification/Tests/SuiteGenerationTest.php b/dev/tests/verification/Tests/SuiteGenerationTest.php
index d2242fb52..485ef6411 100644
--- a/dev/tests/verification/Tests/SuiteGenerationTest.php
+++ b/dev/tests/verification/Tests/SuiteGenerationTest.php
@@ -6,11 +6,13 @@
 
 namespace tests\verification\Tests;
 
+use Magento\FunctionalTestingFramework\Exceptions\TestFrameworkException;
 use Magento\FunctionalTestingFramework\Suite\SuiteGenerator;
 use Magento\FunctionalTestingFramework\Util\Filesystem\DirSetupUtil;
 use Magento\FunctionalTestingFramework\Util\Manifest\DefaultTestManifest;
 use Magento\FunctionalTestingFramework\Util\Manifest\ParallelTestManifest;
 use Magento\FunctionalTestingFramework\Util\Manifest\TestManifestFactory;
+use Magento\FunctionalTestingFramework\Util\Path\FilePathFormatter;
 use PHPUnit\Util\Filesystem;
 use Symfony\Component\Yaml\Yaml;
 use tests\unit\Util\TestLoggingUtil;
@@ -413,11 +415,11 @@ public static function tearDownAfterClass()
      * Getter for manifest file path
      *
      * @return string
+     * @throws TestFrameworkException
      */
     private static function getManifestFilePath()
     {
-        return TESTS_BP .
-            DIRECTORY_SEPARATOR .
+        return FilePathFormatter::format(TESTS_BP) .
             "verification" .
             DIRECTORY_SEPARATOR .
             "_generated" .
diff --git a/src/Magento/FunctionalTestingFramework/Config/FileResolver/Primary.php b/src/Magento/FunctionalTestingFramework/Config/FileResolver/Primary.php
index 2376b3cf1..3a976944b 100644
--- a/src/Magento/FunctionalTestingFramework/Config/FileResolver/Primary.php
+++ b/src/Magento/FunctionalTestingFramework/Config/FileResolver/Primary.php
@@ -6,8 +6,10 @@
 
 namespace Magento\FunctionalTestingFramework\Config\FileResolver;
 
+use Magento\FunctionalTestingFramework\Exceptions\TestFrameworkException;
 use Magento\FunctionalTestingFramework\Util\Iterator\File;
 use Magento\FunctionalTestingFramework\Config\FileResolverInterface;
+use Magento\FunctionalTestingFramework\Util\Path\FilePathFormatter;
 
 /**
  * Provides the list of global configuration files.
@@ -54,6 +56,7 @@ private function getFilePaths($filename, $scope)
      * @param string $filename
      * @param string $scope
      * @return array
+     * @throws TestFrameworkException
      */
     private function getPathPatterns($filename, $scope)
     {
@@ -69,8 +72,9 @@ private function getPathPatterns($filename, $scope)
                 $defaultPath . DIRECTORY_SEPARATOR . $scope . DIRECTORY_SEPARATOR . $filename,
                 $defaultPath . DIRECTORY_SEPARATOR . $scope . DIRECTORY_SEPARATOR . '*' . DIRECTORY_SEPARATOR
                 . $filename,
-                FW_BP . DIRECTORY_SEPARATOR . $scope . DIRECTORY_SEPARATOR . $filename,
-                FW_BP . DIRECTORY_SEPARATOR . $scope . DIRECTORY_SEPARATOR . '*' . DIRECTORY_SEPARATOR . $filename
+                FilePathFormatter::format(FW_BP) . $scope . DIRECTORY_SEPARATOR . $filename,
+                FilePathFormatter::format(FW_BP)  . $scope . DIRECTORY_SEPARATOR . '*' . DIRECTORY_SEPARATOR
+                . $filename
             ];
         }
         return str_replace(DIRECTORY_SEPARATOR, DIRECTORY_SEPARATOR, $patterns);
diff --git a/src/Magento/FunctionalTestingFramework/Config/FileResolver/Root.php b/src/Magento/FunctionalTestingFramework/Config/FileResolver/Root.php
index 3b0940b28..4ebb11942 100644
--- a/src/Magento/FunctionalTestingFramework/Config/FileResolver/Root.php
+++ b/src/Magento/FunctionalTestingFramework/Config/FileResolver/Root.php
@@ -7,7 +7,9 @@
 namespace Magento\FunctionalTestingFramework\Config\FileResolver;
 
 use Magento\FunctionalTestingFramework\Config\FileResolverInterface;
+use Magento\FunctionalTestingFramework\Exceptions\TestFrameworkException;
 use Magento\FunctionalTestingFramework\Util\Iterator\File;
+use Magento\FunctionalTestingFramework\Util\Path\FilePathFormatter;
 
 class Root extends Module
 {
@@ -20,12 +22,13 @@ class Root extends Module
      * @param string $filename
      * @param string $scope
      * @return array|\Iterator,\Countable
+     * @throws TestFrameworkException
      */
     public function get($filename, $scope)
     {
         // first pick up the root level test suite dir
         $paths = glob(
-            TESTS_BP . DIRECTORY_SEPARATOR . self::ROOT_SUITE_DIR
+            FilePathFormatter::format(TESTS_BP) . self::ROOT_SUITE_DIR
             . DIRECTORY_SEPARATOR . $filename
         );
 
diff --git a/src/Magento/FunctionalTestingFramework/Config/SchemaLocator.php b/src/Magento/FunctionalTestingFramework/Config/SchemaLocator.php
index a92f536a3..869c58373 100644
--- a/src/Magento/FunctionalTestingFramework/Config/SchemaLocator.php
+++ b/src/Magento/FunctionalTestingFramework/Config/SchemaLocator.php
@@ -6,6 +6,9 @@
 
 namespace Magento\FunctionalTestingFramework\Config;
 
+use Magento\FunctionalTestingFramework\Exceptions\TestFrameworkException;
+use Magento\FunctionalTestingFramework\Util\Path\FilePathFormatter;
+
 /**
  * Configuration schema locator.
  */
@@ -30,12 +33,14 @@ class SchemaLocator implements \Magento\FunctionalTestingFramework\Config\Schema
      *
      * @param string      $schemaPath
      * @param string|null $perFileSchema
+     * @throws TestFrameworkException
      */
     public function __construct($schemaPath, $perFileSchema = null)
     {
-        if (constant('FW_BP') && file_exists(FW_BP . DIRECTORY_SEPARATOR . $schemaPath)) {
-            $this->schemaPath = FW_BP . DIRECTORY_SEPARATOR . $schemaPath;
-            $this->perFileSchema = $perFileSchema === null ? null : FW_BP . DIRECTORY_SEPARATOR . $perFileSchema;
+        if (constant('FW_BP') && file_exists(FilePathFormatter::format(FW_BP) . $schemaPath)) {
+            $this->schemaPath = FilePathFormatter::format(FW_BP) . $schemaPath;
+            $this->perFileSchema = $perFileSchema === null ? null : FilePathFormatter::format(FW_BP)
+                . $perFileSchema;
         } else {
             $path = dirname(dirname(dirname(__DIR__)));
             $path = str_replace('\\', DIRECTORY_SEPARATOR, $path);
diff --git a/src/Magento/FunctionalTestingFramework/Console/BaseGenerateCommand.php b/src/Magento/FunctionalTestingFramework/Console/BaseGenerateCommand.php
index 5afaf17c5..0953e9d68 100644
--- a/src/Magento/FunctionalTestingFramework/Console/BaseGenerateCommand.php
+++ b/src/Magento/FunctionalTestingFramework/Console/BaseGenerateCommand.php
@@ -8,7 +8,9 @@
 
 namespace Magento\FunctionalTestingFramework\Console;
 
+use Magento\FunctionalTestingFramework\Exceptions\TestFrameworkException;
 use Magento\FunctionalTestingFramework\Test\Handlers\TestObjectHandler;
+use Magento\FunctionalTestingFramework\Util\Path\FilePathFormatter;
 use Symfony\Component\Console\Command\Command;
 use Symfony\Component\Console\Input\InputOption;
 use Symfony\Component\Console\Output\OutputInterface;
@@ -57,10 +59,11 @@ protected function configure()
      * @param OutputInterface $output
      * @param bool $verbose
      * @return void
+     * @throws TestFrameworkException
      */
     protected function removeGeneratedDirectory(OutputInterface $output, bool $verbose)
     {
-        $generatedDirectory = TESTS_MODULE_PATH . DIRECTORY_SEPARATOR . TestGenerator::GENERATED_DIR;
+        $generatedDirectory = FilePathFormatter::format(TESTS_MODULE_PATH) . TestGenerator::GENERATED_DIR;
 
         if (file_exists($generatedDirectory)) {
             DirSetupUtil::rmdirRecursive($generatedDirectory);
diff --git a/src/Magento/FunctionalTestingFramework/Console/BuildProjectCommand.php b/src/Magento/FunctionalTestingFramework/Console/BuildProjectCommand.php
index 5102a3fdf..02dfcda82 100644
--- a/src/Magento/FunctionalTestingFramework/Console/BuildProjectCommand.php
+++ b/src/Magento/FunctionalTestingFramework/Console/BuildProjectCommand.php
@@ -18,6 +18,7 @@
 use Symfony\Component\Process\Process;
 use Magento\FunctionalTestingFramework\Util\Env\EnvProcessor;
 use Symfony\Component\Yaml\Yaml;
+use Magento\FunctionalTestingFramework\Util\Path\FilePathFormatter;
 
 /**
  * Class BuildProjectCommand
@@ -28,7 +29,6 @@
 class BuildProjectCommand extends Command
 {
     const DEFAULT_YAML_INLINE_DEPTH = 10;
-    const CREDENTIALS_FILE_PATH = TESTS_BP . DIRECTORY_SEPARATOR . '.credentials.example';
 
     /**
      * Env processor manages .env files.
@@ -41,6 +41,7 @@ class BuildProjectCommand extends Command
      * Configures the current command.
      *
      * @return void
+     * @throws TestFrameworkException
      */
     protected function configure()
     {
@@ -52,7 +53,7 @@ protected function configure()
                 InputOption::VALUE_NONE,
                 'upgrade existing MFTF tests according to last major release requirements'
             );
-        $this->envProcessor = new EnvProcessor(TESTS_BP . DIRECTORY_SEPARATOR . '.env');
+        $this->envProcessor = new EnvProcessor(FilePathFormatter::format(TESTS_BP) . '.env');
         $env = $this->envProcessor->getEnv();
         foreach ($env as $key => $value) {
             $this->addOption($key, null, InputOption::VALUE_REQUIRED, '', $value);
@@ -65,8 +66,7 @@ protected function configure()
      * @param InputInterface  $input
      * @param OutputInterface $output
      * @return void
-     * @throws \Symfony\Component\Console\Exception\LogicException
-     *
+     * @throws \Exception
      * @SuppressWarnings(PHPMD.UnusedLocalVariable)
      */
     protected function execute(InputInterface $input, OutputInterface $output)
@@ -109,7 +109,7 @@ function ($type, $buffer) use ($output) {
 
         if ($input->getOption('upgrade')) {
             $upgradeCommand = new UpgradeTestsCommand();
-            $upgradeOptions = new ArrayInput(['path' => TESTS_MODULE_PATH]);
+            $upgradeOptions = new ArrayInput(['path' => FilePathFormatter::format(TESTS_MODULE_PATH)]);
             $upgradeCommand->run($upgradeOptions, $output);
         }
     }
@@ -119,6 +119,7 @@ function ($type, $buffer) use ($output) {
      *
      * @param OutputInterface $output
      * @return void
+     * @throws TestFrameworkException
      */
     private function generateConfigFiles(OutputInterface $output)
     {
@@ -126,45 +127,48 @@ private function generateConfigFiles(OutputInterface $output)
         //Find travel path from codeception.yml to FW_BP
         $relativePath = $fileSystem->makePathRelative(FW_BP, TESTS_BP);
 
-        if (!$fileSystem->exists(TESTS_BP . DIRECTORY_SEPARATOR . 'codeception.yml')) {
+        if (!$fileSystem->exists(FilePathFormatter::format(TESTS_BP) . 'codeception.yml')) {
             // read in the codeception.yml file
-            $configDistYml = Yaml::parse(file_get_contents(realpath(FW_BP . "/etc/config/codeception.dist.yml")));
+            $configDistYml = Yaml::parse(file_get_contents(
+                realpath(FilePathFormatter::format(FW_BP) . "etc/config/codeception.dist.yml")
+            ));
             $configDistYml['paths']['support'] = $relativePath . 'src/Magento/FunctionalTestingFramework';
             $configDistYml['paths']['envs'] = $relativePath . 'etc/_envs';
             $configYmlText = Yaml::dump($configDistYml, self::DEFAULT_YAML_INLINE_DEPTH);
 
             // dump output to new codeception.yml file
-            file_put_contents(TESTS_BP . DIRECTORY_SEPARATOR . 'codeception.yml', $configYmlText);
+            file_put_contents(FilePathFormatter::format(TESTS_BP) . 'codeception.yml', $configYmlText);
             $output->writeln("codeception.yml configuration successfully applied.");
         }
 
-        $output->writeln("codeception.yml applied to " . TESTS_BP . DIRECTORY_SEPARATOR . 'codeception.yml');
+        $output->writeln("codeception.yml applied to " . FilePathFormatter::format(TESTS_BP) . 'codeception.yml');
 
         // copy the functional suite yml, will only copy if there are differences between the template the destination
         $fileSystem->copy(
-            realpath(FW_BP . '/etc/config/functional.suite.dist.yml'),
-            TESTS_BP . DIRECTORY_SEPARATOR . 'tests' . DIRECTORY_SEPARATOR . 'functional.suite.yml'
+            realpath(FilePathFormatter::format(FW_BP) . 'etc/config/functional.suite.dist.yml'),
+            FilePathFormatter::format(TESTS_BP) . 'tests' . DIRECTORY_SEPARATOR . 'functional.suite.yml'
         );
         $output->writeln('functional.suite.yml configuration successfully applied.');
 
         $output->writeln("functional.suite.yml applied to " .
-            TESTS_BP . DIRECTORY_SEPARATOR . 'tests' . DIRECTORY_SEPARATOR . 'functional.suite.yml');
+            FilePathFormatter::format(TESTS_BP) . 'tests' . DIRECTORY_SEPARATOR . 'functional.suite.yml');
 
         $fileSystem->copy(
-            FW_BP . '/etc/config/.credentials.example',
-            self::CREDENTIALS_FILE_PATH
+            FilePathFormatter::format(FW_BP) . 'etc/config/.credentials.example',
+            FilePathFormatter::format(TESTS_BP) . '.credentials.example'
         );
 
         // copy command.php into magento instance
-        if (MAGENTO_BP === FW_BP) {
+        if (FilePathFormatter::format(MAGENTO_BP, false)
+            === FilePathFormatter::format(FW_BP, false)) {
             $output->writeln('MFTF standalone detected, command.php copy not applied.');
         } else {
             $fileSystem->copy(
-                realpath(FW_BP . '/etc/config/command.php'),
-                TESTS_BP . DIRECTORY_SEPARATOR . "utils" . DIRECTORY_SEPARATOR .'command.php'
+                realpath(FilePathFormatter::format(FW_BP) . 'etc/config/command.php'),
+                FilePathFormatter::format(TESTS_BP) . "utils" . DIRECTORY_SEPARATOR .'command.php'
             );
             $output->writeln('command.php copied to ' .
-                TESTS_BP . DIRECTORY_SEPARATOR . "utils" . DIRECTORY_SEPARATOR .'command.php');
+                FilePathFormatter::format(TESTS_BP) . "utils" . DIRECTORY_SEPARATOR .'command.php');
         }
 
         // Remove and Create Log File
diff --git a/src/Magento/FunctionalTestingFramework/Console/CleanProjectCommand.php b/src/Magento/FunctionalTestingFramework/Console/CleanProjectCommand.php
index b2b2c01f3..641cfe1ed 100644
--- a/src/Magento/FunctionalTestingFramework/Console/CleanProjectCommand.php
+++ b/src/Magento/FunctionalTestingFramework/Console/CleanProjectCommand.php
@@ -7,6 +7,8 @@
 
 namespace Magento\FunctionalTestingFramework\Console;
 
+use Magento\FunctionalTestingFramework\Exceptions\TestFrameworkException;
+use Magento\FunctionalTestingFramework\Util\Path\FilePathFormatter;
 use Symfony\Component\Console\Command\Command;
 use Symfony\Component\Console\Input\InputInterface;
 use Symfony\Component\Console\Input\InputOption;
@@ -16,21 +18,6 @@
 
 class CleanProjectCommand extends Command
 {
-    const CONFIGURATION_FILES = [
-        // codeception.yml file for top level config
-        TESTS_BP . DIRECTORY_SEPARATOR . 'codeception.yml',
-        // functional.suite.yml for test execution config
-        TESTS_BP . DIRECTORY_SEPARATOR . 'tests' . DIRECTORY_SEPARATOR . 'functional.suite.yml',
-        // Acceptance Tester Actions generated by codeception
-        FW_BP . '/src/Magento/FunctionalTestingFramework/_generated',
-        // AcceptanceTester Class generated by codeception
-        FW_BP . '/src/Magento/FunctionalTestingFramework/AcceptanceTester.php'
-    ];
-
-    const GENERATED_FILES = [
-        TESTS_MODULE_PATH . '/_generated'
-    ];
-
     /**
      * Configures the current command.
      *
@@ -52,22 +39,40 @@ protected function configure()
      * @param OutputInterface $output
      * @return void
      * @throws \Symfony\Component\Console\Exception\LogicException
+     * @throws TestFrameworkException
      */
     protected function execute(InputInterface $input, OutputInterface $output)
     {
+        $configFiles = [
+            // codeception.yml file for top level config
+            FilePathFormatter::format(TESTS_BP) . 'codeception.yml',
+            // functional.suite.yml for test execution config
+            FilePathFormatter::format(TESTS_BP) . 'tests' . DIRECTORY_SEPARATOR . 'functional.suite.yml',
+            // Acceptance Tester Actions generated by codeception
+            FilePathFormatter::format(FW_BP) . 'src/Magento/FunctionalTestingFramework/_generated',
+            // AcceptanceTester Class generated by codeception
+            FilePathFormatter::format(FW_BP) . 'src/Magento/FunctionalTestingFramework/AcceptanceTester.php'
+        ];
+
+        $generatedFiles = [
+            FilePathFormatter::format(TESTS_MODULE_PATH) . '_generated'
+        ];
+
         $isHardReset = $input->getOption('hard');
         $fileSystem = new Filesystem();
         $finder = new Finder();
-        $finder->files()->name('*.php')->in(realpath(FW_BP . '/src/Magento/FunctionalTestingFramework/Group/'));
+        $finder->files()->name('*.php')->in(
+            realpath(FilePathFormatter::format(FW_BP) . 'src/Magento/FunctionalTestingFramework/Group/')
+        );
         $filesForRemoval = [];
 
         // include config files if user specifies a hard reset for deletion
         if ($isHardReset) {
-            $filesForRemoval = array_merge($filesForRemoval, self::CONFIGURATION_FILES);
+            $filesForRemoval = array_merge($filesForRemoval, $configFiles);
         }
 
         // include the files mftf generates during test execution in TESTS_BP for deletion
-        $filesForRemoval = array_merge($filesForRemoval, self::GENERATED_FILES);
+        $filesForRemoval = array_merge($filesForRemoval, $generatedFiles);
 
         if ($output->isVerbose()) {
             $output->writeln('Deleting Files:');
diff --git a/src/Magento/FunctionalTestingFramework/Console/GenerateDevUrnCommand.php b/src/Magento/FunctionalTestingFramework/Console/GenerateDevUrnCommand.php
index 1147704c0..f953deb16 100644
--- a/src/Magento/FunctionalTestingFramework/Console/GenerateDevUrnCommand.php
+++ b/src/Magento/FunctionalTestingFramework/Console/GenerateDevUrnCommand.php
@@ -9,6 +9,7 @@
 namespace Magento\FunctionalTestingFramework\Console;
 
 use Magento\FunctionalTestingFramework\Exceptions\TestFrameworkException;
+use Magento\FunctionalTestingFramework\Util\Path\FilePathFormatter;
 use Symfony\Component\Console\Command\Command;
 use Symfony\Component\Console\Input\InputArgument;
 use Symfony\Component\Console\Input\InputInterface;
@@ -109,24 +110,32 @@ protected function execute(InputInterface $input, OutputInterface $output)
     /**
      * Generates urn => location array for all MFTF schema.
      * @return array
+     * @throws TestFrameworkException
      */
     private function generateResourcesArray()
     {
         $resourcesArray = [
             'urn:magento:mftf:DataGenerator/etc/dataOperation.xsd' =>
-                realpath(FW_BP . '/src/Magento/FunctionalTestingFramework/DataGenerator/etc/dataOperation.xsd'),
+                realpath(FilePathFormatter::format(FW_BP)
+                    . 'src/Magento/FunctionalTestingFramework/DataGenerator/etc/dataOperation.xsd'),
             'urn:magento:mftf:DataGenerator/etc/dataProfileSchema.xsd' =>
-                realpath(FW_BP . '/src/Magento/FunctionalTestingFramework/DataGenerator/etc/dataProfileSchema.xsd'),
+                realpath(FilePathFormatter::format(FW_BP)
+                    . 'src/Magento/FunctionalTestingFramework/DataGenerator/etc/dataProfileSchema.xsd'),
             'urn:magento:mftf:Page/etc/PageObject.xsd' =>
-                realpath(FW_BP . '/src/Magento/FunctionalTestingFramework/Page/etc/PageObject.xsd'),
+                realpath(FilePathFormatter::format(FW_BP)
+                    . 'src/Magento/FunctionalTestingFramework/Page/etc/PageObject.xsd'),
             'urn:magento:mftf:Page/etc/SectionObject.xsd' =>
-                realpath(FW_BP . '/src/Magento/FunctionalTestingFramework/Page/etc/SectionObject.xsd'),
+                realpath(FilePathFormatter::format(FW_BP)
+                    . 'src/Magento/FunctionalTestingFramework/Page/etc/SectionObject.xsd'),
             'urn:magento:mftf:Test/etc/actionGroupSchema.xsd' =>
-                realpath(FW_BP . '/src/Magento/FunctionalTestingFramework/Test/etc/actionGroupSchema.xsd'),
+                realpath(FilePathFormatter::format(FW_BP)
+                    . 'src/Magento/FunctionalTestingFramework/Test/etc/actionGroupSchema.xsd'),
             'urn:magento:mftf:Test/etc/testSchema.xsd' =>
-                realpath(FW_BP . '/src/Magento/FunctionalTestingFramework/Test/etc/testSchema.xsd'),
+                realpath(FilePathFormatter::format(FW_BP)
+                    . 'src/Magento/FunctionalTestingFramework/Test/etc/testSchema.xsd'),
             'urn:magento:mftf:Suite/etc/suiteSchema.xsd' =>
-                realpath(FW_BP . '/src/Magento/FunctionalTestingFramework/Suite/etc/suiteSchema.xsd')
+                realpath(FilePathFormatter::format(FW_BP)
+                    . 'src/Magento/FunctionalTestingFramework/Suite/etc/suiteSchema.xsd')
         ];
         return $resourcesArray;
     }
diff --git a/src/Magento/FunctionalTestingFramework/Console/RunManifestCommand.php b/src/Magento/FunctionalTestingFramework/Console/RunManifestCommand.php
index e487d16cf..37a90bb77 100644
--- a/src/Magento/FunctionalTestingFramework/Console/RunManifestCommand.php
+++ b/src/Magento/FunctionalTestingFramework/Console/RunManifestCommand.php
@@ -13,6 +13,7 @@
 use Symfony\Component\Console\Input\InputInterface;
 use Symfony\Component\Console\Output\OutputInterface;
 use Symfony\Component\Process\Process;
+use Magento\FunctionalTestingFramework\Util\Path\FilePathFormatter;
 
 class RunManifestCommand extends Command
 {
@@ -31,6 +32,13 @@ class RunManifestCommand extends Command
      */
     private $failedTests = [];
 
+    /**
+     * Path for a failed test
+     *
+     * @var string
+     */
+    private $testsFailedFile;
+
     /**
      * Configure the run:manifest command.
      *
@@ -53,6 +61,14 @@ protected function configure()
      */
     protected function execute(InputInterface $input, OutputInterface $output): int
     {
+        $testsOutputDir = FilePathFormatter::format(TESTS_BP) .
+            "tests" .
+            DIRECTORY_SEPARATOR .
+            "_output" .
+            DIRECTORY_SEPARATOR;
+
+        $this->testsFailedFile = $testsOutputDir . "failed";
+
         $path = $input->getArgument("path");
 
         if (!file_exists($path)) {
@@ -117,8 +133,8 @@ private function runManifestLine(string $manifestLine, OutputInterface $output)
      */
     private function aggregateFailed()
     {
-        if (file_exists(RunTestFailedCommand::TESTS_FAILED_FILE)) {
-            $currentFile = file(RunTestFailedCommand::TESTS_FAILED_FILE, FILE_IGNORE_NEW_LINES);
+        if (file_exists($this->testsFailedFile)) {
+            $currentFile = file($this->testsFailedFile, FILE_IGNORE_NEW_LINES);
             $this->failedTests = array_merge(
                 $this->failedTests,
                 $currentFile
@@ -133,8 +149,8 @@ private function aggregateFailed()
      */
     private function deleteFailedFile()
     {
-        if (file_exists(RunTestFailedCommand::TESTS_FAILED_FILE)) {
-            unlink(RunTestFailedCommand::TESTS_FAILED_FILE);
+        if (file_exists($this->testsFailedFile)) {
+            unlink($this->testsFailedFile);
         }
     }
 
@@ -146,7 +162,7 @@ private function deleteFailedFile()
     private function writeFailedFile()
     {
         foreach ($this->failedTests as $test) {
-            file_put_contents(RunTestFailedCommand::TESTS_FAILED_FILE, $test . "\n", FILE_APPEND);
+            file_put_contents($this->testsFailedFile, $test . "\n", FILE_APPEND);
         }
     }
 }
diff --git a/src/Magento/FunctionalTestingFramework/Console/RunTestCommand.php b/src/Magento/FunctionalTestingFramework/Console/RunTestCommand.php
index 77b8be513..373256cfc 100644
--- a/src/Magento/FunctionalTestingFramework/Console/RunTestCommand.php
+++ b/src/Magento/FunctionalTestingFramework/Console/RunTestCommand.php
@@ -8,6 +8,7 @@
 namespace Magento\FunctionalTestingFramework\Console;
 
 use Magento\FunctionalTestingFramework\Config\MftfApplicationConfig;
+use Magento\FunctionalTestingFramework\Util\Path\FilePathFormatter;
 use Magento\FunctionalTestingFramework\Util\TestGenerator;
 use Symfony\Component\Console\Input\ArrayInput;
 use Symfony\Component\Console\Input\InputArgument;
@@ -122,8 +123,7 @@ protected function execute(InputInterface $input, OutputInterface $output): int
     private function runTests(array $tests, OutputInterface $output)
     {
         $codeceptionCommand = realpath(PROJECT_ROOT . '/vendor/bin/codecept') . ' run functional ';
-        $testsDirectory = TESTS_MODULE_PATH .
-            DIRECTORY_SEPARATOR .
+        $testsDirectory = FilePathFormatter::format(TESTS_MODULE_PATH) .
             TestGenerator::GENERATED_DIR .
             DIRECTORY_SEPARATOR .
             TestGenerator::DEFAULT_DIR .
diff --git a/src/Magento/FunctionalTestingFramework/Console/RunTestFailedCommand.php b/src/Magento/FunctionalTestingFramework/Console/RunTestFailedCommand.php
index 336fd5d87..5f0596a7f 100644
--- a/src/Magento/FunctionalTestingFramework/Console/RunTestFailedCommand.php
+++ b/src/Magento/FunctionalTestingFramework/Console/RunTestFailedCommand.php
@@ -8,6 +8,7 @@
 namespace Magento\FunctionalTestingFramework\Console;
 
 use Magento\FunctionalTestingFramework\Config\MftfApplicationConfig;
+use Magento\FunctionalTestingFramework\Util\Path\FilePathFormatter;
 use Symfony\Component\Console\Input\ArrayInput;
 use Symfony\Component\Console\Input\InputInterface;
 use Symfony\Component\Console\Output\OutputInterface;
@@ -22,20 +23,20 @@ class RunTestFailedCommand extends BaseGenerateCommand
      */
     const DEFAULT_TEST_GROUP = 'default';
 
-    const TESTS_OUTPUT_DIR = TESTS_BP .
-    DIRECTORY_SEPARATOR .
-    "tests" .
-    DIRECTORY_SEPARATOR .
-    "_output" .
-    DIRECTORY_SEPARATOR;
-
-    const TESTS_FAILED_FILE = self::TESTS_OUTPUT_DIR . "failed";
-    const TESTS_RERUN_FILE = self::TESTS_OUTPUT_DIR . "rerun_tests";
-    const TESTS_MANIFEST_FILE= TESTS_MODULE_PATH .
-    DIRECTORY_SEPARATOR .
-    "_generated" .
-    DIRECTORY_SEPARATOR .
-    "testManifest.txt";
+    /**
+     * @var string
+     */
+    private $testsFailedFile;
+
+    /**
+     * @var string
+     */
+    private $testsReRunFile;
+
+    /**
+     * @var string
+     */
+    private $testsManifestFile;
 
     /**
      * @var array
@@ -68,6 +69,19 @@ protected function configure()
      */
     protected function execute(InputInterface $input, OutputInterface $output): int
     {
+        $testsOutputDir = FilePathFormatter::format(TESTS_BP) .
+            "tests" .
+            DIRECTORY_SEPARATOR .
+            "_output" .
+            DIRECTORY_SEPARATOR;
+
+        $this->testsFailedFile = $testsOutputDir . "failed";
+        $this->testsReRunFile = $testsOutputDir . "rerun_tests";
+        $this->testsManifestFile= FilePathFormatter::format(TESTS_MODULE_PATH) .
+            "_generated" .
+            DIRECTORY_SEPARATOR .
+            "testManifest.txt";
+
         $force = $input->getOption('force');
         $debug = $input->getOption('debug') ?? MftfApplicationConfig::LEVEL_DEVELOPER; // for backward compatibility
         $allowSkipped = $input->getOption('allow-skipped');
@@ -115,15 +129,15 @@ function ($type, $buffer) use ($output) {
                     $output->write($buffer);
                 }
             ));
-            if (file_exists(self::TESTS_FAILED_FILE)) {
+            if (file_exists($this->testsFailedFile)) {
                 $this->failedList = array_merge(
                     $this->failedList,
-                    $this->readFailedTestFile(self::TESTS_FAILED_FILE)
+                    $this->readFailedTestFile($this->testsFailedFile)
                 );
             }
         }
         foreach ($this->failedList as $test) {
-            $this->writeFailedTestToFile($test, self::TESTS_FAILED_FILE);
+            $this->writeFailedTestToFile($test, $this->testsFailedFile);
         }
 
         return $returnCode;
@@ -138,12 +152,12 @@ private function getFailedTestList()
     {
         $failedTestDetails = ['tests' => [], 'suites' => []];
 
-        if (realpath(self::TESTS_FAILED_FILE)) {
-            $testList = $this->readFailedTestFile(self::TESTS_FAILED_FILE);
+        if (realpath($this->testsFailedFile)) {
+            $testList = $this->readFailedTestFile($this->testsFailedFile);
 
             foreach ($testList as $test) {
                 if (!empty($test)) {
-                    $this->writeFailedTestToFile($test, self::TESTS_RERUN_FILE);
+                    $this->writeFailedTestToFile($test, $this->testsReRunFile);
                     $testInfo = explode(DIRECTORY_SEPARATOR, $test);
                     $testName = explode(":", $testInfo[count($testInfo) - 1])[1];
                     $suiteName = $testInfo[count($testInfo) - 2];
@@ -184,7 +198,7 @@ private function getFailedTestList()
      */
     private function readTestManifestFile()
     {
-        return file(self::TESTS_MANIFEST_FILE, FILE_IGNORE_NEW_LINES);
+        return file($this->testsManifestFile, FILE_IGNORE_NEW_LINES);
     }
 
     /**
@@ -202,6 +216,7 @@ private function readFailedTestFile($filePath)
      * Writes the test name to a file if it does not already exist
      *
      * @param string $test
+     * @param string $filePath
      * @return void
      */
     private function writeFailedTestToFile($test, $filePath)
diff --git a/src/Magento/FunctionalTestingFramework/Console/SetupEnvCommand.php b/src/Magento/FunctionalTestingFramework/Console/SetupEnvCommand.php
index 4d1ce0334..65e8c61e7 100644
--- a/src/Magento/FunctionalTestingFramework/Console/SetupEnvCommand.php
+++ b/src/Magento/FunctionalTestingFramework/Console/SetupEnvCommand.php
@@ -7,6 +7,8 @@
 
 namespace Magento\FunctionalTestingFramework\Console;
 
+use Magento\FunctionalTestingFramework\Exceptions\TestFrameworkException;
+use Magento\FunctionalTestingFramework\Util\Path\FilePathFormatter;
 use Symfony\Component\Console\Command\Command;
 use Symfony\Component\Console\Input\InputOption;
 use Symfony\Component\Console\Exception\InvalidOptionException;
@@ -27,12 +29,13 @@ class SetupEnvCommand extends Command
      * Configures the current command.
      *
      * @return void
+     * @throws TestFrameworkException
      */
     protected function configure()
     {
         $this->setName('setup:env')
             ->setDescription("Generate .env file.");
-        $this->envProcessor = new EnvProcessor(TESTS_BP . DIRECTORY_SEPARATOR . '.env');
+        $this->envProcessor = new EnvProcessor(FilePathFormatter::format(TESTS_BP) . '.env');
         $env = $this->envProcessor->getEnv();
         foreach ($env as $key => $value) {
             $this->addOption($key, null, InputOption::VALUE_REQUIRED, '', $value);
diff --git a/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/CredentialStore.php b/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/CredentialStore.php
index 0514cfe57..94ff40069 100644
--- a/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/CredentialStore.php
+++ b/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/CredentialStore.php
@@ -9,6 +9,7 @@
 use Magento\FunctionalTestingFramework\Exceptions\TestFrameworkException;
 use Magento\FunctionalTestingFramework\DataGenerator\Handlers\SecretStorage\FileStorage;
 use Magento\FunctionalTestingFramework\DataGenerator\Handlers\SecretStorage\VaultStorage;
+use Magento\FunctionalTestingFramework\Util\Path\UrlFormatter;
 
 class CredentialStore
 {
@@ -63,7 +64,7 @@ private function __construct()
         if ($cvAddress !== false && $cvSecretPath !== false) {
             try {
                 $this->credStorage[self::ARRAY_KEY_FOR_VAULT] = new VaultStorage(
-                    rtrim($cvAddress, '/'),
+                    UrlFormatter::format($cvAddress, false),
                     '/' . trim($cvSecretPath, '/')
                 );
             } catch (TestFrameworkException $e) {
diff --git a/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/SecretStorage/FileStorage.php b/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/SecretStorage/FileStorage.php
index 064610c79..be77a6de2 100644
--- a/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/SecretStorage/FileStorage.php
+++ b/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/SecretStorage/FileStorage.php
@@ -6,10 +6,10 @@
 
 namespace Magento\FunctionalTestingFramework\DataGenerator\Handlers\SecretStorage;
 
-use Magento\FunctionalTestingFramework\Console\BuildProjectCommand;
 use Magento\FunctionalTestingFramework\Exceptions\TestFrameworkException;
 use Magento\FunctionalTestingFramework\Config\MftfApplicationConfig;
 use Magento\FunctionalTestingFramework\Util\Logger\LoggingUtil;
+use Magento\FunctionalTestingFramework\Util\Path\FilePathFormatter;
 
 class FileStorage extends BaseStorage
 {
@@ -71,7 +71,7 @@ private function readInCredentialsFile()
         $credsFilePath = str_replace(
             '.credentials.example',
             '.credentials',
-            BuildProjectCommand::CREDENTIALS_FILE_PATH
+            FilePathFormatter::format(TESTS_BP) . '.credentials.example'
         );
 
         if (!file_exists($credsFilePath)) {
diff --git a/src/Magento/FunctionalTestingFramework/DataGenerator/Persist/Curl/AbstractExecutor.php b/src/Magento/FunctionalTestingFramework/DataGenerator/Persist/Curl/AbstractExecutor.php
index b6c3f29ec..e27da3718 100644
--- a/src/Magento/FunctionalTestingFramework/DataGenerator/Persist/Curl/AbstractExecutor.php
+++ b/src/Magento/FunctionalTestingFramework/DataGenerator/Persist/Curl/AbstractExecutor.php
@@ -6,6 +6,8 @@
 
 namespace Magento\FunctionalTestingFramework\DataGenerator\Persist\Curl;
 
+use Magento\FunctionalTestingFramework\Exceptions\TestFrameworkException;
+use Magento\FunctionalTestingFramework\Util\Path\UrlFormatter;
 use Magento\FunctionalTestingFramework\Util\Protocol\CurlInterface;
 
 /**
@@ -23,9 +25,10 @@ abstract class AbstractExecutor implements CurlInterface
     /**
      * Returns base URL for Magento instance
      * @return string
+     * @throws TestFrameworkException
      */
     public function getBaseUrl(): string
     {
-        return getenv('MAGENTO_BASE_URL');
+        return UrlFormatter::format(getenv('MAGENTO_BASE_URL'));
     }
 }
diff --git a/src/Magento/FunctionalTestingFramework/DataGenerator/Persist/Curl/AdminExecutor.php b/src/Magento/FunctionalTestingFramework/DataGenerator/Persist/Curl/AdminExecutor.php
index 4e11f5cec..86b3b742f 100644
--- a/src/Magento/FunctionalTestingFramework/DataGenerator/Persist/Curl/AdminExecutor.php
+++ b/src/Magento/FunctionalTestingFramework/DataGenerator/Persist/Curl/AdminExecutor.php
@@ -6,6 +6,7 @@
 
 namespace Magento\FunctionalTestingFramework\DataGenerator\Persist\Curl;
 
+use Magento\FunctionalTestingFramework\Util\Path\UrlFormatter;
 use Magento\FunctionalTestingFramework\Util\Protocol\CurlInterface;
 use Magento\FunctionalTestingFramework\Util\Protocol\CurlTransport;
 use Magento\FunctionalTestingFramework\Exceptions\TestFrameworkException;
@@ -59,11 +60,20 @@ public function __construct($removeBackend)
     /**
      * Returns base URL for Magento backend instance
      * @return string
+     * @throws TestFrameworkException
      */
     public function getBaseUrl(): string
     {
-        $backendHost = getenv('MAGENTO_BACKEND_BASE_URL') ?: parent::getBaseUrl();
-        return $backendHost . getenv('MAGENTO_BACKEND_NAME') . '/';
+        $backendHost = getenv('MAGENTO_BACKEND_BASE_URL')
+            ?
+            UrlFormatter::format(getenv('MAGENTO_BACKEND_BASE_URL'))
+            :
+            parent::getBaseUrl();
+        return empty(getenv('MAGENTO_BACKEND_NAME'))
+            ?
+            $backendHost
+            :
+            $backendHost . getenv('MAGENTO_BACKEND_NAME') . '/';
     }
 
     /**
diff --git a/src/Magento/FunctionalTestingFramework/DataGenerator/Persist/Curl/WebapiExecutor.php b/src/Magento/FunctionalTestingFramework/DataGenerator/Persist/Curl/WebapiExecutor.php
index 281eaa24d..494ce8f97 100644
--- a/src/Magento/FunctionalTestingFramework/DataGenerator/Persist/Curl/WebapiExecutor.php
+++ b/src/Magento/FunctionalTestingFramework/DataGenerator/Persist/Curl/WebapiExecutor.php
@@ -7,6 +7,7 @@
 namespace Magento\FunctionalTestingFramework\DataGenerator\Persist\Curl;
 
 use Magento\FunctionalTestingFramework\Exceptions\TestFrameworkException;
+use Magento\FunctionalTestingFramework\Util\Path\UrlFormatter;
 use Magento\FunctionalTestingFramework\Util\Protocol\CurlInterface;
 use Magento\FunctionalTestingFramework\Util\Protocol\CurlTransport;
 
@@ -75,24 +76,32 @@ public function __construct($storeCode = null)
     /**
      * Returns base URL for Magento Web API instance
      * @return string
+     * @throws TestFrameworkException
      */
     public function getBaseUrl(): string
     {
-        $baseUrl = parent::getBaseUrl();
-
         $webapiHost = getenv('MAGENTO_RESTAPI_SERVER_HOST');
         $webapiPort = getenv("MAGENTO_RESTAPI_SERVER_PORT");
         $webapiProtocol = getenv("MAGENTO_RESTAPI_SERVER_PROTOCOL");
 
-        if ($webapiHost) {
-            $baseUrl = sprintf('%s://%s/', $webapiProtocol, $webapiHost);
+        if ($webapiHost && $webapiProtocol) {
+            $baseUrl = UrlFormatter::format(
+                sprintf('%s://%s', $webapiProtocol, $webapiHost),
+                false
+            );
+        } elseif ($webapiHost) {
+            $baseUrl = UrlFormatter::format(sprintf('%s', $webapiProtocol, $webapiHost), false);
+        }
+
+        if (!isset($baseUrl)) {
+            $baseUrl = rtrim(parent::getBaseUrl(), '/');
         }
 
         if ($webapiPort) {
-            $baseUrl = rtrim($baseUrl, '/') . ':' . $webapiPort . '/';
+            $baseUrl .= ':' . $webapiPort;
         }
 
-        return $baseUrl;
+        return $baseUrl . '/';
     }
 
     /**
@@ -175,22 +184,23 @@ public function close()
      * Builds and returns URL for request, appending storeCode if needed.
      * @param string $resource
      * @return string
+     * @throws TestFrameworkException
      */
     public function getFormattedUrl($resource)
     {
         $urlResult = $this->getBaseUrl() . 'rest/';
         if ($this->storeCode != null) {
-            $urlResult .= $this->storeCode . "/";
+            $urlResult .= $this->storeCode . '/';
         }
-        $urlResult .= trim($resource, "/");
+        $urlResult .= trim($resource, '/');
         return $urlResult;
     }
 
     /**
      * Return admin auth token.
      *
-     * @throws TestFrameworkException
      * @return string
+     * @throws TestFrameworkException
      */
     public function getAuthToken()
     {
diff --git a/src/Magento/FunctionalTestingFramework/Module/MagentoWebDriver.php b/src/Magento/FunctionalTestingFramework/Module/MagentoWebDriver.php
index 215438420..2dfe237ac 100644
--- a/src/Magento/FunctionalTestingFramework/Module/MagentoWebDriver.php
+++ b/src/Magento/FunctionalTestingFramework/Module/MagentoWebDriver.php
@@ -16,6 +16,7 @@
 use Codeception\Util\Uri;
 use Magento\FunctionalTestingFramework\DataGenerator\Handlers\CredentialStore;
 use Magento\FunctionalTestingFramework\DataGenerator\Persist\Curl\WebapiExecutor;
+use Magento\FunctionalTestingFramework\Util\Path\UrlFormatter;
 use Magento\FunctionalTestingFramework\Util\Protocol\CurlInterface;
 use Magento\FunctionalTestingFramework\Util\ConfigSanitizerUtil;
 use Yandex\Allure\Adapter\AllureException;
@@ -869,7 +870,11 @@ private function curlExecMagentoCLI($command, $arguments): string
             str_replace('index.php', '', rtrim($this->config['url'], '/')),
             '/'
         );
-        $apiURL = $baseUrl . '/' . ltrim(getenv('MAGENTO_CLI_COMMAND_PATH'), '/');
+
+        $apiURL = UrlFormatter::format(
+            $baseUrl . '/' . ltrim(getenv('MAGENTO_CLI_COMMAND_PATH'), '/'),
+            false
+        );
 
         $restExecutor = new WebapiExecutor();
         $executor = new CurlTransport();
diff --git a/src/Magento/FunctionalTestingFramework/Suite/SuiteGenerator.php b/src/Magento/FunctionalTestingFramework/Suite/SuiteGenerator.php
index 9f0045d19..4df7daa75 100644
--- a/src/Magento/FunctionalTestingFramework/Suite/SuiteGenerator.php
+++ b/src/Magento/FunctionalTestingFramework/Suite/SuiteGenerator.php
@@ -6,6 +6,7 @@
 
 namespace Magento\FunctionalTestingFramework\Suite;
 
+use Magento\FunctionalTestingFramework\Exceptions\TestFrameworkException;
 use Magento\FunctionalTestingFramework\Exceptions\TestReferenceException;
 use Magento\FunctionalTestingFramework\Exceptions\XmlException;
 use Magento\FunctionalTestingFramework\Suite\Generators\GroupClassGenerator;
@@ -15,9 +16,14 @@
 use Magento\FunctionalTestingFramework\Util\Filesystem\DirSetupUtil;
 use Magento\FunctionalTestingFramework\Util\Logger\LoggingUtil;
 use Magento\FunctionalTestingFramework\Util\Manifest\BaseTestManifest;
+use Magento\FunctionalTestingFramework\Util\Path\FilePathFormatter;
 use Magento\FunctionalTestingFramework\Util\TestGenerator;
 use Symfony\Component\Yaml\Yaml;
 
+/**
+ * Class SuiteGenerator
+ * @SuppressWarnings(PHPMD.CouplingBetweenObjects)
+ */
 class SuiteGenerator
 {
     const YAML_CODECEPTION_DIST_FILENAME = 'codeception.dist.yml';
@@ -128,11 +134,12 @@ public function generateSuite($suiteName)
      * @return void
      * @throws TestReferenceException
      * @throws XmlException
+     * @throws TestFrameworkException
      */
     private function generateSuiteFromTest($suiteName, $tests = [], $originalSuiteName = null)
     {
         $relativePath = TestGenerator::GENERATED_DIR . DIRECTORY_SEPARATOR . $suiteName;
-        $fullPath = TESTS_MODULE_PATH . DIRECTORY_SEPARATOR . $relativePath . DIRECTORY_SEPARATOR;
+        $fullPath = FilePathFormatter::format(TESTS_MODULE_PATH) . $relativePath . DIRECTORY_SEPARATOR;
 
         DirSetupUtil::createGroupDir($fullPath);
 
@@ -348,9 +355,10 @@ private static function getYamlFileContents()
      * Static getter for the Config yml filepath (as path cannot be stored in a const)
      *
      * @return string
+     * @throws TestFrameworkException
      */
     private static function getYamlConfigFilePath()
     {
-        return TESTS_BP . DIRECTORY_SEPARATOR;
+        return FilePathFormatter::format(TESTS_BP);
     }
 }
diff --git a/src/Magento/FunctionalTestingFramework/Suite/Util/SuiteObjectExtractor.php b/src/Magento/FunctionalTestingFramework/Suite/Util/SuiteObjectExtractor.php
index 7ef3d5e8c..2e4d7f9dd 100644
--- a/src/Magento/FunctionalTestingFramework/Suite/Util/SuiteObjectExtractor.php
+++ b/src/Magento/FunctionalTestingFramework/Suite/Util/SuiteObjectExtractor.php
@@ -13,6 +13,7 @@
 use Magento\FunctionalTestingFramework\Test\Util\BaseObjectExtractor;
 use Magento\FunctionalTestingFramework\Test\Util\TestHookObjectExtractor;
 use Magento\FunctionalTestingFramework\Test\Util\TestObjectExtractor;
+use Magento\FunctionalTestingFramework\Util\Path\FilePathFormatter;
 use Magento\FunctionalTestingFramework\Util\Validation\NameValidationUtil;
 
 class SuiteObjectExtractor extends BaseObjectExtractor
@@ -259,8 +260,7 @@ private function resolveFilePathTestNames($filename, $moduleName = null)
     {
         $filepath = $filename;
         if (!strstr($filepath, DIRECTORY_SEPARATOR)) {
-            $filepath = TESTS_MODULE_PATH .
-                DIRECTORY_SEPARATOR .
+            $filepath = FilePathFormatter::format(TESTS_MODULE_PATH) .
                 $moduleName .
                 DIRECTORY_SEPARATOR .
                 'Test' .
@@ -293,8 +293,7 @@ private function resolveModulePathTestNames($moduleName)
     {
         $testObjects = [];
         $xmlFiles = glob(
-            TESTS_MODULE_PATH .
-            DIRECTORY_SEPARATOR .
+            FilePathFormatter::format(TESTS_MODULE_PATH) .
             $moduleName .
             DIRECTORY_SEPARATOR .
             'Test' .
diff --git a/src/Magento/FunctionalTestingFramework/Util/Env/EnvProcessor.php b/src/Magento/FunctionalTestingFramework/Util/Env/EnvProcessor.php
index f09ab63fa..86f7fa89d 100644
--- a/src/Magento/FunctionalTestingFramework/Util/Env/EnvProcessor.php
+++ b/src/Magento/FunctionalTestingFramework/Util/Env/EnvProcessor.php
@@ -7,6 +7,9 @@
 
 namespace Magento\FunctionalTestingFramework\Util\Env;
 
+use Magento\FunctionalTestingFramework\Exceptions\TestFrameworkException;
+use Magento\FunctionalTestingFramework\Util\Path\FilePathFormatter;
+
 /**
  * Helper class EnvProcessor for reading and writing .env files.
  *
@@ -45,13 +48,14 @@ class EnvProcessor
     /**
      * EnvProcessor constructor.
      * @param string $envFile
+     * @throws TestFrameworkException
      */
     public function __construct(
         string $envFile = ''
     ) {
         $this->envFile = $envFile;
         $this->envExists = file_exists($envFile);
-        $this->envExampleFile = realpath(FW_BP . "/etc/config/.env.example");
+        $this->envExampleFile = realpath(FilePathFormatter::format(FW_BP) . "etc/config/.env.example");
     }
 
     /**
diff --git a/src/Magento/FunctionalTestingFramework/Util/Logger/LoggingUtil.php b/src/Magento/FunctionalTestingFramework/Util/Logger/LoggingUtil.php
index 7ac128f28..97fdada21 100644
--- a/src/Magento/FunctionalTestingFramework/Util/Logger/LoggingUtil.php
+++ b/src/Magento/FunctionalTestingFramework/Util/Logger/LoggingUtil.php
@@ -6,6 +6,8 @@
 
 namespace Magento\FunctionalTestingFramework\Util\Logger;
 
+use Magento\FunctionalTestingFramework\Exceptions\TestFrameworkException;
+use Magento\FunctionalTestingFramework\Util\Path\FilePathFormatter;
 use Monolog\Handler\StreamHandler;
 use Monolog\Logger;
 
@@ -82,9 +84,10 @@ public function getLogger($className): MftfLogger
      * Function which returns a static path to the the log file.
      *
      * @return string
+     * @throws TestFrameworkException
      */
     public function getLoggingPath(): string
     {
-        return TESTS_BP . DIRECTORY_SEPARATOR . "mftf.log";
+        return FilePathFormatter::format(TESTS_BP) . "mftf.log";
     }
 }
diff --git a/src/Magento/FunctionalTestingFramework/Util/Manifest/TestManifestFactory.php b/src/Magento/FunctionalTestingFramework/Util/Manifest/TestManifestFactory.php
index 1cfa6559e..fd7d885bf 100644
--- a/src/Magento/FunctionalTestingFramework/Util/Manifest/TestManifestFactory.php
+++ b/src/Magento/FunctionalTestingFramework/Util/Manifest/TestManifestFactory.php
@@ -6,7 +6,9 @@
 
 namespace Magento\FunctionalTestingFramework\Util\Manifest;
 
+use Magento\FunctionalTestingFramework\Exceptions\TestFrameworkException;
 use Magento\FunctionalTestingFramework\Test\Handlers\TestObjectHandler;
+use Magento\FunctionalTestingFramework\Util\Path\FilePathFormatter;
 use Magento\FunctionalTestingFramework\Util\TestGenerator;
 
 class TestManifestFactory
@@ -26,11 +28,11 @@ private function __construct()
      * @param array  $suiteConfiguration
      * @param string $testPath
      * @return BaseTestManifest
+     * @throws TestFrameworkException
      */
     public static function makeManifest($runConfig, $suiteConfiguration, $testPath = TestGenerator::DEFAULT_DIR)
     {
-        $testDirFullPath = TESTS_MODULE_PATH
-        . DIRECTORY_SEPARATOR
+        $testDirFullPath = FilePathFormatter::format(TESTS_MODULE_PATH)
         . TestGenerator::GENERATED_DIR
         . DIRECTORY_SEPARATOR
         . $testPath;
diff --git a/src/Magento/FunctionalTestingFramework/Util/ModuleResolver.php b/src/Magento/FunctionalTestingFramework/Util/ModuleResolver.php
index d5e6a8920..aa9a08ddb 100644
--- a/src/Magento/FunctionalTestingFramework/Util/ModuleResolver.php
+++ b/src/Magento/FunctionalTestingFramework/Util/ModuleResolver.php
@@ -9,6 +9,7 @@
 use Magento\FunctionalTestingFramework\Config\MftfApplicationConfig;
 use Magento\FunctionalTestingFramework\Exceptions\TestFrameworkException;
 use Magento\FunctionalTestingFramework\Util\Logger\LoggingUtil;
+use Magento\FunctionalTestingFramework\Util\Path\FilePathFormatter;
 use Magento\FunctionalTestingFramework\Util\Path\UrlFormatter;
 use Symfony\Component\HttpFoundation\Response;
 
@@ -314,11 +315,11 @@ private function aggregateTestModulePaths()
         $allModulePaths = [];
 
         // Define the Module paths from magento bp
-        $magentoBaseCodePath = MAGENTO_BP;
+        $magentoBaseCodePath = FilePathFormatter::format(MAGENTO_BP, false);
 
         // Define the Module paths from default TESTS_MODULE_PATH
         $modulePath = defined('TESTS_MODULE_PATH') ? TESTS_MODULE_PATH : TESTS_BP;
-        $modulePath = rtrim($modulePath, DIRECTORY_SEPARATOR);
+        $modulePath = FilePathFormatter::format($modulePath, false);
 
         $vendorCodePath = DIRECTORY_SEPARATOR . self::VENDOR;
         $appCodePath = DIRECTORY_SEPARATOR . self::APP_CODE;
@@ -416,15 +417,16 @@ private static function globRelevantWrapper($testPath, $pattern)
      * Aggregate all code paths with test module composer json files
      *
      * @return array
+     * @throws TestFrameworkException
      */
     private function aggregateTestModulePathsFromComposerJson()
     {
         // Define the module paths
-        $magentoBaseCodePath = MAGENTO_BP;
+        $magentoBaseCodePath = FilePathFormatter::format(MAGENTO_BP, false);
 
         // Define the module paths from default TESTS_MODULE_PATH
         $modulePath = defined('TESTS_MODULE_PATH') ? TESTS_MODULE_PATH : TESTS_BP;
-        $modulePath = rtrim($modulePath, DIRECTORY_SEPARATOR);
+        $modulePath = FilePathFormatter::format($modulePath, false);
 
         $searchCodePaths = [
             $magentoBaseCodePath . DIRECTORY_SEPARATOR . self::DEV_TESTS,
@@ -719,7 +721,7 @@ public function getAdminToken()
             throw new TestFrameworkException($message, $context);
         }
 
-        $url = UrlFormatter::format($this->getBackendUrl()) . $this->adminTokenUrl;
+        $url = $this->getBackendUrl() . $this->adminTokenUrl;
         $data = [
             'username' => $login,
             'password' => $password
@@ -887,7 +889,15 @@ private function getRegisteredModuleList()
      */
     private function getBackendUrl()
     {
-        return getenv('MAGENTO_BACKEND_BASE_URL') ?: getenv('MAGENTO_BASE_URL');
+        try {
+            if (getenv('MAGENTO_BACKEND_BASE_URL')) {
+                return UrlFormatter::format(getenv('MAGENTO_BACKEND_BASE_URL'));
+            } else {
+                return UrlFormatter::format(getenv('MAGENTO_BASE_URL'));
+            }
+        } catch (TestFrameworkException $e) {
+            return null;
+        }
     }
 
     /**
diff --git a/src/Magento/FunctionalTestingFramework/Util/Path/FilePathFormatter.php b/src/Magento/FunctionalTestingFramework/Util/Path/FilePathFormatter.php
index 1337f1e3c..8b496e739 100644
--- a/src/Magento/FunctionalTestingFramework/Util/Path/FilePathFormatter.php
+++ b/src/Magento/FunctionalTestingFramework/Util/Path/FilePathFormatter.php
@@ -8,7 +8,7 @@
 
 use Magento\FunctionalTestingFramework\Exceptions\TestFrameworkException;
 
-class FilePathFormatter implements FileUrlFormatterInterface
+class FilePathFormatter implements FormatterInterface
 {
     /**
      * Return formatted full file path from input string, or false on error
@@ -18,7 +18,7 @@ class FilePathFormatter implements FileUrlFormatterInterface
      * @return string
      * @throws TestFrameworkException
      */
-    public static function format($path, $withTrailingSeparator = false)
+    public static function format($path, $withTrailingSeparator = true)
     {
         $validPath = realpath($path);
 
diff --git a/src/Magento/FunctionalTestingFramework/Util/Path/FileUrlFormatterInterface.php b/src/Magento/FunctionalTestingFramework/Util/Path/FormatterInterface.php
similarity index 92%
rename from src/Magento/FunctionalTestingFramework/Util/Path/FileUrlFormatterInterface.php
rename to src/Magento/FunctionalTestingFramework/Util/Path/FormatterInterface.php
index 3525dd75b..11de71204 100644
--- a/src/Magento/FunctionalTestingFramework/Util/Path/FileUrlFormatterInterface.php
+++ b/src/Magento/FunctionalTestingFramework/Util/Path/FormatterInterface.php
@@ -8,7 +8,7 @@
 
 use Magento\FunctionalTestingFramework\Exceptions\TestFrameworkException;
 
-interface FileUrlFormatterInterface
+interface FormatterInterface
 {
     /**
      * Return formatted path (file path, url, etc) from input string, or false on error
@@ -18,5 +18,5 @@ interface FileUrlFormatterInterface
      * @return string
      * @throws TestFrameworkException
      */
-    public static function format($input, $withTrailingSeparator = false);
+    public static function format($input, $withTrailingSeparator = true);
 }
diff --git a/src/Magento/FunctionalTestingFramework/Util/Path/UrlFormatter.php b/src/Magento/FunctionalTestingFramework/Util/Path/UrlFormatter.php
index 3c2a9499e..2ff1e430b 100644
--- a/src/Magento/FunctionalTestingFramework/Util/Path/UrlFormatter.php
+++ b/src/Magento/FunctionalTestingFramework/Util/Path/UrlFormatter.php
@@ -8,7 +8,7 @@
 
 use Magento\FunctionalTestingFramework\Exceptions\TestFrameworkException;
 
-class UrlFormatter implements FileUrlFormatterInterface
+class UrlFormatter implements FormatterInterface
 {
     /**
      * Return formatted url path from input string, or false on error
@@ -21,18 +21,22 @@ class UrlFormatter implements FileUrlFormatterInterface
     public static function format($url, $withTrailingSeparator = true)
     {
         $sanitizedUrl = rtrim($url, '/');
+
+        // Remove all characters except letters, digits and $-_.+!*'(),{}|\\^~[]`<>#%";/?:@&=
         $sanitizedUrl = filter_var($sanitizedUrl, FILTER_SANITIZE_URL);
 
         if (false === $sanitizedUrl) {
             throw new TestFrameworkException("Invalid url: $url\n");
         }
 
+        // Validate URL according to http://www.faqs.org/rfcs/rfc2396
         $validUrl = filter_var($sanitizedUrl, FILTER_VALIDATE_URL);
 
         if (false !== $validUrl) {
             return $withTrailingSeparator ? $validUrl . '/' : $validUrl;
         }
 
+        // Validation might be failed due to missing URL scheme or host, attempt to build them and re-validate
         $validUrl = filter_var(self::buildUrl($sanitizedUrl), FILTER_VALIDATE_URL);
 
         if (false !== $validUrl) {
diff --git a/src/Magento/FunctionalTestingFramework/Util/TestGenerator.php b/src/Magento/FunctionalTestingFramework/Util/TestGenerator.php
index ce45af963..648f3055f 100644
--- a/src/Magento/FunctionalTestingFramework/Util/TestGenerator.php
+++ b/src/Magento/FunctionalTestingFramework/Util/TestGenerator.php
@@ -10,6 +10,7 @@
 use Magento\FunctionalTestingFramework\DataGenerator\Handlers\CredentialStore;
 use Magento\FunctionalTestingFramework\DataGenerator\Handlers\PersistedObjectHandler;
 use Magento\FunctionalTestingFramework\DataGenerator\Objects\EntityDataObject;
+use Magento\FunctionalTestingFramework\Exceptions\TestFrameworkException;
 use Magento\FunctionalTestingFramework\Exceptions\TestReferenceException;
 use Magento\FunctionalTestingFramework\Suite\Handlers\SuiteObjectHandler;
 use Magento\FunctionalTestingFramework\Test\Handlers\ActionGroupObjectHandler;
@@ -26,6 +27,7 @@
 use Magento\FunctionalTestingFramework\Test\Util\TestObjectExtractor;
 use Magento\FunctionalTestingFramework\Util\Filesystem\DirSetupUtil;
 use Magento\FunctionalTestingFramework\Test\Util\ActionMergeUtil;
+use Magento\FunctionalTestingFramework\Util\Path\FilePathFormatter;
 
 /**
  * Class TestGenerator
@@ -103,14 +105,14 @@ class TestGenerator
      * @param string  $exportDir
      * @param array   $tests
      * @param boolean $debug
+     * @throws TestFrameworkException
      */
     private function __construct($exportDir, $tests, $debug = false)
     {
         // private constructor for factory
         $this->exportDirName = $exportDir ?? self::DEFAULT_DIR;
         $exportDir = $exportDir ?? self::DEFAULT_DIR;
-        $this->exportDirectory = TESTS_MODULE_PATH
-            . DIRECTORY_SEPARATOR
+        $this->exportDirectory = FilePathFormatter::format(TESTS_MODULE_PATH)
             . self::GENERATED_DIR
             . DIRECTORY_SEPARATOR
             . $exportDir;
diff --git a/src/Magento/FunctionalTestingFramework/_bootstrap.php b/src/Magento/FunctionalTestingFramework/_bootstrap.php
index e401123b6..f3bb099d0 100644
--- a/src/Magento/FunctionalTestingFramework/_bootstrap.php
+++ b/src/Magento/FunctionalTestingFramework/_bootstrap.php
@@ -61,7 +61,7 @@
 defined('MAGENTO_BP') || define('MAGENTO_BP', realpath(PROJECT_ROOT));
 // TODO REMOVE THIS CODE ONCE WE HAVE STOPPED SUPPORTING dev/tests/acceptance PATH
 // define TEST_PATH and TEST_MODULE_PATH
-defined('TESTS_BP') || define('TESTS_BP', realpath(MAGENTO_BP . DIRECTORY_SEPARATOR . 'dev/tests/acceptance/'));
+defined('TESTS_BP') || define('TESTS_BP', realpath(MAGENTO_BP . DIRECTORY_SEPARATOR . 'dev/tests/acceptance'));
 
 $RELATIVE_TESTS_MODULE_PATH = '/tests/functional/Magento/FunctionalTest';
 defined('TESTS_MODULE_PATH') || define(

From ab0d3c5bb77a1a70f7544df0287b04d519c13154 Mon Sep 17 00:00:00 2001
From: Kevin Kozan <kkozan@adobe.com>
Date: Thu, 21 Nov 2019 13:12:36 -0600
Subject: [PATCH 51/81] MQE-1520: Consolidate WAIT_TIMEOUT in .env and
 pageload_timeout

- Fixed Verification Tests
---
 dev/tests/_bootstrap.php | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/dev/tests/_bootstrap.php b/dev/tests/_bootstrap.php
index b41f80394..3f57ad139 100644
--- a/dev/tests/_bootstrap.php
+++ b/dev/tests/_bootstrap.php
@@ -45,7 +45,8 @@
     'MAGENTO_BACKEND_NAME' => 'admin',
     'MAGENTO_ADMIN_USERNAME' => 'admin',
     'MAGENTO_ADMIN_PASSWORD' => 'admin123',
-    'DEFAULT_TIMEZONE' => 'America/Los_Angeles'
+    'DEFAULT_TIMEZONE' => 'America/Los_Angeles',
+    'WAIT_TIMEOUT' => '10'
 ];
 
 foreach ($TEST_ENVS as $key => $value) {

From dc004a234a4529dc08d26606dbe519483b175a32 Mon Sep 17 00:00:00 2001
From: Kevin Kozan <kkozan@adobe.com>
Date: Thu, 21 Nov 2019 14:27:32 -0600
Subject: [PATCH 52/81] MQE-1627: RetrieveEntityField generation does not
 consider ActionGroup

- Testgenerator fix
- verification tests addition
---
 .../Resources/PersistenceCustomFieldsTest.txt | 31 +++++++++++++++++++
 .../ActionGroup/PersistenceActionGroup.xml    |  8 +++++
 .../Test/PersistenceCustomFieldsTest.xml      |  1 +
 .../Util/TestGenerator.php                    |  1 +
 4 files changed, 41 insertions(+)

diff --git a/dev/tests/verification/Resources/PersistenceCustomFieldsTest.txt b/dev/tests/verification/Resources/PersistenceCustomFieldsTest.txt
index 38dc364e7..0a6deaca8 100644
--- a/dev/tests/verification/Resources/PersistenceCustomFieldsTest.txt
+++ b/dev/tests/verification/Resources/PersistenceCustomFieldsTest.txt
@@ -111,5 +111,36 @@ class PersistenceCustomFieldsTestCest
 		);
 
 		$I->comment("Exiting Action Group [createdAG] PersistenceActionGroup");
+		$I->comment("Entering Action Group [AGKEY] DataPersistenceSelfReferenceActionGroup");
+		$I->comment("[createData1AGKEY] create 'entity1' entity");
+		PersistedObjectHandler::getInstance()->createEntity(
+			"createData1AGKEY",
+			"test",
+			"entity1",
+			[],
+			[]
+		);
+
+		$I->comment("[createData2AGKEY] create 'entity2' entity");
+		PersistedObjectHandler::getInstance()->createEntity(
+			"createData2AGKEY",
+			"test",
+			"entity2",
+			[],
+			[]
+		);
+
+		$createData3AGKEYFields['key1'] = PersistedObjectHandler::getInstance()->retrieveEntityField('createData1AGKEY', 'field', 'test');
+		$createData3AGKEYFields['key2'] = PersistedObjectHandler::getInstance()->retrieveEntityField('createData2AGKEY', 'field', 'test');
+		$I->comment("[createData3AGKEY] create 'entity3' entity");
+		PersistedObjectHandler::getInstance()->createEntity(
+			"createData3AGKEY",
+			"test",
+			"entity3",
+			[],
+			$createData3AGKEYFields
+		);
+
+		$I->comment("Exiting Action Group [AGKEY] DataPersistenceSelfReferenceActionGroup");
 	}
 }
diff --git a/dev/tests/verification/TestModule/ActionGroup/PersistenceActionGroup.xml b/dev/tests/verification/TestModule/ActionGroup/PersistenceActionGroup.xml
index c4f894cfc..f98fd2406 100644
--- a/dev/tests/verification/TestModule/ActionGroup/PersistenceActionGroup.xml
+++ b/dev/tests/verification/TestModule/ActionGroup/PersistenceActionGroup.xml
@@ -30,4 +30,12 @@
         <getData entity="someEneity" stepKey="getData"/>
         <comment userInput="$createData.field$" stepKey="comment"/>
     </actionGroup>
+    <actionGroup name="DataPersistenceSelfReferenceActionGroup">
+        <createData entity="entity1" stepKey="createData1"/>
+        <createData entity="entity2" stepKey="createData2"/>
+        <createData entity="entity3" stepKey="createData3">
+            <field key="key1">$createData1.field$</field>
+            <field key="key2">$createData2.field$</field>
+        </createData>
+    </actionGroup>
 </actionGroups>
\ No newline at end of file
diff --git a/dev/tests/verification/TestModule/Test/PersistenceCustomFieldsTest.xml b/dev/tests/verification/TestModule/Test/PersistenceCustomFieldsTest.xml
index 747c2c132..c56106693 100644
--- a/dev/tests/verification/TestModule/Test/PersistenceCustomFieldsTest.xml
+++ b/dev/tests/verification/TestModule/Test/PersistenceCustomFieldsTest.xml
@@ -33,5 +33,6 @@
             <argument name="arg2" value="DefaultPerson.firstname"/>
             <argument name="arg3" value="$createdData3.firstname$"/>
         </actionGroup>
+        <actionGroup ref="DataPersistenceSelfReferenceActionGroup" stepKey="AGKEY"/>
     </test>
 </tests>
diff --git a/src/Magento/FunctionalTestingFramework/Util/TestGenerator.php b/src/Magento/FunctionalTestingFramework/Util/TestGenerator.php
index ce45af963..2909ab1dc 100644
--- a/src/Magento/FunctionalTestingFramework/Util/TestGenerator.php
+++ b/src/Magento/FunctionalTestingFramework/Util/TestGenerator.php
@@ -1291,6 +1291,7 @@ public function generateStepsPhp($actionObjects, $generationScope = TestGenerato
                     break;
                 case "field":
                     $fieldKey = $actionObject->getCustomActionAttributes()['key'];
+                    $input = $this->resolveStepKeyReferences($input, $actionObject->getActionOrigin());
                     $input = $this->resolveTestVariable(
                         [$input],
                         $actionObject->getActionOrigin()

From 97f3b3ed6095d78f3d44300aa20807601b87a386 Mon Sep 17 00:00:00 2001
From: Soumya Unnikrishnan <sunnikri@adobe.com>
Date: Thu, 21 Nov 2019 16:54:01 -0600
Subject: [PATCH 53/81] MQE-1711: Switch between Developer mode and Production
 mode takes long time and the test end up time out.

---
 etc/config/command.php                        |  3 ++-
 .../Module/MagentoWebDriver.php               | 25 +++++++++++--------
 .../Test/etc/Actions/customActions.xsd        |  7 ++++++
 .../Util/TestGenerator.php                    |  9 +++++--
 4 files changed, 31 insertions(+), 13 deletions(-)

diff --git a/etc/config/command.php b/etc/config/command.php
index adc372e43..b348bd3e5 100644
--- a/etc/config/command.php
+++ b/etc/config/command.php
@@ -14,6 +14,7 @@
     $tokenPassedIn = urldecode($_POST['token'] ?? '');
     $command = urldecode($_POST['command'] ?? '');
     $arguments = urldecode($_POST['arguments'] ?? '');
+    $timeout = urldecode($_POST['timeout'] ?? 60);
 
     // Token returned will be null if the token we passed in is invalid
     $tokenFromMagento = $tokenModel->loadByToken($tokenPassedIn)->getToken();
@@ -24,7 +25,7 @@
         if ($valid) {
             $fullCommand = escapeshellcmd($magentoBinary . " $command" . " $arguments");
             $process = new Symfony\Component\Process\Process($fullCommand);
-            $process->setIdleTimeout(60);
+            $process->setIdleTimeout($timeout);
             $process->setTimeout(0);
             $idleTimeout = false;
             try {
diff --git a/src/Magento/FunctionalTestingFramework/Module/MagentoWebDriver.php b/src/Magento/FunctionalTestingFramework/Module/MagentoWebDriver.php
index c1ab892d7..de8feaae2 100644
--- a/src/Magento/FunctionalTestingFramework/Module/MagentoWebDriver.php
+++ b/src/Magento/FunctionalTestingFramework/Module/MagentoWebDriver.php
@@ -510,14 +510,16 @@ public function scrollToTopOfPage()
     /**
      * Takes given $command and executes it against bin/magento or custom exposed entrypoint. Returns command output.
      *
-     * @param string $command
-     * @param string $arguments
+     * @param string  $command
+     * @param string  $arguments
+     * @param integer $timeout
      * @return string
+     *
      * @throws TestFrameworkException
      */
-    public function magentoCLI($command, $arguments = null)
+    public function magentoCLI($command, $arguments = null, $timeout = null)
     {
-        return $this->curlExecMagentoCLI($command, $arguments);
+        return $this->curlExecMagentoCLI($command,  $arguments, $timeout);
         //TODO: calling bin/magento from pipeline is timing out, needs investigation (ref: MQE-1774)
 //        try {
 //            return $this->shellExecMagentoCLI($command, $arguments);
@@ -673,16 +675,17 @@ public function fillSecretField($field, $value)
      *
      * @param string $command
      * @param null   $arguments
+     * @param int    $timeout
      * @throws TestFrameworkException
      * @return string
      */
-    public function magentoCLISecret($command, $arguments = null)
+    public function magentoCLISecret($command, $timeout, $arguments = null)
     {
         // to protect any secrets from being printed to console the values are executed only at the webdriver level as a
         // decrypted value
 
         $decryptedCommand = CredentialStore::getInstance()->decryptAllSecretsInString($command);
-        return $this->magentoCLI($decryptedCommand, $arguments);
+        return $this->magentoCLI($decryptedCommand, $timeout, $arguments);
     }
 
     /**
@@ -860,12 +863,13 @@ private function shellExecMagentoCLI($command, $arguments): string
     /**
      * Takes given $command and executes it against exposed MTF CLI entry point. Returns response from server.
      *
-     * @param string $command
-     * @param string $arguments
-     * @return string
+     * @param string  $command
+     * @param string  $arguments
+     * @param integer $timeout
+     *
      * @throws TestFrameworkException
      */
-    private function curlExecMagentoCLI($command, $arguments): string
+    private function curlExecMagentoCLI($command,  $arguments, $timeout): string
     {
         // Remove index.php if it's present in url
         $baseUrl = rtrim(
@@ -882,6 +886,7 @@ private function curlExecMagentoCLI($command, $arguments): string
                 'token' => $restExecutor->getAuthToken(),
                 getenv('MAGENTO_CLI_COMMAND_PARAMETER') => $command,
                 'arguments' => $arguments,
+                'timeout'   => $timeout,
             ],
             CurlInterface::POST,
             []
diff --git a/src/Magento/FunctionalTestingFramework/Test/etc/Actions/customActions.xsd b/src/Magento/FunctionalTestingFramework/Test/etc/Actions/customActions.xsd
index 3a002e126..2424d0d31 100644
--- a/src/Magento/FunctionalTestingFramework/Test/etc/Actions/customActions.xsd
+++ b/src/Magento/FunctionalTestingFramework/Test/etc/Actions/customActions.xsd
@@ -50,6 +50,13 @@
                         </xs:documentation>
                     </xs:annotation>
                 </xs:attribute>
+                <xs:attribute type="xs:string" name="timeout">
+                    <xs:annotation>
+                        <xs:documentation>
+                            Idle timeout in seconds, defaulted to 60s when not specified.
+                        </xs:documentation>
+                    </xs:annotation>
+                </xs:attribute>
                 <xs:attributeGroup ref="commonActionAttributes"/>
             </xs:extension>
         </xs:simpleContent>
diff --git a/src/Magento/FunctionalTestingFramework/Util/TestGenerator.php b/src/Magento/FunctionalTestingFramework/Util/TestGenerator.php
index ce45af963..be9d26c96 100644
--- a/src/Magento/FunctionalTestingFramework/Util/TestGenerator.php
+++ b/src/Magento/FunctionalTestingFramework/Util/TestGenerator.php
@@ -42,6 +42,7 @@ class TestGenerator
     const HOOK_SCOPE = 'hook';
     const SUITE_SCOPE = 'suite';
     const PRESSKEY_ARRAY_ANCHOR_KEY = '987654321098765432109876543210';
+    const COMMAND_WAIT_TIMEOUT = 60;
     const PERSISTED_OBJECT_NOTATION_REGEX = '/\${1,2}[\w.\[\]]+\${1,2}/';
     const NO_STEPKEY_ACTIONS = [
         'comment',
@@ -532,6 +533,7 @@ public function generateStepsPhp($actionObjects, $generationScope = TestGenerato
             $sortOrder = null;
             $storeCode = null;
             $format = null;
+            $commandTimeout = null;
 
             $assertExpected = null;
             $assertActual = null;
@@ -609,10 +611,12 @@ public function generateStepsPhp($actionObjects, $generationScope = TestGenerato
                 $time = $customActionAttributes['time'];
             }
             if (isset($customActionAttributes['timeout'])) {
-                $time = $customActionAttributes['timeout'];
+                $commandTimeout = $customActionAttributes['timeout'];
             }
             $time = $time ?? ActionObject::getDefaultWaitTimeout();
 
+            $commandTimeout = $commandTimeout ?? self::COMMAND_WAIT_TIMEOUT;
+
             if (isset($customActionAttributes['parameterArray']) && $actionObject->getType() != 'pressKey') {
                 // validate the param array is in the correct format
                 $this->validateParameterArray($customActionAttributes['parameterArray']);
@@ -1280,7 +1284,8 @@ public function generateStepsPhp($actionObjects, $generationScope = TestGenerato
                         $actor,
                         $actionObject,
                         $command,
-                        $arguments
+                        $arguments,
+                        $commandTimeout
                     );
                     $testSteps .= sprintf(self::STEP_KEY_ANNOTATION, $stepKey) . PHP_EOL;
                     $testSteps .= sprintf(

From cc5932aac7ae1b903681071573f86d93e730b7ce Mon Sep 17 00:00:00 2001
From: Kevin Kozan <kkozan@adobe.com>
Date: Fri, 22 Nov 2019 10:23:47 -0600
Subject: [PATCH 54/81] MQE-1520: Consolidate WAIT_TIMEOUT in .env and
 pageload_timeout

- CR Fixes
---
 dev/tests/functional/standalone_bootstrap.php         | 4 ++--
 docs/configuration.md                                 | 8 ++++++++
 src/Magento/FunctionalTestingFramework/_bootstrap.php | 4 ++--
 3 files changed, 12 insertions(+), 4 deletions(-)

diff --git a/dev/tests/functional/standalone_bootstrap.php b/dev/tests/functional/standalone_bootstrap.php
index ab6372697..486c7566b 100755
--- a/dev/tests/functional/standalone_bootstrap.php
+++ b/dev/tests/functional/standalone_bootstrap.php
@@ -50,8 +50,8 @@
     defined('DEFAULT_TIMEZONE') || define('DEFAULT_TIMEZONE', 'America/Los_Angeles');
     $env->setEnvironmentVariable('DEFAULT_TIMEZONE', DEFAULT_TIMEZONE);
 
-    defined('WAIT_TIMEOUT') || define('WAIT_TIMEOUT', 10);
-    $env->setEnvironmentVariable('WAIT_TIMEOUT', 10);
+    defined('WAIT_TIMEOUT') || define('WAIT_TIMEOUT', 30);
+    $env->setEnvironmentVariable('WAIT_TIMEOUT', 30);
 
     try {
         new DateTimeZone(DEFAULT_TIMEZONE);
diff --git a/docs/configuration.md b/docs/configuration.md
index 4af2c4e82..5140c01e7 100644
--- a/docs/configuration.md
+++ b/docs/configuration.md
@@ -295,6 +295,14 @@ Denoted in browser log entry as `"SOURCE": "type"`.
 BROWSER_LOG_BLACKLIST=other,console-api
 ```
 
+### WAIT_TIMEOUT
+
+Global MFTF configuration for the default amount of time (in seconds) that a test will wait while loading a page.
+
+```conf
+WAIT_TIMEOUT=30
+```
+
 <!-- Link definitions -->
 
 [`MAGENTO_CLI_COMMAND_PATH`]: #magento_cli_command_path
diff --git a/src/Magento/FunctionalTestingFramework/_bootstrap.php b/src/Magento/FunctionalTestingFramework/_bootstrap.php
index 490ff61de..ebc62e95d 100644
--- a/src/Magento/FunctionalTestingFramework/_bootstrap.php
+++ b/src/Magento/FunctionalTestingFramework/_bootstrap.php
@@ -50,8 +50,8 @@
     defined('DEFAULT_TIMEZONE') || define('DEFAULT_TIMEZONE', 'America/Los_Angeles');
     $env->setEnvironmentVariable('DEFAULT_TIMEZONE', DEFAULT_TIMEZONE);
 
-    defined('WAIT_TIMEOUT') || define('WAIT_TIMEOUT', 10);
-    $env->setEnvironmentVariable('WAIT_TIMEOUT', 10);
+    defined('WAIT_TIMEOUT') || define('WAIT_TIMEOUT', 30);
+    $env->setEnvironmentVariable('WAIT_TIMEOUT', 30);
 
     try {
         new DateTimeZone(DEFAULT_TIMEZONE);

From 16e1f03ec072c6734f37e2338ceb450fa28fbb63 Mon Sep 17 00:00:00 2001
From: Soumya Unnikrishnan <sunnikri@adobe.com>
Date: Fri, 22 Nov 2019 13:45:05 -0600
Subject: [PATCH 55/81] MQE-1711: Switch between Developer mode and Production
 mode takes long time and the test end up time out

---
 .../Resources/BasicFunctionalTest.txt         |  2 +-
 .../Resources/DataReplacementTest.txt         |  2 +-
 .../Module/MagentoWebDriver.php               | 24 ++++++++++---------
 .../Test/Objects/ActionObject.php             |  2 ++
 .../Util/TestGenerator.php                    | 12 ++++++----
 5 files changed, 24 insertions(+), 18 deletions(-)

diff --git a/dev/tests/verification/Resources/BasicFunctionalTest.txt b/dev/tests/verification/Resources/BasicFunctionalTest.txt
index f82c84119..c5dc13e2e 100644
--- a/dev/tests/verification/Resources/BasicFunctionalTest.txt
+++ b/dev/tests/verification/Resources/BasicFunctionalTest.txt
@@ -126,7 +126,7 @@ class BasicFunctionalTestCest
 		$grabMultipleKey1 = $I->grabMultiple(".functionalTestSelector"); // stepKey: grabMultipleKey1
 		$grabTextFromKey1 = $I->grabTextFrom(".functionalTestSelector"); // stepKey: grabTextFromKey1
 		$grabValueFromKey1 = $I->grabValueFrom(".functionalTestSelector"); // stepKey: grabValueFromKey1
-		$magentoCli1 = $I->magentoCLI("maintenance:enable", "\"stuffHere\""); // stepKey: magentoCli1
+		$magentoCli1 = $I->magentoCLI("maintenance:enable", "\"stuffHere\"", 60); // stepKey: magentoCli1
 		$I->comment($magentoCli1);
 		$I->makeScreenshot("screenShotInput"); // stepKey: makeScreenshotKey1
 		$I->maximizeWindow(); // stepKey: maximizeWindowKey1
diff --git a/dev/tests/verification/Resources/DataReplacementTest.txt b/dev/tests/verification/Resources/DataReplacementTest.txt
index 731ae63f8..4b4531a4e 100644
--- a/dev/tests/verification/Resources/DataReplacementTest.txt
+++ b/dev/tests/verification/Resources/DataReplacementTest.txt
@@ -52,7 +52,7 @@ class DataReplacementTestCest
 		$I->searchAndMultiSelectOption("#selector", [msq("uniqueData") . "John", "Doe" . msq("uniqueData")]); // stepKey: parameterArrayReplacementMSQBoth
 		$I->selectMultipleOptions("#Doe" . msq("uniqueData"), "#element", [msq("uniqueData") . "John", "Doe" . msq("uniqueData")]); // stepKey: multiSelectDataReplacement
 		$I->fillField(".selector", "0"); // stepKey: insertZero
-		$insertCommand = $I->magentoCLI("do something Doe" . msq("uniqueData") . " with uniqueness"); // stepKey: insertCommand
+		$insertCommand = $I->magentoCLI("do something Doe" . msq("uniqueData") . " with uniqueness", 60); // stepKey: insertCommand
 		$I->comment($insertCommand);
 		$I->seeInPageSource("StringBefore John StringAfter"); // stepKey: htmlReplace1
 		$I->seeInPageSource("#John"); // stepKey: htmlReplace2
diff --git a/src/Magento/FunctionalTestingFramework/Module/MagentoWebDriver.php b/src/Magento/FunctionalTestingFramework/Module/MagentoWebDriver.php
index de8feaae2..3602ddfb4 100644
--- a/src/Magento/FunctionalTestingFramework/Module/MagentoWebDriver.php
+++ b/src/Magento/FunctionalTestingFramework/Module/MagentoWebDriver.php
@@ -519,7 +519,7 @@ public function scrollToTopOfPage()
      */
     public function magentoCLI($command, $arguments = null, $timeout = null)
     {
-        return $this->curlExecMagentoCLI($command,  $arguments, $timeout);
+        return $this->curlExecMagentoCLI($command, $arguments, $timeout);
         //TODO: calling bin/magento from pipeline is timing out, needs investigation (ref: MQE-1774)
 //        try {
 //            return $this->shellExecMagentoCLI($command, $arguments);
@@ -673,19 +673,19 @@ public function fillSecretField($field, $value)
      * Function used to create data that contains sensitive credentials in a <createData> <field> override.
      * The data is decrypted immediately prior to data creation to avoid exposure in console or log.
      *
-     * @param string $command
-     * @param null   $arguments
-     * @param int    $timeout
+     * @param string  $command
+     * @param null    $arguments
+     * @param null    $timeout
      * @throws TestFrameworkException
      * @return string
      */
-    public function magentoCLISecret($command, $timeout, $arguments = null)
+    public function magentoCLISecret($command, $arguments = null, $timeout = null)
     {
         // to protect any secrets from being printed to console the values are executed only at the webdriver level as a
         // decrypted value
 
         $decryptedCommand = CredentialStore::getInstance()->decryptAllSecretsInString($command);
-        return $this->magentoCLI($decryptedCommand, $timeout, $arguments);
+        return $this->magentoCLI($decryptedCommand, $arguments, $timeout);
     }
 
     /**
@@ -837,20 +837,21 @@ public function makeScreenshot($name = null)
     /**
      * Takes given $command and executes it against bin/magento executable. Returns stdout output from the command.
      *
-     * @param string $command
-     * @param string $arguments
+     * @param string  $command
+     * @param string  $arguments
+     * @param integer $timeout
      *
      * @throws \RuntimeException
      * @return string
      * @SuppressWarnings(PHPMD.UnusedPrivateMethod)
      */
-    private function shellExecMagentoCLI($command, $arguments): string
+    private function shellExecMagentoCLI($command, $arguments, $timeout): string
     {
         $php = PHP_BINDIR ? PHP_BINDIR . DIRECTORY_SEPARATOR. 'php' : 'php';
         $binMagento = realpath(MAGENTO_BP . DIRECTORY_SEPARATOR . 'bin' . DIRECTORY_SEPARATOR . 'magento');
         $command = $php . ' -f ' . $binMagento . ' ' . $command . ' ' . $arguments;
         $process = new Process(escapeshellcmd($command), MAGENTO_BP);
-        $process->setIdleTimeout(60);
+        $process->setIdleTimeout($timeout);
         $process->setTimeout(0);
         $exitCode = $process->run();
         if ($exitCode !== 0) {
@@ -867,9 +868,10 @@ private function shellExecMagentoCLI($command, $arguments): string
      * @param string  $arguments
      * @param integer $timeout
      *
+     * @return string
      * @throws TestFrameworkException
      */
-    private function curlExecMagentoCLI($command,  $arguments, $timeout): string
+    private function curlExecMagentoCLI($command, $arguments, $timeout): string
     {
         // Remove index.php if it's present in url
         $baseUrl = rtrim(
diff --git a/src/Magento/FunctionalTestingFramework/Test/Objects/ActionObject.php b/src/Magento/FunctionalTestingFramework/Test/Objects/ActionObject.php
index 9b1cdf1af..0434ca22d 100644
--- a/src/Magento/FunctionalTestingFramework/Test/Objects/ActionObject.php
+++ b/src/Magento/FunctionalTestingFramework/Test/Objects/ActionObject.php
@@ -63,6 +63,7 @@ class ActionObject
     const DELETE_DATA_MUTUAL_EXCLUSIVE_ATTRIBUTES = ["url", "createDataKey"];
     const EXTERNAL_URL_AREA_INVALID_ACTIONS = ['amOnPage'];
     const FUNCTION_CLOSURE_ACTIONS = ['waitForElementChange', 'performOn', 'executeInSelenium'];
+    const COMMAND_ACTION_ATTRIBUTES = ['magentoCLI', 'magentoCLISecret'];
     const MERGE_ACTION_ORDER_AFTER = 'after';
     const MERGE_ACTION_ORDER_BEFORE = 'before';
     const ACTION_ATTRIBUTE_TIMEZONE = 'timezone';
@@ -72,6 +73,7 @@ class ActionObject
     const ACTION_ATTRIBUTE_VARIABLE_REGEX_PATTERN = '/({{[\w]+\.[\w\[\]]+}})|({{[\w]+\.[\w]+\((?(?!}}).)+\)}})/';
     const STRING_PARAMETER_REGEX = "/'[^']+'/";
     const DEFAULT_WAIT_TIMEOUT = 10;
+    const DEFAULT_COMMAND_WAIT_TIMEOUT = 60;
     const ACTION_ATTRIBUTE_USERINPUT = 'userInput';
     const ACTION_TYPE_COMMENT = 'comment';
 
diff --git a/src/Magento/FunctionalTestingFramework/Util/TestGenerator.php b/src/Magento/FunctionalTestingFramework/Util/TestGenerator.php
index be9d26c96..766f65b60 100644
--- a/src/Magento/FunctionalTestingFramework/Util/TestGenerator.php
+++ b/src/Magento/FunctionalTestingFramework/Util/TestGenerator.php
@@ -42,7 +42,6 @@ class TestGenerator
     const HOOK_SCOPE = 'hook';
     const SUITE_SCOPE = 'suite';
     const PRESSKEY_ARRAY_ANCHOR_KEY = '987654321098765432109876543210';
-    const COMMAND_WAIT_TIMEOUT = 60;
     const PERSISTED_OBJECT_NOTATION_REGEX = '/\${1,2}[\w.\[\]]+\${1,2}/';
     const NO_STEPKEY_ACTIONS = [
         'comment',
@@ -611,11 +610,14 @@ public function generateStepsPhp($actionObjects, $generationScope = TestGenerato
                 $time = $customActionAttributes['time'];
             }
             if (isset($customActionAttributes['timeout'])) {
-                $commandTimeout = $customActionAttributes['timeout'];
+                $time = $customActionAttributes['timeout'];
             }
-            $time = $time ?? ActionObject::getDefaultWaitTimeout();
 
-            $commandTimeout = $commandTimeout ?? self::COMMAND_WAIT_TIMEOUT;
+            if (in_array($actionObject->getType(), ActionObject::COMMAND_ACTION_ATTRIBUTES)) {
+                $time = $time ?? ActionObject::DEFAULT_COMMAND_WAIT_TIMEOUT;
+            } else {
+                $time = $time ?? ActionObject::getDefaultWaitTimeout();
+            }
 
             if (isset($customActionAttributes['parameterArray']) && $actionObject->getType() != 'pressKey') {
                 // validate the param array is in the correct format
@@ -1285,7 +1287,7 @@ public function generateStepsPhp($actionObjects, $generationScope = TestGenerato
                         $actionObject,
                         $command,
                         $arguments,
-                        $commandTimeout
+                        $time
                     );
                     $testSteps .= sprintf(self::STEP_KEY_ANNOTATION, $stepKey) . PHP_EOL;
                     $testSteps .= sprintf(

From fceb17f52104dadb3e38a562a8c8730a9b2bd023 Mon Sep 17 00:00:00 2001
From: Soumya Unnikrishnan <sunnikri@adobe.com>
Date: Fri, 22 Nov 2019 14:46:57 -0600
Subject: [PATCH 56/81] MQE-1711: Switch between Developer mode and Production
 mode takes long time and the test end up time out fixed static checks + unit
 tests, added documentation.

---
 .../Resources/ActionGroupWithStepKeyReferences.txt          | 2 +-
 docs/test/actions.md                                        | 1 +
 .../FunctionalTestingFramework/Module/MagentoWebDriver.php  | 6 +++---
 3 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/dev/tests/verification/Resources/ActionGroupWithStepKeyReferences.txt b/dev/tests/verification/Resources/ActionGroupWithStepKeyReferences.txt
index eddaaf784..2bb414230 100644
--- a/dev/tests/verification/Resources/ActionGroupWithStepKeyReferences.txt
+++ b/dev/tests/verification/Resources/ActionGroupWithStepKeyReferences.txt
@@ -45,7 +45,7 @@ class ActionGroupWithStepKeyReferencesCest
 		$I->fillField($action1); // stepKey: action1ActionGroup
 		$I->comment("Invocation stepKey will be appended in non stepKey instances");
 		$action3ActionGroup = $I->executeJS($action3ActionGroup); // stepKey: action3ActionGroup
-		$action4ActionGroup = $I->magentoCLI($action4ActionGroup, "\"stuffHere\""); // stepKey: action4ActionGroup
+		$action4ActionGroup = $I->magentoCLI($action4ActionGroup, "\"stuffHere\"", 60); // stepKey: action4ActionGroup
 		$I->comment($action4ActionGroup);
 		$date = new \DateTime();
 		$date->setTimestamp(strtotime("{$action5}"));
diff --git a/docs/test/actions.md b/docs/test/actions.md
index 9b125d7fc..c5dc83fdb 100644
--- a/docs/test/actions.md
+++ b/docs/test/actions.md
@@ -1262,6 +1262,7 @@ Attribute|Type|Use|Description
 ---|---|---|---
 `command`|string |optional| CLI command to be executed in Magento environment.
 `arguments`|string |optional| Unescaped arguments to be passed in with the CLI command.
+`timeout`|string|optional| Number of seconds CLI command can run without outputting anything.
 `stepKey`|string|required| A unique identifier of the action.
 `before`|string|optional| `stepKey` of action that must be executed next.
 `after`|string|optional| `stepKey` of preceding action.
diff --git a/src/Magento/FunctionalTestingFramework/Module/MagentoWebDriver.php b/src/Magento/FunctionalTestingFramework/Module/MagentoWebDriver.php
index 3602ddfb4..2d09e7f2a 100644
--- a/src/Magento/FunctionalTestingFramework/Module/MagentoWebDriver.php
+++ b/src/Magento/FunctionalTestingFramework/Module/MagentoWebDriver.php
@@ -673,9 +673,9 @@ public function fillSecretField($field, $value)
      * Function used to create data that contains sensitive credentials in a <createData> <field> override.
      * The data is decrypted immediately prior to data creation to avoid exposure in console or log.
      *
-     * @param string  $command
-     * @param null    $arguments
-     * @param null    $timeout
+     * @param string $command
+     * @param null   $arguments
+     * @param null   $timeout
      * @throws TestFrameworkException
      * @return string
      */

From 2e1887e1b92bdd9333767541488449e4d15f285d Mon Sep 17 00:00:00 2001
From: Soumya Unnikrishnan <sunnikri@adobe.com>
Date: Fri, 22 Nov 2019 16:08:52 -0600
Subject: [PATCH 57/81] MQE-1711: Switch between Developer mode and Production
 mode takes long time and the test end up time out

---
 etc/config/command.php                                        | 2 +-
 src/Magento/FunctionalTestingFramework/Util/TestGenerator.php | 1 -
 2 files changed, 1 insertion(+), 2 deletions(-)

diff --git a/etc/config/command.php b/etc/config/command.php
index b348bd3e5..e3b8f1191 100644
--- a/etc/config/command.php
+++ b/etc/config/command.php
@@ -14,7 +14,7 @@
     $tokenPassedIn = urldecode($_POST['token'] ?? '');
     $command = urldecode($_POST['command'] ?? '');
     $arguments = urldecode($_POST['arguments'] ?? '');
-    $timeout = urldecode($_POST['timeout'] ?? 60);
+    $timeout = floatval(urldecode($_POST['timeout'] ?? 60));
 
     // Token returned will be null if the token we passed in is invalid
     $tokenFromMagento = $tokenModel->loadByToken($tokenPassedIn)->getToken();
diff --git a/src/Magento/FunctionalTestingFramework/Util/TestGenerator.php b/src/Magento/FunctionalTestingFramework/Util/TestGenerator.php
index 766f65b60..a566e95f6 100644
--- a/src/Magento/FunctionalTestingFramework/Util/TestGenerator.php
+++ b/src/Magento/FunctionalTestingFramework/Util/TestGenerator.php
@@ -532,7 +532,6 @@ public function generateStepsPhp($actionObjects, $generationScope = TestGenerato
             $sortOrder = null;
             $storeCode = null;
             $format = null;
-            $commandTimeout = null;
 
             $assertExpected = null;
             $assertActual = null;

From 8f72ffc123da6a0a1b7868ba2468f05c580f2ce2 Mon Sep 17 00:00:00 2001
From: Soumya Unnikrishnan <sunnikri@adobe.com>
Date: Mon, 25 Nov 2019 11:21:00 -0600
Subject: [PATCH 58/81] MQE-1520: Consolidate WAIT_TIMEOUT in .env and
 pageload_timeout Updated .env.example

---
 etc/config/.env.example | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/etc/config/.env.example b/etc/config/.env.example
index bb1d1ad63..e290c7815 100644
--- a/etc/config/.env.example
+++ b/etc/config/.env.example
@@ -53,7 +53,7 @@ MODULE_WHITELIST=Magento_Framework,ConfigurableProductWishlist,ConfigurableProdu
 #ALLOW_SKIPPED=true
 
 #*** Default timeout for wait actions
-#WAIT_TIMEOUT=10
+#WAIT_TIMEOUT=30
 
 #*** Uncomment and set to enable browser log entries on actions in Allure. Blacklist is used to filter logs of a specific "source"
 #ENABLE_BROWSER_LOG=true

From d4faf137dbefe25850deea423a1c23cd803f57a1 Mon Sep 17 00:00:00 2001
From: Soumya Unnikrishnan <sunnikri@adobe.com>
Date: Mon, 25 Nov 2019 15:07:05 -0600
Subject: [PATCH 59/81] MQE-1711: Switch between Developer mode and Production
 mode takes long time and the test end up time out fixed issue with signature
 of magentoCLI

---
 .../ActionGroupWithStepKeyReferences.txt       |  2 +-
 .../Resources/BasicFunctionalTest.txt          |  4 +++-
 .../TestModule/Test/BasicFunctionalTest.xml    |  1 +
 .../Module/MagentoWebDriver.php                | 18 +++++++++---------
 .../Util/TestGenerator.php                     |  4 ++--
 5 files changed, 16 insertions(+), 13 deletions(-)

diff --git a/dev/tests/verification/Resources/ActionGroupWithStepKeyReferences.txt b/dev/tests/verification/Resources/ActionGroupWithStepKeyReferences.txt
index 2bb414230..d1cae1da4 100644
--- a/dev/tests/verification/Resources/ActionGroupWithStepKeyReferences.txt
+++ b/dev/tests/verification/Resources/ActionGroupWithStepKeyReferences.txt
@@ -45,7 +45,7 @@ class ActionGroupWithStepKeyReferencesCest
 		$I->fillField($action1); // stepKey: action1ActionGroup
 		$I->comment("Invocation stepKey will be appended in non stepKey instances");
 		$action3ActionGroup = $I->executeJS($action3ActionGroup); // stepKey: action3ActionGroup
-		$action4ActionGroup = $I->magentoCLI($action4ActionGroup, "\"stuffHere\"", 60); // stepKey: action4ActionGroup
+		$action4ActionGroup = $I->magentoCLI($action4ActionGroup, 60, "\"stuffHere\""); // stepKey: action4ActionGroup
 		$I->comment($action4ActionGroup);
 		$date = new \DateTime();
 		$date->setTimestamp(strtotime("{$action5}"));
diff --git a/dev/tests/verification/Resources/BasicFunctionalTest.txt b/dev/tests/verification/Resources/BasicFunctionalTest.txt
index c5dc13e2e..2bae80f80 100644
--- a/dev/tests/verification/Resources/BasicFunctionalTest.txt
+++ b/dev/tests/verification/Resources/BasicFunctionalTest.txt
@@ -126,8 +126,10 @@ class BasicFunctionalTestCest
 		$grabMultipleKey1 = $I->grabMultiple(".functionalTestSelector"); // stepKey: grabMultipleKey1
 		$grabTextFromKey1 = $I->grabTextFrom(".functionalTestSelector"); // stepKey: grabTextFromKey1
 		$grabValueFromKey1 = $I->grabValueFrom(".functionalTestSelector"); // stepKey: grabValueFromKey1
-		$magentoCli1 = $I->magentoCLI("maintenance:enable", "\"stuffHere\"", 60); // stepKey: magentoCli1
+		$magentoCli1 = $I->magentoCLI("maintenance:enable", 60, "\"stuffHere\""); // stepKey: magentoCli1
 		$I->comment($magentoCli1);
+		$magentoCli2 = $I->magentoCLI("maintenance:enable", 120, "\"stuffHere\""); // stepKey: magentoCli2
+		$I->comment($magentoCli2);
 		$I->makeScreenshot("screenShotInput"); // stepKey: makeScreenshotKey1
 		$I->maximizeWindow(); // stepKey: maximizeWindowKey1
 		$I->moveBack(); // stepKey: moveBackKey1
diff --git a/dev/tests/verification/TestModule/Test/BasicFunctionalTest.xml b/dev/tests/verification/TestModule/Test/BasicFunctionalTest.xml
index ff0708554..09fedd1b8 100644
--- a/dev/tests/verification/TestModule/Test/BasicFunctionalTest.xml
+++ b/dev/tests/verification/TestModule/Test/BasicFunctionalTest.xml
@@ -76,6 +76,7 @@
         <grabTextFrom selector=".functionalTestSelector" stepKey="grabTextFromKey1" />
         <grabValueFrom selector=".functionalTestSelector" stepKey="grabValueFromKey1" />
         <magentoCLI command="maintenance:enable" arguments="&quot;stuffHere&quot;" stepKey="magentoCli1"/>
+        <magentoCLI command="maintenance:enable" arguments="&quot;stuffHere&quot;" timeout="120" stepKey="magentoCli2"/>
         <makeScreenshot userInput="screenShotInput" stepKey="makeScreenshotKey1"/>
         <maximizeWindow stepKey="maximizeWindowKey1"/>
         <moveBack stepKey="moveBackKey1"/>
diff --git a/src/Magento/FunctionalTestingFramework/Module/MagentoWebDriver.php b/src/Magento/FunctionalTestingFramework/Module/MagentoWebDriver.php
index 2d09e7f2a..87e3b0c08 100644
--- a/src/Magento/FunctionalTestingFramework/Module/MagentoWebDriver.php
+++ b/src/Magento/FunctionalTestingFramework/Module/MagentoWebDriver.php
@@ -511,15 +511,15 @@ public function scrollToTopOfPage()
      * Takes given $command and executes it against bin/magento or custom exposed entrypoint. Returns command output.
      *
      * @param string  $command
-     * @param string  $arguments
      * @param integer $timeout
+     * @param string  $arguments
      * @return string
      *
      * @throws TestFrameworkException
      */
-    public function magentoCLI($command, $arguments = null, $timeout = null)
+    public function magentoCLI($command, $timeout = null, $arguments = null)
     {
-        return $this->curlExecMagentoCLI($command, $arguments, $timeout);
+        return $this->curlExecMagentoCLI($command, $timeout, $arguments);
         //TODO: calling bin/magento from pipeline is timing out, needs investigation (ref: MQE-1774)
 //        try {
 //            return $this->shellExecMagentoCLI($command, $arguments);
@@ -674,18 +674,18 @@ public function fillSecretField($field, $value)
      * The data is decrypted immediately prior to data creation to avoid exposure in console or log.
      *
      * @param string $command
-     * @param null   $arguments
      * @param null   $timeout
+     * @param null   $arguments
      * @throws TestFrameworkException
      * @return string
      */
-    public function magentoCLISecret($command, $arguments = null, $timeout = null)
+    public function magentoCLISecret($command, $timeout = null, $arguments = null)
     {
         // to protect any secrets from being printed to console the values are executed only at the webdriver level as a
         // decrypted value
 
         $decryptedCommand = CredentialStore::getInstance()->decryptAllSecretsInString($command);
-        return $this->magentoCLI($decryptedCommand, $arguments, $timeout);
+        return $this->magentoCLI($decryptedCommand, $timeout, $arguments);
     }
 
     /**
@@ -845,7 +845,7 @@ public function makeScreenshot($name = null)
      * @return string
      * @SuppressWarnings(PHPMD.UnusedPrivateMethod)
      */
-    private function shellExecMagentoCLI($command, $arguments, $timeout): string
+    private function shellExecMagentoCLI($command, $arguments, $timeout = 60): string
     {
         $php = PHP_BINDIR ? PHP_BINDIR . DIRECTORY_SEPARATOR. 'php' : 'php';
         $binMagento = realpath(MAGENTO_BP . DIRECTORY_SEPARATOR . 'bin' . DIRECTORY_SEPARATOR . 'magento');
@@ -865,13 +865,13 @@ private function shellExecMagentoCLI($command, $arguments, $timeout): string
      * Takes given $command and executes it against exposed MTF CLI entry point. Returns response from server.
      *
      * @param string  $command
-     * @param string  $arguments
      * @param integer $timeout
+     * @param string  $arguments
      *
      * @return string
      * @throws TestFrameworkException
      */
-    private function curlExecMagentoCLI($command, $arguments, $timeout): string
+    private function curlExecMagentoCLI($command, $timeout, $arguments): string
     {
         // Remove index.php if it's present in url
         $baseUrl = rtrim(
diff --git a/src/Magento/FunctionalTestingFramework/Util/TestGenerator.php b/src/Magento/FunctionalTestingFramework/Util/TestGenerator.php
index a566e95f6..a73054b08 100644
--- a/src/Magento/FunctionalTestingFramework/Util/TestGenerator.php
+++ b/src/Magento/FunctionalTestingFramework/Util/TestGenerator.php
@@ -1285,8 +1285,8 @@ public function generateStepsPhp($actionObjects, $generationScope = TestGenerato
                         $actor,
                         $actionObject,
                         $command,
-                        $arguments,
-                        $time
+                        $time,
+                        $arguments
                     );
                     $testSteps .= sprintf(self::STEP_KEY_ANNOTATION, $stepKey) . PHP_EOL;
                     $testSteps .= sprintf(

From 67472f582a3d386e2c47c6a17cb107edbf07d360 Mon Sep 17 00:00:00 2001
From: Soumya Unnikrishnan <sunnikri@adobe.com>
Date: Mon, 25 Nov 2019 15:23:52 -0600
Subject: [PATCH 60/81] MQE-1711: Switch between Developer mode and Production
 mode takes long time and the test end up time out fixed issue with signature
 of magentoCLI

---
 .../FunctionalTestingFramework/Module/MagentoWebDriver.php    | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/Magento/FunctionalTestingFramework/Module/MagentoWebDriver.php b/src/Magento/FunctionalTestingFramework/Module/MagentoWebDriver.php
index 87e3b0c08..66d103b0c 100644
--- a/src/Magento/FunctionalTestingFramework/Module/MagentoWebDriver.php
+++ b/src/Magento/FunctionalTestingFramework/Module/MagentoWebDriver.php
@@ -838,14 +838,14 @@ public function makeScreenshot($name = null)
      * Takes given $command and executes it against bin/magento executable. Returns stdout output from the command.
      *
      * @param string  $command
-     * @param string  $arguments
      * @param integer $timeout
+     * @param string  $arguments
      *
      * @throws \RuntimeException
      * @return string
      * @SuppressWarnings(PHPMD.UnusedPrivateMethod)
      */
-    private function shellExecMagentoCLI($command, $arguments, $timeout = 60): string
+    private function shellExecMagentoCLI($command, $timeout, $arguments): string
     {
         $php = PHP_BINDIR ? PHP_BINDIR . DIRECTORY_SEPARATOR. 'php' : 'php';
         $binMagento = realpath(MAGENTO_BP . DIRECTORY_SEPARATOR . 'bin' . DIRECTORY_SEPARATOR . 'magento');

From 0f7bd4a3d63a7c80cdd4d80c6d326d0918698144 Mon Sep 17 00:00:00 2001
From: Soumya Unnikrishnan <sunnikri@adobe.com>
Date: Tue, 26 Nov 2019 10:34:47 -0600
Subject: [PATCH 61/81] MQE-1711: Switch between Developer mode and Production
 mode takes long time and the test end up time out Added verification test for
 magentoCLISecret.

---
 dev/tests/verification/Resources/BasicFunctionalTest.txt      | 4 ++++
 .../verification/TestModule/Test/BasicFunctionalTest.xml      | 2 ++
 2 files changed, 6 insertions(+)

diff --git a/dev/tests/verification/Resources/BasicFunctionalTest.txt b/dev/tests/verification/Resources/BasicFunctionalTest.txt
index 2bae80f80..bc7fe0b94 100644
--- a/dev/tests/verification/Resources/BasicFunctionalTest.txt
+++ b/dev/tests/verification/Resources/BasicFunctionalTest.txt
@@ -130,6 +130,10 @@ class BasicFunctionalTestCest
 		$I->comment($magentoCli1);
 		$magentoCli2 = $I->magentoCLI("maintenance:enable", 120, "\"stuffHere\""); // stepKey: magentoCli2
 		$I->comment($magentoCli2);
+		$magentoCli3 = $I->magentoCLISecret("config:set somePath " . CredentialStore::getInstance()->getSecret("someKey"), 60); // stepKey: magentoCli3
+		$I->comment($magentoCli3); // stepKey: magentoCli3
+		$magentoCli4 = $I->magentoCLISecret("config:set somePath " . CredentialStore::getInstance()->getSecret("someKey"), 120); // stepKey: magentoCli4
+		$I->comment($magentoCli4); // stepKey: magentoCli4
 		$I->makeScreenshot("screenShotInput"); // stepKey: makeScreenshotKey1
 		$I->maximizeWindow(); // stepKey: maximizeWindowKey1
 		$I->moveBack(); // stepKey: moveBackKey1
diff --git a/dev/tests/verification/TestModule/Test/BasicFunctionalTest.xml b/dev/tests/verification/TestModule/Test/BasicFunctionalTest.xml
index 09fedd1b8..c23a3ce60 100644
--- a/dev/tests/verification/TestModule/Test/BasicFunctionalTest.xml
+++ b/dev/tests/verification/TestModule/Test/BasicFunctionalTest.xml
@@ -77,6 +77,8 @@
         <grabValueFrom selector=".functionalTestSelector" stepKey="grabValueFromKey1" />
         <magentoCLI command="maintenance:enable" arguments="&quot;stuffHere&quot;" stepKey="magentoCli1"/>
         <magentoCLI command="maintenance:enable" arguments="&quot;stuffHere&quot;" timeout="120" stepKey="magentoCli2"/>
+        <magentoCLI command="config:set somePath {{_CREDS.someKey}}" stepKey="magentoCli3"/>
+        <magentoCLI command="config:set somePath {{_CREDS.someKey}}" timeout="120" stepKey="magentoCli4"/>
         <makeScreenshot userInput="screenShotInput" stepKey="makeScreenshotKey1"/>
         <maximizeWindow stepKey="maximizeWindowKey1"/>
         <moveBack stepKey="moveBackKey1"/>

From f6a7763225ab6df1b331f00b8c33f82459c5dd08 Mon Sep 17 00:00:00 2001
From: Soumya Unnikrishnan <sunnikri@adobe.com>
Date: Tue, 26 Nov 2019 10:42:59 -0600
Subject: [PATCH 62/81] MQE-1711: Switch between Developer mode and Production
 mode takes long time and the test end up time out Resolved conflicts in
 ActionObject.

---
 .../FunctionalTestingFramework/Test/Objects/ActionObject.php     | 1 -
 1 file changed, 1 deletion(-)

diff --git a/src/Magento/FunctionalTestingFramework/Test/Objects/ActionObject.php b/src/Magento/FunctionalTestingFramework/Test/Objects/ActionObject.php
index eb5bfd5ce..ad5ca0c33 100644
--- a/src/Magento/FunctionalTestingFramework/Test/Objects/ActionObject.php
+++ b/src/Magento/FunctionalTestingFramework/Test/Objects/ActionObject.php
@@ -72,7 +72,6 @@ class ActionObject
     const ACTION_ATTRIBUTE_VARIABLE_REGEX_PARAMETER = '/\(.+\)/';
     const ACTION_ATTRIBUTE_VARIABLE_REGEX_PATTERN = '/({{[\w]+\.[\w\[\]]+}})|({{[\w]+\.[\w]+\((?(?!}}).)+\)}})/';
     const STRING_PARAMETER_REGEX = "/'[^']+'/";
-    const DEFAULT_WAIT_TIMEOUT = 10;
     const DEFAULT_COMMAND_WAIT_TIMEOUT = 60;
     const ACTION_ATTRIBUTE_USERINPUT = 'userInput';
     const ACTION_TYPE_COMMENT = 'comment';

From 0cdb29408df31a431409abd12dce85fa6f5e62a0 Mon Sep 17 00:00:00 2001
From: Tom Reece <treece@adobe.com>
Date: Mon, 2 Dec 2019 06:43:48 -0600
Subject: [PATCH 63/81] MQE-1463: Fixes for Suite documentation in devdocs

---
 docs/suite.md | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/docs/suite.md b/docs/suite.md
index e49dd02ab..045f557e1 100644
--- a/docs/suite.md
+++ b/docs/suite.md
@@ -4,9 +4,10 @@ Suites are essentially groups of tests that run in the specific conditions (prec
 They enable you including, excluding, and grouping tests for a customized test run when you need it.
 You can form suites using separate tests, groups, and modules.
 
-Each suite must be defined in the `<magento 2 root>/dev/tests/acceptance/tests/_suite/suite.xml` file.
-The generated tests for each suite go into a separate directory under `<magento 2 root>/dev/tests/acceptance/tests/functional/Magento/FunctionalTest/_generated/`.
-By default, all generated tests are stored in the _default_ suite under `.../Magento/FunctionalTest/_generated/default/`
+Each suite must be defined in the `<VendorName>/<ModuleName>/Test/Mftf/Suite` directory.
+
+The tests for each suite will be generated in a separate directory under `<magento 2 root>/dev/tests/acceptance/tests/functional/Magento/FunctionalTest/_generated/`.
+All tests that aren't in a suite will be generated in the _default_ suite at `.../Magento/FunctionalTest/_generated/default/`
 
 <div class="bs-callout bs-callout-info">
  If a test is generated into at least one custom suite, it will not appear in the _default_ suite.
@@ -59,8 +60,6 @@ The code lives in one place and executes once per suite.
 -  Set up preconditions and postconditions using [actions] in [`<before>`] and [`<after>`] correspondingly, just similar to use in a [test].
 -  Clean up after suites just like after tests.
   The MFTF enforces the presence of both `<before>` and `<after>` if either is present.
--  Do not reference in the subsequent tests to data that was persisted in the preconditions.
-  Referencing to `$stepKey.field$` of these actions is not valid.
 
 ## Test writing
 

From 691b6753f439a516af5d6e17987c7ed89abc885a Mon Sep 17 00:00:00 2001
From: Tomash Khamlai <tomash.khamlai@gmail.com>
Date: Mon, 2 Dec 2019 19:45:10 +0200
Subject: [PATCH 64/81] Add link to action-groups.md

Signed-off-by: Tomash Khamlai <tomash.khamlai@gmail.com>
---
 docs/merging.md | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/docs/merging.md b/docs/merging.md
index 99df18464..e4b91fb5f 100644
--- a/docs/merging.md
+++ b/docs/merging.md
@@ -6,7 +6,7 @@ The MFTF allows you to merge test components defined in XML files, such as:
 -  [`<pages>`][]
 -  [`<sections>`][]
 -  [`<data>`][]
--  `<action groups>`
+-  [`<action groups>`][]
 
 You can create, delete, or update the component.
 It is useful for supporting rapid test creation for extensions and customizations.
@@ -569,3 +569,4 @@ The `_defaultSample` results corresponds to:
 [`<pages>`]: ./page.md
 [`<sections>`]: ./section.md
 [`<tests>`]: ./test.md
+[`<action groups>`]: ./test/action-groups.md

From 1471d80270db517a4bd8871bca60e8cb4840fb3a Mon Sep 17 00:00:00 2001
From: Donald Booth <dobooth@adobe.com>
Date: Mon, 2 Dec 2019 14:45:13 -0800
Subject: [PATCH 65/81] Grammar and formatting

---
 docs/suite.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/suite.md b/docs/suite.md
index 045f557e1..3d737ece5 100644
--- a/docs/suite.md
+++ b/docs/suite.md
@@ -6,8 +6,8 @@ You can form suites using separate tests, groups, and modules.
 
 Each suite must be defined in the `<VendorName>/<ModuleName>/Test/Mftf/Suite` directory.
 
-The tests for each suite will be generated in a separate directory under `<magento 2 root>/dev/tests/acceptance/tests/functional/Magento/FunctionalTest/_generated/`.
-All tests that aren't in a suite will be generated in the _default_ suite at `.../Magento/FunctionalTest/_generated/default/`
+The tests for each suite are generated in a separate directory under `<magento 2 root>/dev/tests/acceptance/tests/functional/Magento/FunctionalTest/_generated/`.
+All tests that are not within a suite are generated in the _default_ suite at `.../Magento/FunctionalTest/_generated/default/`.
 
 <div class="bs-callout bs-callout-info">
  If a test is generated into at least one custom suite, it will not appear in the _default_ suite.

From 3becb62e67c8200074144beffcd054b7f96ee717 Mon Sep 17 00:00:00 2001
From: Soumya Unnikrishnan <sunnikri@adobe.com>
Date: Tue, 3 Dec 2019 08:50:42 -0600
Subject: [PATCH 66/81] MQE-1902: Running bin/magento from MagentoWebDriver
 causing timeout issues on pipeline Updated shellExecMagentoCLI to mimic
 command.php Excluding cron from shell execution to prevent timeouts in
 pipeline.

---
 .../Module/MagentoWebDriver.php               | 90 ++++++++++++++++---
 1 file changed, 76 insertions(+), 14 deletions(-)

diff --git a/src/Magento/FunctionalTestingFramework/Module/MagentoWebDriver.php b/src/Magento/FunctionalTestingFramework/Module/MagentoWebDriver.php
index cbde1ddfc..37c39b0ec 100644
--- a/src/Magento/FunctionalTestingFramework/Module/MagentoWebDriver.php
+++ b/src/Magento/FunctionalTestingFramework/Module/MagentoWebDriver.php
@@ -22,6 +22,7 @@
 use Yandex\Allure\Adapter\AllureException;
 use Magento\FunctionalTestingFramework\Util\Protocol\CurlTransport;
 use Symfony\Component\Process\Process;
+use Symfony\Component\Process\Exception\ProcessTimedOutException;
 use Yandex\Allure\Adapter\Support\AttachmentSupport;
 use Magento\FunctionalTestingFramework\Exceptions\TestFrameworkException;
 use Magento\FunctionalTestingFramework\Config\MftfApplicationConfig;
@@ -52,6 +53,8 @@ class MagentoWebDriver extends WebDriver
 {
     use AttachmentSupport;
 
+    const COMMAND_CRON_RUN = 'cron:run';
+
     /**
      * List of known magento loading masks by selector
      *
@@ -520,13 +523,14 @@ public function scrollToTopOfPage()
      */
     public function magentoCLI($command, $timeout = null, $arguments = null)
     {
-        return $this->curlExecMagentoCLI($command, $timeout, $arguments);
-        //TODO: calling bin/magento from pipeline is timing out, needs investigation (ref: MQE-1774)
-//        try {
-//            return $this->shellExecMagentoCLI($command, $arguments);
-//        } catch (\Exception $exception) {
-//            return $this->curlExecMagentoCLI($command, $arguments);
-//        }
+        $magentoBinary = realpath(MAGENTO_BP . DIRECTORY_SEPARATOR . 'bin' . DIRECTORY_SEPARATOR . 'magento');
+        $valid = $this->validateCommand($magentoBinary, $command);
+        // execute from shell when running tests from web root -- excludes cron jobs.
+        if ($valid && strpos($command, self::COMMAND_CRON_RUN) === false) {
+            return $this->shellExecMagentoCLI($magentoBinary, $command, $timeout, $arguments);
+        } else {
+            return $this->curlExecMagentoCLI($command, $timeout, $arguments);
+        }
     }
 
     /**
@@ -838,6 +842,7 @@ public function makeScreenshot($name = null)
     /**
      * Takes given $command and executes it against bin/magento executable. Returns stdout output from the command.
      *
+     * @param string  $magentoBinary
      * @param string  $command
      * @param integer $timeout
      * @param string  $arguments
@@ -846,20 +851,41 @@ public function makeScreenshot($name = null)
      * @return string
      * @SuppressWarnings(PHPMD.UnusedPrivateMethod)
      */
-    private function shellExecMagentoCLI($command, $timeout, $arguments): string
+    private function shellExecMagentoCLI($magentoBinary, $command, $timeout, $arguments): string
     {
         $php = PHP_BINDIR ? PHP_BINDIR . DIRECTORY_SEPARATOR. 'php' : 'php';
-        $binMagento = realpath(MAGENTO_BP . DIRECTORY_SEPARATOR . 'bin' . DIRECTORY_SEPARATOR . 'magento');
-        $command = $php . ' -f ' . $binMagento . ' ' . $command . ' ' . $arguments;
-        $process = new Process(escapeshellcmd($command), MAGENTO_BP);
+        $fullCommand = $php . ' -f ' . $magentoBinary . ' ' . $command . ' ' . $arguments;
+        $process = new Process(escapeshellcmd($fullCommand), MAGENTO_BP);
         $process->setIdleTimeout($timeout);
         $process->setTimeout(0);
-        $exitCode = $process->run();
+        try {
+            $process->run();
+            $output = $process->getOutput();
+            if (!$process->isSuccessful()) {
+                $failureOutput = $process->getErrorOutput();
+                if (!empty($failureOutput)) {
+                    $output = $failureOutput;
+                }
+            }
+            if (empty($output)) {
+                $output = "CLI did not return output.";
+            }
+
+        } catch (ProcessTimedOutException $exception) {
+            $output = "CLI command timed out, no output available.";
+
+        }
+
+        if ($this->checkForFilePath($output)) {
+            $output = "CLI output suppressed, filepath detected in output.";
+        }
+
+        $exitCode = $process->getExitCode();
+
         if ($exitCode !== 0) {
             throw new \RuntimeException($process->getErrorOutput());
         }
-
-        return $process->getOutput();
+        return $output;
     }
 
     /**
@@ -904,4 +930,40 @@ private function curlExecMagentoCLI($command, $timeout, $arguments): string
 
         return $response;
     }
+
+    /**
+     * Checks magento list of CLI commands for given $command. Does not check command parameters, just base command.
+     * @param string $magentoBinary
+     * @param string $command
+     * @return bool
+     */
+    private function validateCommand($magentoBinary, $command)
+    {
+        exec($magentoBinary . ' list', $commandList);
+        // Trim list of commands after first whitespace
+        $commandList = array_map(array($this, 'trimAfterWhitespace'), $commandList);
+        return in_array($this->trimAfterWhitespace($command), $commandList);
+    }
+
+    /**
+     * Returns given string trimmed of everything after the first found whitespace.
+     * @param string $string
+     * @return string
+     */
+    private function trimAfterWhitespace($string)
+    {
+        return strtok($string, ' ');
+    }
+
+    /**
+     * Detects file path in string.
+     * @param string $string
+     * @return boolean
+     */
+    private function checkForFilePath($string)
+    {
+        return preg_match('/\/[\S]+\//', $string);
+    }
+
 }
+

From 41afe3d9594f26ea1f700015a536ab0a807c4de2 Mon Sep 17 00:00:00 2001
From: Soumya Unnikrishnan <sunnikri@adobe.com>
Date: Tue, 3 Dec 2019 09:17:27 -0600
Subject: [PATCH 67/81] MQE-1902: Running bin/magento from MagentoWebDriver
 causing timeout issues on pipeline fixed phpunit checks

---
 .../Module/MagentoWebDriver.php                 | 17 +++++++++--------
 1 file changed, 9 insertions(+), 8 deletions(-)

diff --git a/src/Magento/FunctionalTestingFramework/Module/MagentoWebDriver.php b/src/Magento/FunctionalTestingFramework/Module/MagentoWebDriver.php
index 37c39b0ec..fa5bf2110 100644
--- a/src/Magento/FunctionalTestingFramework/Module/MagentoWebDriver.php
+++ b/src/Magento/FunctionalTestingFramework/Module/MagentoWebDriver.php
@@ -849,7 +849,6 @@ public function makeScreenshot($name = null)
      *
      * @throws \RuntimeException
      * @return string
-     * @SuppressWarnings(PHPMD.UnusedPrivateMethod)
      */
     private function shellExecMagentoCLI($magentoBinary, $command, $timeout, $arguments): string
     {
@@ -870,10 +869,8 @@ private function shellExecMagentoCLI($magentoBinary, $command, $timeout, $argume
             if (empty($output)) {
                 $output = "CLI did not return output.";
             }
-
         } catch (ProcessTimedOutException $exception) {
             $output = "CLI command timed out, no output available.";
-
         }
 
         if ($this->checkForFilePath($output)) {
@@ -933,21 +930,25 @@ private function curlExecMagentoCLI($command, $timeout, $arguments): string
 
     /**
      * Checks magento list of CLI commands for given $command. Does not check command parameters, just base command.
+     *
      * @param string $magentoBinary
      * @param string $command
-     * @return bool
+     *
+     * @return boolean
      */
     private function validateCommand($magentoBinary, $command)
     {
         exec($magentoBinary . ' list', $commandList);
         // Trim list of commands after first whitespace
-        $commandList = array_map(array($this, 'trimAfterWhitespace'), $commandList);
+        $commandList = array_map([$this, 'trimAfterWhitespace'], $commandList);
         return in_array($this->trimAfterWhitespace($command), $commandList);
     }
 
     /**
      * Returns given string trimmed of everything after the first found whitespace.
+     *
      * @param string $string
+     *
      * @return string
      */
     private function trimAfterWhitespace($string)
@@ -957,13 +958,13 @@ private function trimAfterWhitespace($string)
 
     /**
      * Detects file path in string.
+     *
      * @param string $string
+     *
      * @return boolean
      */
     private function checkForFilePath($string)
     {
         return preg_match('/\/[\S]+\//', $string);
     }
-
-}
-
+}
\ No newline at end of file

From dea0c05b621710d159fa88504ff2f777486a3303 Mon Sep 17 00:00:00 2001
From: Soumya Unnikrishnan <sunnikri@adobe.com>
Date: Tue, 3 Dec 2019 10:42:34 -0600
Subject: [PATCH 68/81] MQE-1902: Running bin/magento from MagentoWebDriver
 causing timeout issues on pipeline Not fetching output from cron.

---
 .../Module/MagentoWebDriver.php               | 21 +++++++++++--------
 1 file changed, 12 insertions(+), 9 deletions(-)

diff --git a/src/Magento/FunctionalTestingFramework/Module/MagentoWebDriver.php b/src/Magento/FunctionalTestingFramework/Module/MagentoWebDriver.php
index fa5bf2110..ae12ad754 100644
--- a/src/Magento/FunctionalTestingFramework/Module/MagentoWebDriver.php
+++ b/src/Magento/FunctionalTestingFramework/Module/MagentoWebDriver.php
@@ -525,8 +525,8 @@ public function magentoCLI($command, $timeout = null, $arguments = null)
     {
         $magentoBinary = realpath(MAGENTO_BP . DIRECTORY_SEPARATOR . 'bin' . DIRECTORY_SEPARATOR . 'magento');
         $valid = $this->validateCommand($magentoBinary, $command);
-        // execute from shell when running tests from web root -- excludes cron jobs.
-        if ($valid && strpos($command, self::COMMAND_CRON_RUN) === false) {
+        // execute from shell when running tests from web root.
+        if ($valid) {
             return $this->shellExecMagentoCLI($magentoBinary, $command, $timeout, $arguments);
         } else {
             return $this->curlExecMagentoCLI($command, $timeout, $arguments);
@@ -854,16 +854,18 @@ private function shellExecMagentoCLI($magentoBinary, $command, $timeout, $argume
     {
         $php = PHP_BINDIR ? PHP_BINDIR . DIRECTORY_SEPARATOR. 'php' : 'php';
         $fullCommand = $php . ' -f ' . $magentoBinary . ' ' . $command . ' ' . $arguments;
-        $process = new Process(escapeshellcmd($fullCommand), MAGENTO_BP);
+        $process = Process::fromShellCommandline(escapeshellcmd($fullCommand), MAGENTO_BP);
         $process->setIdleTimeout($timeout);
         $process->setTimeout(0);
         try {
             $process->run();
-            $output = $process->getOutput();
-            if (!$process->isSuccessful()) {
-                $failureOutput = $process->getErrorOutput();
-                if (!empty($failureOutput)) {
-                    $output = $failureOutput;
+            if (strpos($command, self::COMMAND_CRON_RUN) === false) {
+                $output = $process->getOutput();
+                if (!$process->isSuccessful()) {
+                    $failureOutput = $process->getErrorOutput();
+                    if (!empty($failureOutput)) {
+                        $output = $failureOutput;
+                    }
                 }
             }
             if (empty($output)) {
@@ -967,4 +969,5 @@ private function checkForFilePath($string)
     {
         return preg_match('/\/[\S]+\//', $string);
     }
-}
\ No newline at end of file
+}
+

From 50d9cd27a4edb4b0801abc821469d45f5196d26b Mon Sep 17 00:00:00 2001
From: Soumya Unnikrishnan <sunnikri@adobe.com>
Date: Tue, 3 Dec 2019 14:24:27 -0600
Subject: [PATCH 69/81] MQE-1902: Running bin/magento from MagentoWebDriver
 causing timeout issues on pipeline Reverted changes, build still hangs with
 starting cron process from shell.

---
 .../Module/MagentoWebDriver.php                 | 17 ++++++++---------
 1 file changed, 8 insertions(+), 9 deletions(-)

diff --git a/src/Magento/FunctionalTestingFramework/Module/MagentoWebDriver.php b/src/Magento/FunctionalTestingFramework/Module/MagentoWebDriver.php
index ae12ad754..6bdfbcd12 100644
--- a/src/Magento/FunctionalTestingFramework/Module/MagentoWebDriver.php
+++ b/src/Magento/FunctionalTestingFramework/Module/MagentoWebDriver.php
@@ -525,8 +525,9 @@ public function magentoCLI($command, $timeout = null, $arguments = null)
     {
         $magentoBinary = realpath(MAGENTO_BP . DIRECTORY_SEPARATOR . 'bin' . DIRECTORY_SEPARATOR . 'magento');
         $valid = $this->validateCommand($magentoBinary, $command);
-        // execute from shell when running tests from web root.
-        if ($valid) {
+        // execute from shell when running tests from web root -- excluding cron
+        //TODO: investigate why cron:run hangs with shell execution on pipeline
+        if ($valid && ($command !== self::COMMAND_CRON_RUN)) {
             return $this->shellExecMagentoCLI($magentoBinary, $command, $timeout, $arguments);
         } else {
             return $this->curlExecMagentoCLI($command, $timeout, $arguments);
@@ -859,13 +860,11 @@ private function shellExecMagentoCLI($magentoBinary, $command, $timeout, $argume
         $process->setTimeout(0);
         try {
             $process->run();
-            if (strpos($command, self::COMMAND_CRON_RUN) === false) {
-                $output = $process->getOutput();
-                if (!$process->isSuccessful()) {
-                    $failureOutput = $process->getErrorOutput();
-                    if (!empty($failureOutput)) {
-                        $output = $failureOutput;
-                    }
+            $output = $process->getOutput();
+            if (!$process->isSuccessful()) {
+                $failureOutput = $process->getErrorOutput();
+                if (!empty($failureOutput)) {
+                    $output = $failureOutput;
                 }
             }
             if (empty($output)) {

From 5946652fcd956441c9fabf36f948f657e12013a7 Mon Sep 17 00:00:00 2001
From: Soumya Unnikrishnan <sunnikri@adobe.com>
Date: Tue, 3 Dec 2019 14:27:23 -0600
Subject: [PATCH 70/81] MQE-1902: Running bin/magento from MagentoWebDriver
 causing timeout issues on pipeline Reverted changes, build still hangs with
 starting cron process from shell.

---
 .../FunctionalTestingFramework/Module/MagentoWebDriver.php      | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/Magento/FunctionalTestingFramework/Module/MagentoWebDriver.php b/src/Magento/FunctionalTestingFramework/Module/MagentoWebDriver.php
index 6bdfbcd12..43437d116 100644
--- a/src/Magento/FunctionalTestingFramework/Module/MagentoWebDriver.php
+++ b/src/Magento/FunctionalTestingFramework/Module/MagentoWebDriver.php
@@ -527,7 +527,7 @@ public function magentoCLI($command, $timeout = null, $arguments = null)
         $valid = $this->validateCommand($magentoBinary, $command);
         // execute from shell when running tests from web root -- excluding cron
         //TODO: investigate why cron:run hangs with shell execution on pipeline
-        if ($valid && ($command !== self::COMMAND_CRON_RUN)) {
+        if ($valid && strpos($command, self::COMMAND_CRON_RUN) === false) {
             return $this->shellExecMagentoCLI($magentoBinary, $command, $timeout, $arguments);
         } else {
             return $this->curlExecMagentoCLI($command, $timeout, $arguments);

From 84b9f87123d00bd3b36cc2bad4358666ca48c756 Mon Sep 17 00:00:00 2001
From: Soumya Unnikrishnan <sunnikri@adobe.com>
Date: Tue, 3 Dec 2019 15:03:40 -0600
Subject: [PATCH 71/81] MQE-1902: Running bin/magento from MagentoWebDriver
 causing timeout issues on pipeline fixed phpunit checks

---
 .../FunctionalTestingFramework/Module/MagentoWebDriver.php       | 1 -
 1 file changed, 1 deletion(-)

diff --git a/src/Magento/FunctionalTestingFramework/Module/MagentoWebDriver.php b/src/Magento/FunctionalTestingFramework/Module/MagentoWebDriver.php
index 43437d116..710ea993e 100644
--- a/src/Magento/FunctionalTestingFramework/Module/MagentoWebDriver.php
+++ b/src/Magento/FunctionalTestingFramework/Module/MagentoWebDriver.php
@@ -969,4 +969,3 @@ private function checkForFilePath($string)
         return preg_match('/\/[\S]+\//', $string);
     }
 }
-

From e207c475faae37be3c1be9066572c5d25f135c16 Mon Sep 17 00:00:00 2001
From: Soumya Unnikrishnan <sunnikri@adobe.com>
Date: Wed, 4 Dec 2019 11:28:53 -0600
Subject: [PATCH 72/81] MQE-1902: Running bin/magento from MagentoWebDriver
 causing timeout issues on pipeline reverting changes of community PR#343

---
 .../Module/MagentoWebDriver.php               | 167 +++---------------
 1 file changed, 29 insertions(+), 138 deletions(-)

diff --git a/src/Magento/FunctionalTestingFramework/Module/MagentoWebDriver.php b/src/Magento/FunctionalTestingFramework/Module/MagentoWebDriver.php
index 710ea993e..dacfbb2d4 100644
--- a/src/Magento/FunctionalTestingFramework/Module/MagentoWebDriver.php
+++ b/src/Magento/FunctionalTestingFramework/Module/MagentoWebDriver.php
@@ -523,15 +523,35 @@ public function scrollToTopOfPage()
      */
     public function magentoCLI($command, $timeout = null, $arguments = null)
     {
-        $magentoBinary = realpath(MAGENTO_BP . DIRECTORY_SEPARATOR . 'bin' . DIRECTORY_SEPARATOR . 'magento');
-        $valid = $this->validateCommand($magentoBinary, $command);
-        // execute from shell when running tests from web root -- excluding cron
-        //TODO: investigate why cron:run hangs with shell execution on pipeline
-        if ($valid && strpos($command, self::COMMAND_CRON_RUN) === false) {
-            return $this->shellExecMagentoCLI($magentoBinary, $command, $timeout, $arguments);
-        } else {
-            return $this->curlExecMagentoCLI($command, $timeout, $arguments);
-        }
+        // Remove index.php if it's present in url
+        $baseUrl = rtrim(
+            str_replace('index.php', '', rtrim($this->config['url'], '/')),
+            '/'
+        );
+
+        $apiURL = UrlFormatter::format(
+            $baseUrl . '/' . ltrim(getenv('MAGENTO_CLI_COMMAND_PATH'), '/'),
+            false
+        );
+
+        $restExecutor = new WebapiExecutor();
+        $executor = new CurlTransport();
+        $executor->write(
+            $apiURL,
+            [
+                'token' => $restExecutor->getAuthToken(),
+                getenv('MAGENTO_CLI_COMMAND_PARAMETER') => $command,
+                'arguments' => $arguments,
+                'timeout'   => $timeout,
+            ],
+            CurlInterface::POST,
+            []
+        );
+        $response = $executor->read();
+        $restExecutor->close();
+        $executor->close();
+
+        return $response;
     }
 
     /**
@@ -839,133 +859,4 @@ public function makeScreenshot($name = null)
         $this->debug("Screenshot saved to $screenName");
         AllureHelper::addAttachmentToCurrentStep($screenName, 'Screenshot');
     }
-
-    /**
-     * Takes given $command and executes it against bin/magento executable. Returns stdout output from the command.
-     *
-     * @param string  $magentoBinary
-     * @param string  $command
-     * @param integer $timeout
-     * @param string  $arguments
-     *
-     * @throws \RuntimeException
-     * @return string
-     */
-    private function shellExecMagentoCLI($magentoBinary, $command, $timeout, $arguments): string
-    {
-        $php = PHP_BINDIR ? PHP_BINDIR . DIRECTORY_SEPARATOR. 'php' : 'php';
-        $fullCommand = $php . ' -f ' . $magentoBinary . ' ' . $command . ' ' . $arguments;
-        $process = Process::fromShellCommandline(escapeshellcmd($fullCommand), MAGENTO_BP);
-        $process->setIdleTimeout($timeout);
-        $process->setTimeout(0);
-        try {
-            $process->run();
-            $output = $process->getOutput();
-            if (!$process->isSuccessful()) {
-                $failureOutput = $process->getErrorOutput();
-                if (!empty($failureOutput)) {
-                    $output = $failureOutput;
-                }
-            }
-            if (empty($output)) {
-                $output = "CLI did not return output.";
-            }
-        } catch (ProcessTimedOutException $exception) {
-            $output = "CLI command timed out, no output available.";
-        }
-
-        if ($this->checkForFilePath($output)) {
-            $output = "CLI output suppressed, filepath detected in output.";
-        }
-
-        $exitCode = $process->getExitCode();
-
-        if ($exitCode !== 0) {
-            throw new \RuntimeException($process->getErrorOutput());
-        }
-        return $output;
-    }
-
-    /**
-     * Takes given $command and executes it against exposed MTF CLI entry point. Returns response from server.
-     *
-     * @param string  $command
-     * @param integer $timeout
-     * @param string  $arguments
-     *
-     * @return string
-     * @throws TestFrameworkException
-     */
-    private function curlExecMagentoCLI($command, $timeout, $arguments): string
-    {
-        // Remove index.php if it's present in url
-        $baseUrl = rtrim(
-            str_replace('index.php', '', rtrim($this->config['url'], '/')),
-            '/'
-        );
-
-        $apiURL = UrlFormatter::format(
-            $baseUrl . '/' . ltrim(getenv('MAGENTO_CLI_COMMAND_PATH'), '/'),
-            false
-        );
-
-        $restExecutor = new WebapiExecutor();
-        $executor = new CurlTransport();
-        $executor->write(
-            $apiURL,
-            [
-                'token' => $restExecutor->getAuthToken(),
-                getenv('MAGENTO_CLI_COMMAND_PARAMETER') => $command,
-                'arguments' => $arguments,
-                'timeout'   => $timeout,
-            ],
-            CurlInterface::POST,
-            []
-        );
-        $response = $executor->read();
-        $restExecutor->close();
-        $executor->close();
-
-        return $response;
-    }
-
-    /**
-     * Checks magento list of CLI commands for given $command. Does not check command parameters, just base command.
-     *
-     * @param string $magentoBinary
-     * @param string $command
-     *
-     * @return boolean
-     */
-    private function validateCommand($magentoBinary, $command)
-    {
-        exec($magentoBinary . ' list', $commandList);
-        // Trim list of commands after first whitespace
-        $commandList = array_map([$this, 'trimAfterWhitespace'], $commandList);
-        return in_array($this->trimAfterWhitespace($command), $commandList);
-    }
-
-    /**
-     * Returns given string trimmed of everything after the first found whitespace.
-     *
-     * @param string $string
-     *
-     * @return string
-     */
-    private function trimAfterWhitespace($string)
-    {
-        return strtok($string, ' ');
-    }
-
-    /**
-     * Detects file path in string.
-     *
-     * @param string $string
-     *
-     * @return boolean
-     */
-    private function checkForFilePath($string)
-    {
-        return preg_match('/\/[\S]+\//', $string);
-    }
 }

From 712cc518039c2f87ee93e163d3e638019147ab63 Mon Sep 17 00:00:00 2001
From: Soumya Unnikrishnan <sunnikri@adobe.com>
Date: Wed, 4 Dec 2019 11:30:51 -0600
Subject: [PATCH 73/81] MQE-1902: Running bin/magento from MagentoWebDriver
 causing timeout issues on pipeline reverting changes of community PR#343

---
 .../FunctionalTestingFramework/Module/MagentoWebDriver.php      | 2 --
 1 file changed, 2 deletions(-)

diff --git a/src/Magento/FunctionalTestingFramework/Module/MagentoWebDriver.php b/src/Magento/FunctionalTestingFramework/Module/MagentoWebDriver.php
index dacfbb2d4..5c86e0346 100644
--- a/src/Magento/FunctionalTestingFramework/Module/MagentoWebDriver.php
+++ b/src/Magento/FunctionalTestingFramework/Module/MagentoWebDriver.php
@@ -53,8 +53,6 @@ class MagentoWebDriver extends WebDriver
 {
     use AttachmentSupport;
 
-    const COMMAND_CRON_RUN = 'cron:run';
-
     /**
      * List of known magento loading masks by selector
      *

From fa8e51bab2821daddc4408a5c7516cd3174e5b7a Mon Sep 17 00:00:00 2001
From: Soumya Unnikrishnan <sunnikri@adobe.com>
Date: Wed, 4 Dec 2019 13:06:18 -0600
Subject: [PATCH 74/81] MQE-1902: Running bin/magento from MagentoWebDriver
 causing timeout issues on pipeline removing unused references

---
 .../FunctionalTestingFramework/Module/MagentoWebDriver.php      | 2 --
 1 file changed, 2 deletions(-)

diff --git a/src/Magento/FunctionalTestingFramework/Module/MagentoWebDriver.php b/src/Magento/FunctionalTestingFramework/Module/MagentoWebDriver.php
index 5c86e0346..ff0c621dd 100644
--- a/src/Magento/FunctionalTestingFramework/Module/MagentoWebDriver.php
+++ b/src/Magento/FunctionalTestingFramework/Module/MagentoWebDriver.php
@@ -21,8 +21,6 @@
 use Magento\FunctionalTestingFramework\Util\ConfigSanitizerUtil;
 use Yandex\Allure\Adapter\AllureException;
 use Magento\FunctionalTestingFramework\Util\Protocol\CurlTransport;
-use Symfony\Component\Process\Process;
-use Symfony\Component\Process\Exception\ProcessTimedOutException;
 use Yandex\Allure\Adapter\Support\AttachmentSupport;
 use Magento\FunctionalTestingFramework\Exceptions\TestFrameworkException;
 use Magento\FunctionalTestingFramework\Config\MftfApplicationConfig;

From a51f3bfcebfcd51fc90aa1ef2efacf8960f5e826 Mon Sep 17 00:00:00 2001
From: Kevin Kozan <kkozan@adobe.com>
Date: Thu, 5 Dec 2019 10:08:41 -0600
Subject: [PATCH 75/81] MQE-1893: CHANGELOG.MD and Composer version bump

- CHANGELOG update
- Composer file and lock update
---
 CHANGELOG.md  | 25 +++++++++++++++++++++++++
 bin/mftf      |  4 +++-
 composer.json |  2 +-
 composer.lock |  2 +-
 4 files changed, 30 insertions(+), 3 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index fd1e02008..e9b935493 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,5 +1,30 @@
 Magento Functional Testing Framework Changelog
 ================================================
+2.5.4
+-----
+
+* Traceability
+    * Introduced new `mftf doctor` command
+        * Command verifies and troubleshoots some configuration steps required for running tests
+        * Please see DevDocs for more details
+    * `<*Data>` actions now contain `API Endpoint` and `Request Header` artifacts.
+    * Introduced new `.env` configurations `ENABLE_BROWSER_LOG` and `BROWSER_LOG_BLACKLIST`
+        * Configuration enables allure artifacts for browser log entries if they are present after the step.
+        * Blacklist filters out logs from specific sources.
+* Customizability
+    * Introduced `timeout=""` to `magentoCLI` actions.
+
+### GitHub Issues/Pull requests:
+* [#317](https://github.com/magento/magento2-functional-testing-framework/pull/317) -- RetrieveEntityField generation does not consider ActionGroup as part of namespace
+* [#433](https://github.com/magento/magento2-functional-testing-framework/pull/433) -- Add possibility to include multiple non primitive types in an array
+
+### Fixes
+* A test now contains attachments for every exception encountered in the test (fix for a test `<after>` exception overriding all test exceptions).
+* Fixed hard requirement for `MAGENTO_BASE_URL` to contain a leading `/`.
+* `magentoCLI` actions for `config:sensitive:set` no longer obscure CLI output.
+* `WAIT_TIMEOUT` in the `.env` now correctly sets `pageload_timeout` configuration.
+* Fixed an issue where `run:group` could not consolidate a `group` that had tests in and out of `<suite>`s.
+
 2.5.3
 -----
 
diff --git a/bin/mftf b/bin/mftf
index 9e5879280..7a9ca1cf2 100755
--- a/bin/mftf
+++ b/bin/mftf
@@ -27,9 +27,11 @@ try {
 
 
 try {
+    $version = json_decode(file_get_contents(FW_BP . DIRECTORY_SEPARATOR . 'composer.json'), true);
+    $version = $version['version'];
     $application = new Symfony\Component\Console\Application();
     $application->setName('Magento Functional Testing Framework CLI');
-    $application->setVersion('2.5.3');
+    $application->setVersion($version);
     /** @var \Magento\FunctionalTestingFramework\Console\CommandListInterface $commandList */
     $commandList = new \Magento\FunctionalTestingFramework\Console\CommandList;
     foreach ($commandList->getCommands() as $command) {
diff --git a/composer.json b/composer.json
index e3e890fa1..2702f6b77 100755
--- a/composer.json
+++ b/composer.json
@@ -2,7 +2,7 @@
     "name": "magento/magento2-functional-testing-framework",
     "description": "Magento2 Functional Testing Framework",
     "type": "library",
-    "version": "2.5.3",
+    "version": "2.5.4",
     "license": "AGPL-3.0",
     "keywords": ["magento", "automation", "functional", "testing"],
     "config": {
diff --git a/composer.lock b/composer.lock
index 4bf9950d9..2ba66b1c9 100644
--- a/composer.lock
+++ b/composer.lock
@@ -4,7 +4,7 @@
         "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies",
         "This file is @generated automatically"
     ],
-    "content-hash": "7d881849489a99d20aa632226a061fbd",
+    "content-hash": "ee4db031a218f7a29853b2748fe493fe",
     "packages": [
         {
             "name": "allure-framework/allure-codeception",

From 6ab962ce0b4ca7472a9d3c8e8d94e70ebb0b603e Mon Sep 17 00:00:00 2001
From: Kevin Kozan <kkozan@adobe.com>
Date: Thu, 5 Dec 2019 10:13:13 -0600
Subject: [PATCH 76/81] MQE-1650: Update MFTF configuration to read Test
 entities from new location

- Version change of composer/composer for 2.2 compat
---
 composer.json |  2 +-
 composer.lock | 12 ++++++------
 2 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/composer.json b/composer.json
index e3e890fa1..281dad854 100755
--- a/composer.json
+++ b/composer.json
@@ -13,7 +13,7 @@
         "ext-curl": "*",
         "allure-framework/allure-codeception": "~1.3.0",
         "codeception/codeception": "~2.4.5",
-        "composer/composer": "^1.6",
+        "composer/composer": "^1.4",
         "consolidation/robo": "^1.0.0",
         "csharpru/vault-php": "~3.5.3",
         "csharpru/vault-php-guzzle6-transport": "^2.0",
diff --git a/composer.lock b/composer.lock
index 4bf9950d9..007349dcb 100644
--- a/composer.lock
+++ b/composer.lock
@@ -4,7 +4,7 @@
         "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies",
         "This file is @generated automatically"
     ],
-    "content-hash": "7d881849489a99d20aa632226a061fbd",
+    "content-hash": "6d6e03ab3a48beba35d47504ddf95f0a",
     "packages": [
         {
             "name": "allure-framework/allure-codeception",
@@ -486,16 +486,16 @@
         },
         {
             "name": "composer/composer",
-            "version": "1.9.0",
+            "version": "1.9.1",
             "source": {
                 "type": "git",
                 "url": "https://github.com/composer/composer.git",
-                "reference": "314aa57fdcfc942065996f59fb73a8b3f74f3fa5"
+                "reference": "bb01f2180df87ce7992b8331a68904f80439dd2f"
             },
             "dist": {
                 "type": "zip",
-                "url": "https://api.github.com/repos/composer/composer/zipball/314aa57fdcfc942065996f59fb73a8b3f74f3fa5",
-                "reference": "314aa57fdcfc942065996f59fb73a8b3f74f3fa5",
+                "url": "https://api.github.com/repos/composer/composer/zipball/bb01f2180df87ce7992b8331a68904f80439dd2f",
+                "reference": "bb01f2180df87ce7992b8331a68904f80439dd2f",
                 "shasum": ""
             },
             "require": {
@@ -562,7 +562,7 @@
                 "dependency",
                 "package"
             ],
-            "time": "2019-08-02T18:55:33+00:00"
+            "time": "2019-11-01T16:20:17+00:00"
         },
         {
             "name": "composer/semver",

From ca2e420a87770d047fb8b60a31d26ffd0504e57a Mon Sep 17 00:00:00 2001
From: Kevin Kozan <kkozan@adobe.com>
Date: Fri, 6 Dec 2019 10:55:38 -0600
Subject: [PATCH 77/81] MQE-1893: CHANGELOG.MD and Composer version bump

- Reupdate lock file
---
 composer.lock | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/composer.lock b/composer.lock
index 007349dcb..8f2fcb8e9 100644
--- a/composer.lock
+++ b/composer.lock
@@ -4,7 +4,7 @@
         "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies",
         "This file is @generated automatically"
     ],
-    "content-hash": "6d6e03ab3a48beba35d47504ddf95f0a",
+    "content-hash": "59e95cc1ae6311e93111bd7ced180d29",
     "packages": [
         {
             "name": "allure-framework/allure-codeception",

From 61e6c2713697b7ba2bcc1c3ecc6ca772535862e9 Mon Sep 17 00:00:00 2001
From: Ji Lu <jilu1@adobe.com>
Date: Fri, 6 Dec 2019 12:32:16 -0600
Subject: [PATCH 78/81] MQE-1908: Change doctor command to work with Curl
 version of MagentoCLI

---
 .../Console/DoctorCommand.php                     | 13 +++++++++++--
 .../Module/MagentoWebDriverDoctor.php             | 15 ++++++++-------
 2 files changed, 19 insertions(+), 9 deletions(-)

diff --git a/src/Magento/FunctionalTestingFramework/Console/DoctorCommand.php b/src/Magento/FunctionalTestingFramework/Console/DoctorCommand.php
index 4bd05b836..0b144aabc 100644
--- a/src/Magento/FunctionalTestingFramework/Console/DoctorCommand.php
+++ b/src/Magento/FunctionalTestingFramework/Console/DoctorCommand.php
@@ -8,6 +8,7 @@
 namespace Magento\FunctionalTestingFramework\Console;
 
 use Codeception\Configuration;
+use Magento\FunctionalTestingFramework\Util\Path\UrlFormatter;
 use Symfony\Component\EventDispatcher\EventDispatcher;
 use Codeception\SuiteManager;
 use Magento\FunctionalTestingFramework\Config\MftfApplicationConfig;
@@ -128,9 +129,17 @@ private function checkAuthenticationToMagentoAdmin()
             $this->ioStyle->success('Successful');
             $result = true;
         } catch (TestFrameworkException $e) {
+            if (getenv('MAGENTO_BACKEND_BASE_URL')) {
+                $urlVar = 'MAGENTO_BACKEND_BASE_URL';
+            } else {
+                $urlVar = 'MAGENTO_BASE_URL';
+            }
             $this->ioStyle->error(
-                $e->getMessage()
-                . "\nPlease verify MAGENTO_ADMIN_USERNAME and MAGENTO_ADMIN_PASSWORD in .env."
+                $e->getMessage() . "\n"
+                . "Please verify the following configuration settings in .env:\n"
+                . $urlVar . ' = ' . getenv($urlVar) . "\n"
+                . "MAGENTO_ADMIN_USERNAME = " . $e->getContext()['MAGENTO_ADMIN_USERNAME'] . "\n"
+                . "MAGENTO_ADMIN_PASSWORD = " . $e->getContext()['MAGENTO_ADMIN_PASSWORD']
             );
         }
         return $result;
diff --git a/src/Magento/FunctionalTestingFramework/Module/MagentoWebDriverDoctor.php b/src/Magento/FunctionalTestingFramework/Module/MagentoWebDriverDoctor.php
index 98eb8bd4f..1407c957b 100644
--- a/src/Magento/FunctionalTestingFramework/Module/MagentoWebDriverDoctor.php
+++ b/src/Magento/FunctionalTestingFramework/Module/MagentoWebDriverDoctor.php
@@ -15,7 +15,7 @@
  */
 class MagentoWebDriverDoctor extends MagentoWebDriver
 {
-    const MAGENTO_CLI_COMMAND = 'list';
+    const MAGENTO_CLI_COMMAND = 'info:currency:list';
     const EXCEPTION_CONTEXT_SELENIUM = 'selenium';
     const EXCEPTION_CONTEXT_ADMIN = 'admin';
     const EXCEPTION_CONTEXT_STOREFRONT = 'store';
@@ -150,18 +150,19 @@ private function loadPageAtUrl($url)
     private function runMagentoCLI()
     {
         try {
-            $regex = '~^.*(?<name>Magento CLI).*[\r\n]+(?<usage>Usage:).*~';
+            $regex = '~^.*[\r\n]+.*(?<name>Currency).*(?<code>Code).*~';
             $output = parent::magentoCLI(self::MAGENTO_CLI_COMMAND);
             preg_match($regex, $output, $matches);
 
-            if (isset($matches['name']) && isset($matches['usage'])) {
+            if (isset($matches['name']) && isset($matches['code'])) {
                 return;
             }
         } catch (\Exception $e) {
-            throw new TestFrameworkException(
-                "Failed to run Magento CLI command\n"
-                . "Please reference Magento DevDoc to setup command.php and .htaccess files."
-            );
         }
+
+        throw new TestFrameworkException(
+            "Failed to run Magento CLI command\n"
+            . "Please reference Magento DevDoc to setup command.php and .htaccess files."
+        );
     }
 }

From b4dbaa4e4683d5dbb833c935c2faed39a710f729 Mon Sep 17 00:00:00 2001
From: Ji Lu <jilu1@adobe.com>
Date: Fri, 6 Dec 2019 13:19:02 -0600
Subject: [PATCH 79/81] MQE-1905: Inconsistent environment variable naming for
 MAGENTO_BACKEND_BASE_HOST

---
 etc/config/.env.example | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/etc/config/.env.example b/etc/config/.env.example
index e290c7815..7320d8b8b 100644
--- a/etc/config/.env.example
+++ b/etc/config/.env.example
@@ -5,7 +5,7 @@
 MAGENTO_BASE_URL=http://devdocs.magento.com/
 
 #*** Uncomment if you are running Admin Panel on separate domain (used with MAGENTO_BACKEND_NAME) ***#
-# MAGENTO_BACKEND_BASE_HOST=http://admin.example.com/
+# MAGENTO_BACKEND_BASE_URL=http://admin.example.com/
 
 #*** Set the Admin Username and Password for your Magento instance ***#
 MAGENTO_BACKEND_NAME=admin

From a243dbdfb2cdfad597f3be78c9b872edc30bd195 Mon Sep 17 00:00:00 2001
From: Ji Lu <jilu1@adobe.com>
Date: Fri, 6 Dec 2019 15:35:08 -0600
Subject: [PATCH 80/81] MQE-1908: Change doctor command to work with Curl
 version of MagentoCLI

---
 .../FunctionalTestingFramework/Console/DoctorCommand.php   | 7 ++-----
 1 file changed, 2 insertions(+), 5 deletions(-)

diff --git a/src/Magento/FunctionalTestingFramework/Console/DoctorCommand.php b/src/Magento/FunctionalTestingFramework/Console/DoctorCommand.php
index 0b144aabc..6fc9afa55 100644
--- a/src/Magento/FunctionalTestingFramework/Console/DoctorCommand.php
+++ b/src/Magento/FunctionalTestingFramework/Console/DoctorCommand.php
@@ -135,11 +135,8 @@ private function checkAuthenticationToMagentoAdmin()
                 $urlVar = 'MAGENTO_BASE_URL';
             }
             $this->ioStyle->error(
-                $e->getMessage() . "\n"
-                . "Please verify the following configuration settings in .env:\n"
-                . $urlVar . ' = ' . getenv($urlVar) . "\n"
-                . "MAGENTO_ADMIN_USERNAME = " . $e->getContext()['MAGENTO_ADMIN_USERNAME'] . "\n"
-                . "MAGENTO_ADMIN_PASSWORD = " . $e->getContext()['MAGENTO_ADMIN_PASSWORD']
+                $e->getMessage() . "\nPlease verify if " . $urlVar . ", "
+                . "MAGENTO_ADMIN_USERNAME and MAGENTO_ADMIN_PASSWORD in .env are valid."
             );
         }
         return $result;

From 9cd8891635460fa7003dd2d0769a16bb7999cafb Mon Sep 17 00:00:00 2001
From: Soumya Unnikrishnan <sunnikri@adobe.com>
Date: Mon, 9 Dec 2019 11:45:38 -0600
Subject: [PATCH 81/81] MQE-1917: ENABLE_BROWSER_LOG = false attaches JS logs
 to allure

---
 .../Extension/TestContextExtension.php                          | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/Magento/FunctionalTestingFramework/Extension/TestContextExtension.php b/src/Magento/FunctionalTestingFramework/Extension/TestContextExtension.php
index 07153040c..c50156f08 100644
--- a/src/Magento/FunctionalTestingFramework/Extension/TestContextExtension.php
+++ b/src/Magento/FunctionalTestingFramework/Extension/TestContextExtension.php
@@ -182,7 +182,7 @@ public function beforeStep(\Codeception\Event\StepEvent $e)
     public function afterStep(\Codeception\Event\StepEvent $e)
     {
         $browserLog = $this->getDriver()->webDriver->manage()->getLog("browser");
-        if (getenv('ENABLE_BROWSER_LOG')) {
+        if (getenv('ENABLE_BROWSER_LOG') === 'true') {
             foreach (explode(',', getenv('BROWSER_LOG_BLACKLIST')) as $source) {
                 $browserLog = BrowserLogUtil::filterLogsOfType($browserLog, $source);
             }