Merge pull request #528 from magento/MFTF2.5.4-RC

MFTF 2.5.4

Author: KevinBKozan

CHANGELOG.md

@@ -1,5 +1,30 @@
 Magento Functional Testing Framework Changelog
 ================================================
+2.5.4
+-----
+
+* Traceability
+    * Introduced new `mftf doctor` command
+        * Command verifies and troubleshoots some configuration steps required for running tests
+        * Please see DevDocs for more details
+    * `<*Data>` actions now contain `API Endpoint` and `Request Header` artifacts.
+    * Introduced new `.env` configurations `ENABLE_BROWSER_LOG` and `BROWSER_LOG_BLACKLIST`
+        * Configuration enables allure artifacts for browser log entries if they are present after the step.
+        * Blacklist filters out logs from specific sources.
+* Customizability
+    * Introduced `timeout=""` to `magentoCLI` actions.
+
+### GitHub Issues/Pull requests:
+* [#317](https://github.com/magento/magento2-functional-testing-framework/pull/317) -- RetrieveEntityField generation does not consider ActionGroup as part of namespace
+* [#433](https://github.com/magento/magento2-functional-testing-framework/pull/433) -- Add possibility to include multiple non primitive types in an array
+
+### Fixes
+* A test now contains attachments for every exception encountered in the test (fix for a test `<after>` exception overriding all test exceptions).
+* Fixed hard requirement for `MAGENTO_BASE_URL` to contain a leading `/`.
+* `magentoCLI` actions for `config:sensitive:set` no longer obscure CLI output.
+* `WAIT_TIMEOUT` in the `.env` now correctly sets `pageload_timeout` configuration.
+* Fixed an issue where `run:group` could not consolidate a `group` that had tests in and out of `<suite>`s.
+
 2.5.3
 -----
 

bin/mftf

@@ -27,9 +27,11 @@ try {
 
 
 try {
+    $version = json_decode(file_get_contents(FW_BP . DIRECTORY_SEPARATOR . 'composer.json'), true);
+    $version = $version['version'];
     $application = new Symfony\Component\Console\Application();
     $application->setName('Magento Functional Testing Framework CLI');
-    $application->setVersion('2.5.3');
+    $application->setVersion($version);
     /** @var \Magento\FunctionalTestingFramework\Console\CommandListInterface $commandList */
     $commandList = new \Magento\FunctionalTestingFramework\Console\CommandList;
     foreach ($commandList->getCommands() as $command) {

composer.json

@@ -2,7 +2,7 @@
     "name": "magento/magento2-functional-testing-framework",
     "description": "Magento2 Functional Testing Framework",
     "type": "library",
-    "version": "2.5.3",
+    "version": "2.5.4",
     "license": "AGPL-3.0",
     "keywords": ["magento", "automation", "functional", "testing"],
     "config": {
@@ -12,7 +12,7 @@
         "php": "7.0.2||7.0.4||~7.0.6||~7.1.0||~7.2.0||~7.3.0",
         "ext-curl": "*",
         "allure-framework/allure-codeception": "~1.3.0",
-        "codeception/codeception": "~2.3.4 || ~2.4.0 ",
+        "codeception/codeception": "~2.4.5",
         "composer/composer": "^1.4",
         "consolidation/robo": "^1.0.0",
         "csharpru/vault-php": "~3.5.3",

composer.lock

@@ -45,7 +45,8 @@
     'MAGENTO_BACKEND_NAME' => 'admin',
     'MAGENTO_ADMIN_USERNAME' => 'admin',
     'MAGENTO_ADMIN_PASSWORD' => 'admin123',
-    'DEFAULT_TIMEZONE' => 'America/Los_Angeles'
+    'DEFAULT_TIMEZONE' => 'America/Los_Angeles',
+    'WAIT_TIMEOUT' => '10'
 ];
 
 foreach ($TEST_ENVS as $key => $value) {

dev/tests/_bootstrap.php

@@ -15,10 +15,12 @@
 
 require_once realpath(PROJECT_ROOT . '/vendor/autoload.php');
 
+$envFilePath = dirname(dirname(__DIR__)) . DIRECTORY_SEPARATOR;
+defined('ENV_FILE_PATH') || define('ENV_FILE_PATH', $envFilePath);
+
 //Load constants from .env file
-$envFilePath = dirname(dirname(__DIR__));
-if (file_exists($envFilePath . DIRECTORY_SEPARATOR . '.env')) {
-    $env = new \Dotenv\Loader($envFilePath . DIRECTORY_SEPARATOR . '.env');
+if (file_exists(ENV_FILE_PATH . '.env')) {
+    $env = new \Dotenv\Loader(ENV_FILE_PATH . '.env');
     $env->load();
 
     foreach ($_ENV as $key => $var) {
@@ -47,7 +49,10 @@
 
     defined('DEFAULT_TIMEZONE') || define('DEFAULT_TIMEZONE', 'America/Los_Angeles');
     $env->setEnvironmentVariable('DEFAULT_TIMEZONE', DEFAULT_TIMEZONE);
-    
+
+    defined('WAIT_TIMEOUT') || define('WAIT_TIMEOUT', 30);
+    $env->setEnvironmentVariable('WAIT_TIMEOUT', 30);
+
     try {
         new DateTimeZone(DEFAULT_TIMEZONE);
     } catch (\Exception $e) {

dev/tests/functional/standalone_bootstrap.php

@@ -6,6 +6,7 @@
 namespace Tests\unit\Magento\FunctionalTestingFramework\Allure;
 
 use Magento\FunctionalTestingFramework\Allure\AllureHelper;
+use Magento\FunctionalTestingFramework\Allure\Event\AddUniqueAttachmentEvent;
 use Yandex\Allure\Adapter\Allure;
 use Yandex\Allure\Adapter\Event\AddAttachmentEvent;
 use Yandex\Allure\Adapter\Event\StepFinishedEvent;
@@ -24,6 +25,7 @@ class AllureHelperTest extends TestCase
     public function tearDown()
     {
         Allure::setDefaultLifecycle();
+        AspectMock::clean();
     }
 
     /**
@@ -85,13 +87,48 @@ public function testAddAttachmentToLastStep()
     }
 
     /**
-     * Mock file system manipulation function
+     * AddAttachment actions should have files with different attachment names
+     * @throws \Yandex\Allure\Adapter\AllureException
+     */
+    public function testAddAttachementUniqueName()
+    {
+        $this->mockCopyFile();
+        $expectedData = "string";
+        $expectedCaption = "caption";
+
+        //Prepare Allure lifecycle
+        Allure::lifecycle()->fire(new StepStartedEvent('firstStep'));
+
+        //Call function twice
+        AllureHelper::addAttachmentToCurrentStep($expectedData, $expectedCaption);
+        AllureHelper::addAttachmentToCurrentStep($expectedData, $expectedCaption);
+
+        // Assert file names for both attachments are not the same.
+        $step = Allure::lifecycle()->getStepStorage()->pollLast();
+        $attachmentOne = $step->getAttachments()[0]->getSource();
+        $attachmentTwo = $step->getAttachments()[1]->getSource();
+        $this->assertNotEquals($attachmentOne, $attachmentTwo);
+    }
+
+    /**
+     * Mock entire attachment writing mechanisms
      * @throws \Exception
      */
     public function mockAttachmentWriteEvent()
     {
-        AspectMock::double(AddAttachmentEvent::class, [
+        AspectMock::double(AddUniqueAttachmentEvent::class, [
             "getAttachmentFileName" => self::MOCK_FILENAME
         ]);
     }
+
+    /**
+     * Mock only file writing mechanism
+     * @throws \Exception
+     */
+    public function mockCopyFile()
+    {
+        AspectMock::double(AddUniqueAttachmentEvent::class, [
+            "copyFile" => true
+        ]);
+    }
 }