diff --git a/CHANGELOG.md b/CHANGELOG.md index fd1e02008..e9b935493 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,30 @@ Magento Functional Testing Framework Changelog ================================================ +2.5.4 +----- + +* Traceability + * Introduced new `mftf doctor` command + * Command verifies and troubleshoots some configuration steps required for running tests + * Please see DevDocs for more details + * `<*Data>` actions now contain `API Endpoint` and `Request Header` artifacts. + * Introduced new `.env` configurations `ENABLE_BROWSER_LOG` and `BROWSER_LOG_BLACKLIST` + * Configuration enables allure artifacts for browser log entries if they are present after the step. + * Blacklist filters out logs from specific sources. +* Customizability + * Introduced `timeout=""` to `magentoCLI` actions. + +### GitHub Issues/Pull requests: +* [#317](https://github.com/magento/magento2-functional-testing-framework/pull/317) -- RetrieveEntityField generation does not consider ActionGroup as part of namespace +* [#433](https://github.com/magento/magento2-functional-testing-framework/pull/433) -- Add possibility to include multiple non primitive types in an array + +### Fixes +* A test now contains attachments for every exception encountered in the test (fix for a test `` exception overriding all test exceptions). +* Fixed hard requirement for `MAGENTO_BASE_URL` to contain a leading `/`. +* `magentoCLI` actions for `config:sensitive:set` no longer obscure CLI output. +* `WAIT_TIMEOUT` in the `.env` now correctly sets `pageload_timeout` configuration. +* Fixed an issue where `run:group` could not consolidate a `group` that had tests in and out of ``s. + 2.5.3 ----- diff --git a/bin/mftf b/bin/mftf index 9e5879280..7a9ca1cf2 100755 --- a/bin/mftf +++ b/bin/mftf @@ -27,9 +27,11 @@ try { try { + $version = json_decode(file_get_contents(FW_BP . DIRECTORY_SEPARATOR . 'composer.json'), true); + $version = $version['version']; $application = new Symfony\Component\Console\Application(); $application->setName('Magento Functional Testing Framework CLI'); - $application->setVersion('2.5.3'); + $application->setVersion($version); /** @var \Magento\FunctionalTestingFramework\Console\CommandListInterface $commandList */ $commandList = new \Magento\FunctionalTestingFramework\Console\CommandList; foreach ($commandList->getCommands() as $command) { diff --git a/composer.json b/composer.json index f4d192b8d..158287fbf 100755 --- a/composer.json +++ b/composer.json @@ -2,7 +2,7 @@ "name": "magento/magento2-functional-testing-framework", "description": "Magento2 Functional Testing Framework", "type": "library", - "version": "2.5.3", + "version": "2.5.4", "license": "AGPL-3.0", "keywords": ["magento", "automation", "functional", "testing"], "config": { @@ -12,7 +12,7 @@ "php": "7.0.2||7.0.4||~7.0.6||~7.1.0||~7.2.0||~7.3.0", "ext-curl": "*", "allure-framework/allure-codeception": "~1.3.0", - "codeception/codeception": "~2.3.4 || ~2.4.0 ", + "codeception/codeception": "~2.4.5", "composer/composer": "^1.4", "consolidation/robo": "^1.0.0", "csharpru/vault-php": "~3.5.3", diff --git a/composer.lock b/composer.lock index 34ea00854..8f2fcb8e9 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "43ac7d340b672794730bbbe4b47894e0", + "content-hash": "59e95cc1ae6311e93111bd7ced180d29", "packages": [ { "name": "allure-framework/allure-codeception", @@ -250,7 +250,7 @@ { "name": "Tobias Nyholm", "email": "tobias.nyholm@gmail.com", - "homepage": "https://github.com/Nyholm" + "homepage": "https://github.com/nyholm" } ], "description": "Library of all the php-cache adapters", @@ -263,31 +263,28 @@ }, { "name": "codeception/codeception", - "version": "2.3.9", + "version": "2.4.5", "source": { "type": "git", "url": "https://github.com/Codeception/Codeception.git", - "reference": "104f46fa0bde339f1bcc3a375aac21eb36e65a1e" + "reference": "5fee32d5c82791548931cbc34806b4de6aa1abfc" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/Codeception/Codeception/zipball/104f46fa0bde339f1bcc3a375aac21eb36e65a1e", - "reference": "104f46fa0bde339f1bcc3a375aac21eb36e65a1e", + "url": "https://api.github.com/repos/Codeception/Codeception/zipball/5fee32d5c82791548931cbc34806b4de6aa1abfc", + "reference": "5fee32d5c82791548931cbc34806b4de6aa1abfc", "shasum": "" }, "require": { - "behat/gherkin": "~4.4.0", - "codeception/stub": "^1.0", + "behat/gherkin": "^4.4.0", + "codeception/phpunit-wrapper": "^6.0.9|^7.0.6", + "codeception/stub": "^2.0", "ext-json": "*", "ext-mbstring": "*", "facebook/webdriver": ">=1.1.3 <2.0", "guzzlehttp/guzzle": ">=4.1.4 <7.0", "guzzlehttp/psr7": "~1.0", - "php": ">=5.4.0 <8.0", - "phpunit/php-code-coverage": ">=2.2.4 <6.0", - "phpunit/phpunit": ">=4.8.28 <5.0.0 || >=5.6.3 <7.0", - "sebastian/comparator": ">1.1 <3.0", - "sebastian/diff": ">=1.4 <3.0", + "php": ">=5.6.0 <8.0", "symfony/browser-kit": ">=2.7 <5.0", "symfony/console": ">=2.7 <5.0", "symfony/css-selector": ">=2.7 <5.0", @@ -353,27 +350,70 @@ "functional testing", "unit testing" ], - "time": "2018-02-26T23:29:41+00:00" + "time": "2018-08-01T07:21:49+00:00" }, { - "name": "codeception/stub", - "version": "1.0.4", + "name": "codeception/phpunit-wrapper", + "version": "6.7.0", "source": { "type": "git", - "url": "https://github.com/Codeception/Stub.git", - "reference": "681b62348837a5ef07d10d8a226f5bc358cc8805" + "url": "https://github.com/Codeception/phpunit-wrapper.git", + "reference": "93f59e028826464eac086052fa226e58967f6907" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/Codeception/Stub/zipball/681b62348837a5ef07d10d8a226f5bc358cc8805", - "reference": "681b62348837a5ef07d10d8a226f5bc358cc8805", + "url": "https://api.github.com/repos/Codeception/phpunit-wrapper/zipball/93f59e028826464eac086052fa226e58967f6907", + "reference": "93f59e028826464eac086052fa226e58967f6907", "shasum": "" }, "require": { - "phpunit/phpunit-mock-objects": ">2.3 <7.0" + "phpunit/php-code-coverage": ">=4.0.4 <6.0", + "phpunit/phpunit": ">=6.5.13 <7.0", + "sebastian/comparator": ">=1.2.4 <3.0", + "sebastian/diff": ">=1.4 <4.0" + }, + "replace": { + "codeception/phpunit-wrapper": "*" }, "require-dev": { - "phpunit/phpunit": ">=4.8 <8.0" + "codeception/specify": "*", + "vlucas/phpdotenv": "^3.0" + }, + "type": "library", + "autoload": { + "psr-4": { + "Codeception\\PHPUnit\\": "src\\" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Davert", + "email": "davert.php@resend.cc" + } + ], + "description": "PHPUnit classes used by Codeception", + "time": "2019-08-18T15:43:35+00:00" + }, + { + "name": "codeception/stub", + "version": "2.1.0", + "source": { + "type": "git", + "url": "https://github.com/Codeception/Stub.git", + "reference": "853657f988942f7afb69becf3fd0059f192c705a" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/Codeception/Stub/zipball/853657f988942f7afb69becf3fd0059f192c705a", + "reference": "853657f988942f7afb69becf3fd0059f192c705a", + "shasum": "" + }, + "require": { + "codeception/phpunit-wrapper": ">6.0.15 <6.1.0 | ^6.6.1 | ^7.7.1 | ^8.0.3" }, "type": "library", "autoload": { @@ -386,7 +426,7 @@ "MIT" ], "description": "Flexible Stub wrapper for PHPUnit's Mock Builder", - "time": "2018-05-17T09:31:08+00:00" + "time": "2019-03-02T15:35:10+00:00" }, { "name": "composer/ca-bundle", @@ -446,16 +486,16 @@ }, { "name": "composer/composer", - "version": "1.9.0", + "version": "1.9.1", "source": { "type": "git", "url": "https://github.com/composer/composer.git", - "reference": "314aa57fdcfc942065996f59fb73a8b3f74f3fa5" + "reference": "bb01f2180df87ce7992b8331a68904f80439dd2f" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/composer/composer/zipball/314aa57fdcfc942065996f59fb73a8b3f74f3fa5", - "reference": "314aa57fdcfc942065996f59fb73a8b3f74f3fa5", + "url": "https://api.github.com/repos/composer/composer/zipball/bb01f2180df87ce7992b8331a68904f80439dd2f", + "reference": "bb01f2180df87ce7992b8331a68904f80439dd2f", "shasum": "" }, "require": { @@ -522,7 +562,7 @@ "dependency", "package" ], - "time": "2019-08-02T18:55:33+00:00" + "time": "2019-11-01T16:20:17+00:00" }, { "name": "composer/semver", @@ -1061,6 +1101,7 @@ ], "description": "Promoting the interoperability of container objects (DIC, SL, etc.)", "homepage": "https://github.com/container-interop/container-interop", + "abandoned": "psr/container", "time": "2017-02-14T19:40:03+00:00" }, { diff --git a/dev/tests/_bootstrap.php b/dev/tests/_bootstrap.php index b41f80394..3f57ad139 100644 --- a/dev/tests/_bootstrap.php +++ b/dev/tests/_bootstrap.php @@ -45,7 +45,8 @@ 'MAGENTO_BACKEND_NAME' => 'admin', 'MAGENTO_ADMIN_USERNAME' => 'admin', 'MAGENTO_ADMIN_PASSWORD' => 'admin123', - 'DEFAULT_TIMEZONE' => 'America/Los_Angeles' + 'DEFAULT_TIMEZONE' => 'America/Los_Angeles', + 'WAIT_TIMEOUT' => '10' ]; foreach ($TEST_ENVS as $key => $value) { diff --git a/dev/tests/functional/standalone_bootstrap.php b/dev/tests/functional/standalone_bootstrap.php index 763062d04..486c7566b 100755 --- a/dev/tests/functional/standalone_bootstrap.php +++ b/dev/tests/functional/standalone_bootstrap.php @@ -15,10 +15,12 @@ require_once realpath(PROJECT_ROOT . '/vendor/autoload.php'); +$envFilePath = dirname(dirname(__DIR__)) . DIRECTORY_SEPARATOR; +defined('ENV_FILE_PATH') || define('ENV_FILE_PATH', $envFilePath); + //Load constants from .env file -$envFilePath = dirname(dirname(__DIR__)); -if (file_exists($envFilePath . DIRECTORY_SEPARATOR . '.env')) { - $env = new \Dotenv\Loader($envFilePath . DIRECTORY_SEPARATOR . '.env'); +if (file_exists(ENV_FILE_PATH . '.env')) { + $env = new \Dotenv\Loader(ENV_FILE_PATH . '.env'); $env->load(); foreach ($_ENV as $key => $var) { @@ -47,7 +49,10 @@ defined('DEFAULT_TIMEZONE') || define('DEFAULT_TIMEZONE', 'America/Los_Angeles'); $env->setEnvironmentVariable('DEFAULT_TIMEZONE', DEFAULT_TIMEZONE); - + + defined('WAIT_TIMEOUT') || define('WAIT_TIMEOUT', 30); + $env->setEnvironmentVariable('WAIT_TIMEOUT', 30); + try { new DateTimeZone(DEFAULT_TIMEZONE); } catch (\Exception $e) { diff --git a/dev/tests/unit/Magento/FunctionalTestFramework/Allure/AllureHelperTest.php b/dev/tests/unit/Magento/FunctionalTestFramework/Allure/AllureHelperTest.php index 048c6f7de..b7eafaead 100644 --- a/dev/tests/unit/Magento/FunctionalTestFramework/Allure/AllureHelperTest.php +++ b/dev/tests/unit/Magento/FunctionalTestFramework/Allure/AllureHelperTest.php @@ -6,6 +6,7 @@ namespace Tests\unit\Magento\FunctionalTestingFramework\Allure; use Magento\FunctionalTestingFramework\Allure\AllureHelper; +use Magento\FunctionalTestingFramework\Allure\Event\AddUniqueAttachmentEvent; use Yandex\Allure\Adapter\Allure; use Yandex\Allure\Adapter\Event\AddAttachmentEvent; use Yandex\Allure\Adapter\Event\StepFinishedEvent; @@ -24,6 +25,7 @@ class AllureHelperTest extends TestCase public function tearDown() { Allure::setDefaultLifecycle(); + AspectMock::clean(); } /** @@ -85,13 +87,48 @@ public function testAddAttachmentToLastStep() } /** - * Mock file system manipulation function + * AddAttachment actions should have files with different attachment names + * @throws \Yandex\Allure\Adapter\AllureException + */ + public function testAddAttachementUniqueName() + { + $this->mockCopyFile(); + $expectedData = "string"; + $expectedCaption = "caption"; + + //Prepare Allure lifecycle + Allure::lifecycle()->fire(new StepStartedEvent('firstStep')); + + //Call function twice + AllureHelper::addAttachmentToCurrentStep($expectedData, $expectedCaption); + AllureHelper::addAttachmentToCurrentStep($expectedData, $expectedCaption); + + // Assert file names for both attachments are not the same. + $step = Allure::lifecycle()->getStepStorage()->pollLast(); + $attachmentOne = $step->getAttachments()[0]->getSource(); + $attachmentTwo = $step->getAttachments()[1]->getSource(); + $this->assertNotEquals($attachmentOne, $attachmentTwo); + } + + /** + * Mock entire attachment writing mechanisms * @throws \Exception */ public function mockAttachmentWriteEvent() { - AspectMock::double(AddAttachmentEvent::class, [ + AspectMock::double(AddUniqueAttachmentEvent::class, [ "getAttachmentFileName" => self::MOCK_FILENAME ]); } + + /** + * Mock only file writing mechanism + * @throws \Exception + */ + public function mockCopyFile() + { + AspectMock::double(AddUniqueAttachmentEvent::class, [ + "copyFile" => true + ]); + } } diff --git a/dev/tests/unit/Magento/FunctionalTestFramework/Console/BaseGenerateCommandTest.php b/dev/tests/unit/Magento/FunctionalTestFramework/Console/BaseGenerateCommandTest.php new file mode 100644 index 000000000..1b1686ce2 --- /dev/null +++ b/dev/tests/unit/Magento/FunctionalTestFramework/Console/BaseGenerateCommandTest.php @@ -0,0 +1,207 @@ + $testOne], [], []); + + $testArray = ['Test1' => $testOne]; + $suiteArray = ['Suite1' => $suiteOne]; + + $this->mockHandlers($testArray, $suiteArray); + + $actual = json_decode($this->callTestConfig(['Test1']), true); + $expected = ['tests' => null, 'suites' => ['Suite1' => ['Test1']]]; + $this->assertEquals($expected, $actual); + } + + public function testOneTestTwoSuitesConfig() + { + $testOne = new TestObject('Test1', [], [], []); + $suiteOne = new SuiteObject('Suite1', ['Test1' => $testOne], [], []); + $suiteTwo = new SuiteObject('Suite2', ['Test1' => $testOne], [], []); + + $testArray = ['Test1' => $testOne]; + $suiteArray = ['Suite1' => $suiteOne, 'Suite2' => $suiteTwo]; + + $this->mockHandlers($testArray, $suiteArray); + + $actual = json_decode($this->callTestConfig(['Test1']), true); + $expected = ['tests' => null, 'suites' => ['Suite1' => ['Test1'], 'Suite2' => ['Test1']]]; + $this->assertEquals($expected, $actual); + } + + public function testOneTestOneGroup() + { + $testOne = new TestObject('Test1', [], ['group' => ['Group1']], []); + + $testArray = ['Test1' => $testOne]; + $suiteArray = []; + + $this->mockHandlers($testArray, $suiteArray); + + $actual = json_decode($this->callGroupConfig(['Group1']), true); + $expected = ['tests' => ['Test1'], 'suites' => null]; + $this->assertEquals($expected, $actual); + } + + public function testThreeTestsTwoGroup() + { + $testOne = new TestObject('Test1', [], ['group' => ['Group1']], []); + $testTwo = new TestObject('Test2', [], ['group' => ['Group1']], []); + $testThree = new TestObject('Test3', [], ['group' => ['Group2']], []); + + $testArray = ['Test1' => $testOne, 'Test2' => $testTwo, 'Test3' => $testThree]; + $suiteArray = []; + + $this->mockHandlers($testArray, $suiteArray); + + $actual = json_decode($this->callGroupConfig(['Group1', 'Group2']), true); + $expected = ['tests' => ['Test1', 'Test2', 'Test3'], 'suites' => null]; + $this->assertEquals($expected, $actual); + } + + public function testOneTestOneSuiteOneGroupConfig() + { + $testOne = new TestObject('Test1', [], ['group' => ['Group1']], []); + $suiteOne = new SuiteObject('Suite1', ['Test1' => $testOne], [], []); + + $testArray = ['Test1' => $testOne]; + $suiteArray = ['Suite1' => $suiteOne]; + + $this->mockHandlers($testArray, $suiteArray); + + $actual = json_decode($this->callGroupConfig(['Group1']), true); + $expected = ['tests' => null, 'suites' => ['Suite1' => ['Test1']]]; + $this->assertEquals($expected, $actual); + } + + public function testTwoTestOneSuiteTwoGroupConfig() + { + $testOne = new TestObject('Test1', [], ['group' => ['Group1']], []); + $testTwo = new TestObject('Test2', [], ['group' => ['Group2']], []); + $suiteOne = new SuiteObject('Suite1', ['Test1' => $testOne, 'Test2' => $testTwo], [], []); + + $testArray = ['Test1' => $testOne, 'Test2' => $testTwo]; + $suiteArray = ['Suite1' => $suiteOne]; + + $this->mockHandlers($testArray, $suiteArray); + + $actual = json_decode($this->callGroupConfig(['Group1', 'Group2']), true); + $expected = ['tests' => null, 'suites' => ['Suite1' => ['Test1', 'Test2']]]; + $this->assertEquals($expected, $actual); + } + + public function testTwoTestTwoSuiteOneGroupConfig() + { + $testOne = new TestObject('Test1', [], ['group' => ['Group1']], []); + $testTwo = new TestObject('Test2', [], ['group' => ['Group1']], []); + $suiteOne = new SuiteObject('Suite1', ['Test1' => $testOne], [], []); + $suiteTwo = new SuiteObject('Suite2', ['Test2' => $testTwo], [], []); + + $testArray = ['Test1' => $testOne, 'Test2' => $testTwo]; + $suiteArray = ['Suite1' => $suiteOne, 'Suite2' => $suiteTwo]; + + $this->mockHandlers($testArray, $suiteArray); + + $actual = json_decode($this->callGroupConfig(['Group1']), true); + $expected = ['tests' => null, 'suites' => ['Suite1' => ['Test1'], 'Suite2' => ['Test2']]]; + $this->assertEquals($expected, $actual); + } + + /** + * Test specific usecase of a test that is in a group with the group being called along with the suite + * i.e. run:group Group1 Suite1 + * @throws \Exception + */ + public function testThreeTestOneSuiteOneGroupMix() + { + $testOne = new TestObject('Test1', [], [], []); + $testTwo = new TestObject('Test2', [], [], []); + $testThree = new TestObject('Test3', [], ['group' => ['Group1']], []); + $suiteOne = new SuiteObject( + 'Suite1', + ['Test1' => $testOne, 'Test2' => $testTwo, 'Test3' => $testThree], + [], + [] + ); + + $testArray = ['Test1' => $testOne, 'Test2' => $testTwo, 'Test3' => $testThree]; + $suiteArray = ['Suite1' => $suiteOne]; + + $this->mockHandlers($testArray, $suiteArray); + + $actual = json_decode($this->callGroupConfig(['Group1', 'Suite1']), true); + $expected = ['tests' => null, 'suites' => ['Suite1' => []]]; + $this->assertEquals($expected, $actual); + } + + /** + * Mock handlers to skip parsing + * @param array $testArray + * @param array $suiteArray + * @throws \Exception + */ + public function mockHandlers($testArray, $suiteArray) + { + AspectMock::double(TestObjectHandler::class, ['initTestData' => ''])->make(); + $handler = TestObjectHandler::getInstance(); + $property = new \ReflectionProperty(TestObjectHandler::class, 'tests'); + $property->setAccessible(true); + $property->setValue($handler, $testArray); + + AspectMock::double(SuiteObjectHandler::class, ['initSuiteData' => ''])->make(); + $handler = SuiteObjectHandler::getInstance(); + $property = new \ReflectionProperty(SuiteObjectHandler::class, 'suiteObjects'); + $property->setAccessible(true); + $property->setValue($handler, $suiteArray); + } + + /** + * Changes visibility and runs getTestAndSuiteConfiguration + * @param array $testArray + * @return string + */ + public function callTestConfig($testArray) + { + $command = new BaseGenerateCommand(); + $class = new \ReflectionClass($command); + $method = $class->getMethod('getTestAndSuiteConfiguration'); + $method->setAccessible(true); + return $method->invokeArgs($command, [$testArray]); + } + + /** + * Changes visibility and runs getGroupAndSuiteConfiguration + * @param array $groupArray + * @return string + */ + public function callGroupConfig($groupArray) + { + $command = new BaseGenerateCommand(); + $class = new \ReflectionClass($command); + $method = $class->getMethod('getGroupAndSuiteConfiguration'); + $method->setAccessible(true); + return $method->invokeArgs($command, [$groupArray]); + } +} diff --git a/dev/tests/unit/Magento/FunctionalTestFramework/DataGenerator/Persist/OperationDataArrayResolverTest.php b/dev/tests/unit/Magento/FunctionalTestFramework/DataGenerator/Persist/OperationDataArrayResolverTest.php index 9e425e020..fb6dcd865 100644 --- a/dev/tests/unit/Magento/FunctionalTestFramework/DataGenerator/Persist/OperationDataArrayResolverTest.php +++ b/dev/tests/unit/Magento/FunctionalTestFramework/DataGenerator/Persist/OperationDataArrayResolverTest.php @@ -9,6 +9,7 @@ use Magento\FunctionalTestingFramework\DataGenerator\Handlers\DataObjectHandler; use Magento\FunctionalTestingFramework\DataGenerator\Handlers\OperationDefinitionObjectHandler; use Magento\FunctionalTestingFramework\DataGenerator\Persist\OperationDataArrayResolver; +use Magento\FunctionalTestingFramework\Util\Iterator\AbstractIterator; use Magento\FunctionalTestingFramework\Util\MagentoTestCase; use tests\unit\Util\EntityDataObjectBuilder; use tests\unit\Util\OperationDefinitionBuilder; @@ -355,6 +356,127 @@ public function testNestedMetadataArrayOfValue() $this->assertEquals(self::NESTED_METADATA_ARRAY_RESULT, $result); } + public function testNestedMetadataArrayOfDiverseObjects() + { + + $entityDataObjBuilder = new EntityDataObjectBuilder(); + $parentDataObject = $entityDataObjBuilder + ->withName("parentObject") + ->withType("parentType") + ->withLinkedEntities(['child1Object' => 'childType1','child2Object' => 'childType2']) + ->build(); + + $child1DataObject = $entityDataObjBuilder + ->withName('child1Object') + ->withType('childType1') + ->withDataFields(['city' => 'Testcity','zip' => 12345]) + ->build(); + + $child2DataObject = $entityDataObjBuilder + ->withName('child2Object') + ->withType('childType2') + ->withDataFields(['city' => 'Testcity 2','zip' => 54321,'state' => 'Teststate']) + ->build(); + + $mockDOHInstance = AspectMock::double( + DataObjectHandler::class, + [ + 'getObject' => function ($name) use ($child1DataObject, $child2DataObject) { + switch ($name) { + case 'child1Object': + return $child1DataObject; + case 'child2Object': + return $child2DataObject; + } + } + ] + )->make(); + AspectMock::double(DataObjectHandler::class, [ + 'getInstance' => $mockDOHInstance + ]); + + $operationDefinitionBuilder = new OperationDefinitionBuilder(); + $child1OperationDefinition = $operationDefinitionBuilder + ->withName('createchildType1') + ->withOperation('create') + ->withType('childType1') + ->withMetadata([ + 'city' => 'string', + 'zip' => 'integer' + ])->build(); + + $child2OperationDefinition = $operationDefinitionBuilder + ->withName('createchildType2') + ->withOperation('create') + ->withType('childType2') + ->withMetadata([ + 'city' => 'string', + 'zip' => 'integer', + 'state' => 'string' + ])->build(); + + $mockODOHInstance = AspectMock::double( + OperationDefinitionObjectHandler::class, + [ + 'getObject' => function ($name) use ($child1OperationDefinition, $child2OperationDefinition) { + switch ($name) { + case 'createchildType1': + return $child1OperationDefinition; + case 'createchildType2': + return $child2OperationDefinition; + } + } + ] + )->make(); + AspectMock::double( + OperationDefinitionObjectHandler::class, + [ + 'getInstance' => $mockODOHInstance + ] + ); + + $arrayObElementBuilder = new OperationElementBuilder(); + $arrayElement = $arrayObElementBuilder + ->withKey('address') + ->withType(['childType1','childType2']) + ->withFields([]) + ->withElementType(OperationDefinitionObjectHandler::ENTITY_OPERATION_ARRAY) + //->withNestedElements(['childType1' => $child1Element, 'childType2' => $child2Element]) + ->build(); + + $parentOpElementBuilder = new OperationElementBuilder(); + $parentElement = $parentOpElementBuilder + ->withKey('parentType') + ->withType('parentType') + ->addElements(['address' => $arrayElement]) + ->build(); + + $operationResolver = new OperationDataArrayResolver(); + $result = $operationResolver->resolveOperationDataArray($parentDataObject, [$parentElement], 'create', false); + + $expectedResult = [ + 'parentType' => [ + 'address' => [ + [ + 'city' => 'Testcity', + 'zip' => '12345' + ], + [ + 'city' => 'Testcity 2', + 'zip' => '54321', + 'state' => 'Teststate' + ] + ], + 'name' => 'Hopper', + 'gpa' => '3.5678', + 'phone' => '5555555', + 'isPrimary' => '1' + ] + ]; + + $this->assertEquals($expectedResult, $result); + } + /** * After class functionality * @return void diff --git a/dev/tests/unit/Magento/FunctionalTestFramework/Extension/BrowserLogUtilTest.php b/dev/tests/unit/Magento/FunctionalTestFramework/Extension/BrowserLogUtilTest.php new file mode 100644 index 000000000..32cac698e --- /dev/null +++ b/dev/tests/unit/Magento/FunctionalTestFramework/Extension/BrowserLogUtilTest.php @@ -0,0 +1,76 @@ + "WARNING", + "message" => "warningMessage", + "source" => "console-api", + "timestamp" => 1234567890 + ]; + $entryTwo = [ + "level" => "ERROR", + "message" => "errorMessage", + "source" => "other", + "timestamp" => 1234567890 + ]; + $entryThree = [ + "level" => "LOG", + "message" => "logMessage", + "source" => "javascript", + "timestamp" => 1234567890 + ]; + $log = [ + $entryOne, + $entryTwo, + $entryThree + ]; + + $actual = BrowserLogUtil::getLogsOfType($log, 'console-api'); + + self::assertEquals($entryOne, $actual[0]); + } + + public function testFilterLogsOfType() + { + $entryOne = [ + "level" => "WARNING", + "message" => "warningMessage", + "source" => "console-api", + "timestamp" => 1234567890 + ]; + $entryTwo = [ + "level" => "ERROR", + "message" => "errorMessage", + "source" => "other", + "timestamp" => 1234567890 + ]; + $entryThree = [ + "level" => "LOG", + "message" => "logMessage", + "source" => "javascript", + "timestamp" => 1234567890 + ]; + $log = [ + $entryOne, + $entryTwo, + $entryThree + ]; + + $actual = BrowserLogUtil::filterLogsOfType($log, 'console-api'); + + self::assertEquals($entryTwo, $actual[0]); + self::assertEquals($entryThree, $actual[1]); + } +} diff --git a/dev/tests/unit/Magento/FunctionalTestFramework/Util/Path/FilePathFormatterTest.php b/dev/tests/unit/Magento/FunctionalTestFramework/Util/Path/FilePathFormatterTest.php new file mode 100644 index 000000000..bac14113d --- /dev/null +++ b/dev/tests/unit/Magento/FunctionalTestFramework/Util/Path/FilePathFormatterTest.php @@ -0,0 +1,84 @@ +assertEquals($expectedPath, FilePathFormatter::format($path, $withTrailingSeparator)); + } else { + // Assert no exception + FilePathFormatter::format($path, $withTrailingSeparator); + $this->assertTrue(true); + } + } + + /** + * Test file format with exception + * + * @dataProvider formatExceptionDataProvider + * @param string $path + * @param boolean $withTrailingSeparator + * @return void + * @throws TestFrameworkException + */ + public function testFormatWithException($path, $withTrailingSeparator) + { + $this->expectException(TestFrameworkException::class); + $this->expectExceptionMessage("Invalid or non-existing file: $path\n"); + FilePathFormatter::format($path, $withTrailingSeparator); + } + + /** + * Data input + * + * @return array + */ + public function formatDataProvider() + { + $path1 = rtrim(TESTS_BP, '/'); + $path2 = $path1 . DIRECTORY_SEPARATOR; + return [ + [$path1, null, $path1], + [$path1, false, $path1], + [$path1, true, $path2], + [$path2, null, $path1], + [$path2, false, $path1], + [$path2, true, $path2], + [__DIR__. DIRECTORY_SEPARATOR . basename(__FILE__), null, __FILE__], + ['', null, null] // Empty string is valid + ]; + } + + /** + * Invalid data input + * + * @return array + */ + public function formatExceptionDataProvider() + { + return [ + ['abc', null], + ['X://some\dir/@', null], + ]; + } +} diff --git a/dev/tests/unit/Magento/FunctionalTestFramework/Util/Path/UrlFormatterTest.php b/dev/tests/unit/Magento/FunctionalTestFramework/Util/Path/UrlFormatterTest.php new file mode 100644 index 000000000..c66c9558c --- /dev/null +++ b/dev/tests/unit/Magento/FunctionalTestFramework/Util/Path/UrlFormatterTest.php @@ -0,0 +1,91 @@ +assertEquals($expectedPath, UrlFormatter::format($path, $withTrailingSeparator)); + } + + /** + * Test url format with exception + * + * @dataProvider formatExceptionDataProvider + * @param string $path + * @param boolean $withTrailingSeparator + * @return void + * @throws TestFrameworkException + */ + public function testFormatWithException($path, $withTrailingSeparator) + { + $this->expectException(TestFrameworkException::class); + $this->expectExceptionMessage("Invalid url: $path\n"); + UrlFormatter::format($path, $withTrailingSeparator); + } + + /** + * Data input + * + * @return array + */ + public function formatDataProvider() + { + $url1 = 'http://magento.local/index.php'; + $url2 = $url1 . '/'; + $url3 = 'https://www.example.com/index.php/admin'; + $url4 = $url3 . '/'; + $url5 = 'www.google.com'; + $url6 = 'http://www.google.com/'; + $url7 = 'http://127.0.0.1:8200'; + $url8 = 'wwøw.goåoøgle.coøm'; + $url9 = 'http://www.google.com'; + return [ + [$url1, null, $url1], + [$url1, false, $url1], + [$url1, true, $url2], + [$url2, null, $url1], + [$url2, false, $url1], + [$url2, true, $url2], + [$url3, null, $url3], + [$url3, false, $url3], + [$url3, true, $url4], + [$url4, null, $url3], + [$url4, false, $url3], + [$url4, true, $url4], + [$url5, true, $url6], + [$url7, false, $url7], + [$url8, false, $url9], + ]; + } + + /** + * Invalid data input + * + * @return array + */ + public function formatExceptionDataProvider() + { + return [ + ['', null], + ]; + } +} diff --git a/dev/tests/verification/Resources/ActionGroupWithStepKeyReferences.txt b/dev/tests/verification/Resources/ActionGroupWithStepKeyReferences.txt index eddaaf784..d1cae1da4 100644 --- a/dev/tests/verification/Resources/ActionGroupWithStepKeyReferences.txt +++ b/dev/tests/verification/Resources/ActionGroupWithStepKeyReferences.txt @@ -45,7 +45,7 @@ class ActionGroupWithStepKeyReferencesCest $I->fillField($action1); // stepKey: action1ActionGroup $I->comment("Invocation stepKey will be appended in non stepKey instances"); $action3ActionGroup = $I->executeJS($action3ActionGroup); // stepKey: action3ActionGroup - $action4ActionGroup = $I->magentoCLI($action4ActionGroup, "\"stuffHere\""); // stepKey: action4ActionGroup + $action4ActionGroup = $I->magentoCLI($action4ActionGroup, 60, "\"stuffHere\""); // stepKey: action4ActionGroup $I->comment($action4ActionGroup); $date = new \DateTime(); $date->setTimestamp(strtotime("{$action5}")); diff --git a/dev/tests/verification/Resources/BasicFunctionalTest.txt b/dev/tests/verification/Resources/BasicFunctionalTest.txt index f82c84119..bc7fe0b94 100644 --- a/dev/tests/verification/Resources/BasicFunctionalTest.txt +++ b/dev/tests/verification/Resources/BasicFunctionalTest.txt @@ -126,8 +126,14 @@ class BasicFunctionalTestCest $grabMultipleKey1 = $I->grabMultiple(".functionalTestSelector"); // stepKey: grabMultipleKey1 $grabTextFromKey1 = $I->grabTextFrom(".functionalTestSelector"); // stepKey: grabTextFromKey1 $grabValueFromKey1 = $I->grabValueFrom(".functionalTestSelector"); // stepKey: grabValueFromKey1 - $magentoCli1 = $I->magentoCLI("maintenance:enable", "\"stuffHere\""); // stepKey: magentoCli1 + $magentoCli1 = $I->magentoCLI("maintenance:enable", 60, "\"stuffHere\""); // stepKey: magentoCli1 $I->comment($magentoCli1); + $magentoCli2 = $I->magentoCLI("maintenance:enable", 120, "\"stuffHere\""); // stepKey: magentoCli2 + $I->comment($magentoCli2); + $magentoCli3 = $I->magentoCLISecret("config:set somePath " . CredentialStore::getInstance()->getSecret("someKey"), 60); // stepKey: magentoCli3 + $I->comment($magentoCli3); // stepKey: magentoCli3 + $magentoCli4 = $I->magentoCLISecret("config:set somePath " . CredentialStore::getInstance()->getSecret("someKey"), 120); // stepKey: magentoCli4 + $I->comment($magentoCli4); // stepKey: magentoCli4 $I->makeScreenshot("screenShotInput"); // stepKey: makeScreenshotKey1 $I->maximizeWindow(); // stepKey: maximizeWindowKey1 $I->moveBack(); // stepKey: moveBackKey1 diff --git a/dev/tests/verification/Resources/DataReplacementTest.txt b/dev/tests/verification/Resources/DataReplacementTest.txt index 731ae63f8..4b4531a4e 100644 --- a/dev/tests/verification/Resources/DataReplacementTest.txt +++ b/dev/tests/verification/Resources/DataReplacementTest.txt @@ -52,7 +52,7 @@ class DataReplacementTestCest $I->searchAndMultiSelectOption("#selector", [msq("uniqueData") . "John", "Doe" . msq("uniqueData")]); // stepKey: parameterArrayReplacementMSQBoth $I->selectMultipleOptions("#Doe" . msq("uniqueData"), "#element", [msq("uniqueData") . "John", "Doe" . msq("uniqueData")]); // stepKey: multiSelectDataReplacement $I->fillField(".selector", "0"); // stepKey: insertZero - $insertCommand = $I->magentoCLI("do something Doe" . msq("uniqueData") . " with uniqueness"); // stepKey: insertCommand + $insertCommand = $I->magentoCLI("do something Doe" . msq("uniqueData") . " with uniqueness", 60); // stepKey: insertCommand $I->comment($insertCommand); $I->seeInPageSource("StringBefore John StringAfter"); // stepKey: htmlReplace1 $I->seeInPageSource("#John"); // stepKey: htmlReplace2 diff --git a/dev/tests/verification/Resources/PersistenceCustomFieldsTest.txt b/dev/tests/verification/Resources/PersistenceCustomFieldsTest.txt index 38dc364e7..0a6deaca8 100644 --- a/dev/tests/verification/Resources/PersistenceCustomFieldsTest.txt +++ b/dev/tests/verification/Resources/PersistenceCustomFieldsTest.txt @@ -111,5 +111,36 @@ class PersistenceCustomFieldsTestCest ); $I->comment("Exiting Action Group [createdAG] PersistenceActionGroup"); + $I->comment("Entering Action Group [AGKEY] DataPersistenceSelfReferenceActionGroup"); + $I->comment("[createData1AGKEY] create 'entity1' entity"); + PersistedObjectHandler::getInstance()->createEntity( + "createData1AGKEY", + "test", + "entity1", + [], + [] + ); + + $I->comment("[createData2AGKEY] create 'entity2' entity"); + PersistedObjectHandler::getInstance()->createEntity( + "createData2AGKEY", + "test", + "entity2", + [], + [] + ); + + $createData3AGKEYFields['key1'] = PersistedObjectHandler::getInstance()->retrieveEntityField('createData1AGKEY', 'field', 'test'); + $createData3AGKEYFields['key2'] = PersistedObjectHandler::getInstance()->retrieveEntityField('createData2AGKEY', 'field', 'test'); + $I->comment("[createData3AGKEY] create 'entity3' entity"); + PersistedObjectHandler::getInstance()->createEntity( + "createData3AGKEY", + "test", + "entity3", + [], + $createData3AGKEYFields + ); + + $I->comment("Exiting Action Group [AGKEY] DataPersistenceSelfReferenceActionGroup"); } } diff --git a/dev/tests/verification/TestModule/ActionGroup/PersistenceActionGroup.xml b/dev/tests/verification/TestModule/ActionGroup/PersistenceActionGroup.xml index c4f894cfc..f98fd2406 100644 --- a/dev/tests/verification/TestModule/ActionGroup/PersistenceActionGroup.xml +++ b/dev/tests/verification/TestModule/ActionGroup/PersistenceActionGroup.xml @@ -30,4 +30,12 @@ + + + + + $createData1.field$ + $createData2.field$ + + \ No newline at end of file diff --git a/dev/tests/verification/TestModule/Test/BasicFunctionalTest.xml b/dev/tests/verification/TestModule/Test/BasicFunctionalTest.xml index ff0708554..c23a3ce60 100644 --- a/dev/tests/verification/TestModule/Test/BasicFunctionalTest.xml +++ b/dev/tests/verification/TestModule/Test/BasicFunctionalTest.xml @@ -76,6 +76,9 @@ + + + diff --git a/dev/tests/verification/TestModule/Test/PersistenceCustomFieldsTest.xml b/dev/tests/verification/TestModule/Test/PersistenceCustomFieldsTest.xml index 747c2c132..c56106693 100644 --- a/dev/tests/verification/TestModule/Test/PersistenceCustomFieldsTest.xml +++ b/dev/tests/verification/TestModule/Test/PersistenceCustomFieldsTest.xml @@ -33,5 +33,6 @@ + diff --git a/dev/tests/verification/Tests/SuiteGenerationTest.php b/dev/tests/verification/Tests/SuiteGenerationTest.php index d2242fb52..485ef6411 100644 --- a/dev/tests/verification/Tests/SuiteGenerationTest.php +++ b/dev/tests/verification/Tests/SuiteGenerationTest.php @@ -6,11 +6,13 @@ namespace tests\verification\Tests; +use Magento\FunctionalTestingFramework\Exceptions\TestFrameworkException; use Magento\FunctionalTestingFramework\Suite\SuiteGenerator; use Magento\FunctionalTestingFramework\Util\Filesystem\DirSetupUtil; use Magento\FunctionalTestingFramework\Util\Manifest\DefaultTestManifest; use Magento\FunctionalTestingFramework\Util\Manifest\ParallelTestManifest; use Magento\FunctionalTestingFramework\Util\Manifest\TestManifestFactory; +use Magento\FunctionalTestingFramework\Util\Path\FilePathFormatter; use PHPUnit\Util\Filesystem; use Symfony\Component\Yaml\Yaml; use tests\unit\Util\TestLoggingUtil; @@ -413,11 +415,11 @@ public static function tearDownAfterClass() * Getter for manifest file path * * @return string + * @throws TestFrameworkException */ private static function getManifestFilePath() { - return TESTS_BP . - DIRECTORY_SEPARATOR . + return FilePathFormatter::format(TESTS_BP) . "verification" . DIRECTORY_SEPARATOR . "_generated" . diff --git a/docs/commands/mftf.md b/docs/commands/mftf.md index 484f5c84f..2440428f9 100644 --- a/docs/commands/mftf.md +++ b/docs/commands/mftf.md @@ -109,6 +109,24 @@ You can include options to set configuration parameter values for your environme vendor/bin/mftf build:project --MAGENTO_BASE_URL=http://magento.local/ --MAGENTO_BACKEND_NAME=admin214365 ``` +### `doctor` + +#### Description + +Diagnose MFTF configuration and setup. Currently this command will check the following: +- Verify admin credentials are valid. Allowing MFTF authenticates and runs API requests to Magento through cURL +- Verify that Selenium is up and running and available for MFTF +- Verify that new session of browser can open Magento admin and store front urls +- Verify that MFTF can run MagentoCLI commands + +#### Usage + +```bash +vendor/bin/mftf doctor +``` + +#### Options + ### `generate:tests` #### Description diff --git a/docs/configuration.md b/docs/configuration.md index c741e2f60..5140c01e7 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -277,6 +277,32 @@ Example: CREDENTIAL_VAULT_SECRET_BASE_PATH=secret ``` +### ENABLE_BROWSER_LOG + +Enables addition of browser logs to Allure steps + +```conf +ENABLE_BROWSER_LOG=true +``` + +### BROWSER_LOG_BLACKLIST + +Blacklists types of browser log entries from appearing in Allure steps. + +Denoted in browser log entry as `"SOURCE": "type"`. + +```conf +BROWSER_LOG_BLACKLIST=other,console-api +``` + +### WAIT_TIMEOUT + +Global MFTF configuration for the default amount of time (in seconds) that a test will wait while loading a page. + +```conf +WAIT_TIMEOUT=30 +``` + [`MAGENTO_CLI_COMMAND_PATH`]: #magento_cli_command_path diff --git a/docs/credentials.md b/docs/credentials.md index 3065454b0..a2850cfe8 100644 --- a/docs/credentials.md +++ b/docs/credentials.md @@ -1,6 +1,6 @@ # Credentials -When you test functionality that involves external services such as UPS, FedEx, PayPal, or SignifyD, +When you test functionality that involves external services such as UPS, FedEx, PayPal, or SignifyD, use the MFTF credentials feature to hide sensitive [data][] like integration tokens and API keys. Currently the MFTF supports two types of credential storage: @@ -53,9 +53,9 @@ magento/carriers_usps_password=Lmgxvrq89uPwECeV #magento/carriers_dhl_id_us=dhl_test_user #magento/carriers_dhl_password_us=Mlgxv3dsagVeG .... -``` +``` -Or add new key & value pairs for your own credentials. The keys use the following format: +Or add new key/value pairs for your own credentials. The keys use the following format: ```conf /= @@ -64,7 +64,7 @@ Or add new key & value pairs for your own credentials. The keys use the followin
The `/` symbol is not supported in a `key_name` other than the one after your vendor or extension name.
- + Otherwise you are free to use any other `key_name` you like, as they are merely the keys to reference from your tests. ```conf @@ -74,10 +74,10 @@ vendor/my_awesome_service_token=rRVSVnh3cbDsVG39oTMz4A ## Configure Vault Storage -Hashicorp vault secures, stores, and tightly controls access to data in modern computing. -It provides advanced data protection for your testing credentials. +Hashicorp vault secures, stores, and tightly controls access to data in modern computing. +It provides advanced data protection for your testing credentials. -The MFTF works with both `vault enterprise` and `vault open source` that use `KV Version 2` secret engine. +The MFTF works with both `vault enterprise` and `vault open source` that use `KV Version 2` secret engine. ### Install vault CLI @@ -95,8 +95,8 @@ vault login -method -path ### Store secrets in vault -The MFTF uses the `KV Version 2` secret engine for secret storage. -More information for working with `KV Version 2` can be found in [Vault KV2][Vault KV2]. +The MFTF uses the `KV Version 2` secret engine for secret storage. +More information for working with `KV Version 2` can be found in [Vault KV2][Vault KV2]. #### Secrets path and key convention @@ -125,9 +125,9 @@ vault kv put secret/mftf/magento/carriers_usps_password carriers_usps_password=L ### Setup MFTF to use vault -Add vault configuration environment variables [`CREDENTIAL_VAULT_ADDRESS`][] and [`CREDENTIAL_VAULT_SECRET_BASE_PATH`][] +Add vault configuration environment variables [`CREDENTIAL_VAULT_ADDRESS`][] and [`CREDENTIAL_VAULT_SECRET_BASE_PATH`][] from `etc/config/.env.example` in `.env`. -Set values according to your vault server configuration. +Set values according to your vault server configuration. ```conf # Default vault dev server @@ -137,7 +137,7 @@ CREDENTIAL_VAULT_SECRET_BASE_PATH=secret ## Configure both File Storage and Vault Storage -It is possible and sometimes useful to setup and use both `.credentials` file and vault for secret storage at the same time. +It is possible and sometimes useful to setup and use both `.credentials` file and vault for secret storage at the same time. In this case, the MFTF tests are able to read secret data at runtime from both storage options, but the local `.credentials` file will take precedence. @@ -150,11 +150,12 @@ Define the value as a reference to the corresponding key in the credentials file - `_CREDS` is an environment constant pointing to the `.credentials` file - `my_data_key` is a key in the the `.credentials` file or vault that contains the value to be used in a test step + - for File Storage, ensure your key contains the vendor prefix, i.e. `vendor/my_data_key` -For example, reference secret data in the [`fillField`][] action with the `userInput` attribute. +For example, to reference secret data in the [`fillField`][] action, use the `userInput` attribute using a typical File Storage: ```xml - + ``` diff --git a/docs/guides/action-groups.md b/docs/guides/action-groups.md new file mode 100644 index 000000000..30c531e4e --- /dev/null +++ b/docs/guides/action-groups.md @@ -0,0 +1,82 @@ +# Action Group Best Practices + +We strive to write tests using only action groups. Fortunately, we have built up a large set of action groups to get started. We can make use of them and extend them for our own specific needs. In some cases, we may never even need to write action groups of our own. We may be able to simply chain together calls to existing action groups to implement our new test case. + +## Why use Action Groups? + +Action groups simplify maintainability by reducing duplication. Because they are re-usable building blocks, odds are that they are already made use of by existing tests in the Magento codebase. This proves their stability through real-world use. Take for example, the action group named `LoginAsAdmin`: + +```xml + + + Login to Backend Admin using provided User Data. PLEASE NOTE: This Action Group does NOT validate that you are Logged In. + + + + + + + + + + + +``` + +Logging in to the admin panel is one of the most used action groups. It is used around 1,500 times at the time of this writing. + +Imagine if this was not an action group and instead we were to copy and paste these 5 actions every time. In that scenario, if a small change was needed, it would require a lot of work. But with the action group, we can make the change in one place. + +## How to extend action groups + +Again using `LoginAsAdmin` as our example, we trim away metadata to clearly reveal that this action group performs 5 actions: + +```xml + + ... + + + + + + +``` + +This works against the standard Magento admin panel login page. Bu imagine we are working on a Magento extension that adds a CAPTCHA field to the login page. If we create and activate this extension and then run all existing tests, we can expect almost everything to fail because the CAPTCHA field is left unfilled. + +We can overcome this by making use of MFTF's extensibility. All we need to do is to provide a "merge" that modifies the existing `LoginAsAdmin` action group. Our merge file will look like: + +```xml + + + +``` + +Because the name of this merge is also `LoginAsAdmin`, the two get merged together and an additional step happens everytime this action group is used. + +To continue this example, imagine someone else is working on a 'Two-Factor Authentication' extension and they also provide a merge for the `LoginAsAdmin` action group. Their merge looks similar to what we have already seen. The only difference is that this time we fill a different field: + +```xml + + + +``` + +Bringing it all together, our resulting `LoginAsAdmin` action group becomes this: + +```xml + + ... + + + + + + + + +``` + +No one file contains this exact content as above, but instead all three files come together to form this action group. + +This extensibility can be applied in many ways. We can use it to affect existing Magento entities such as tests, action groups, and data. Not so obvious is that this tehcnique can be used within your own entities to make them more maintainable as well. diff --git a/docs/merging.md b/docs/merging.md index 99df18464..e4b91fb5f 100644 --- a/docs/merging.md +++ b/docs/merging.md @@ -6,7 +6,7 @@ The MFTF allows you to merge test components defined in XML files, such as: - [``][] - [``][] - [``][] -- `` +- [``][] You can create, delete, or update the component. It is useful for supporting rapid test creation for extensions and customizations. @@ -569,3 +569,4 @@ The `_defaultSample` results corresponds to: [``]: ./page.md [``]: ./section.md [``]: ./test.md +[``]: ./test/action-groups.md diff --git a/docs/suite.md b/docs/suite.md index 6232d2201..38e54926f 100644 --- a/docs/suite.md +++ b/docs/suite.md @@ -4,9 +4,10 @@ Suites are essentially groups of tests that run in specific conditions (precondi They enable including, excluding, and grouping tests for a customized test run. You can form suites using separate tests, groups, and modules. -Each suite must be defined in the `/dev/tests/acceptance/tests/_suite/suite.xml` file. -The generated tests for each suite go into a separate directory under `/dev/tests/acceptance/tests/functional/Magento/FunctionalTest/_generated/`. -By default, all generated tests are stored in the _default_ suite under `.../Magento/FunctionalTest/_generated/default/` +Each suite must be defined in the `//Test/Mftf/Suite` directory. + +The tests for each suite are generated in a separate directory under `/dev/tests/acceptance/tests/functional/Magento/FunctionalTest/_generated/`. +All tests that are not within a suite are generated in the _default_ suite at `.../Magento/FunctionalTest/_generated/default/`.
If a test is generated into at least one custom suite, it will not appear in the _default_ suite. @@ -60,8 +61,6 @@ The code lives in one place and executes once per suite. - Set up preconditions and postconditions using [actions] in [``] and [``] correspondingly, just similar to use in a [test]. - Clean up after suites just like after tests. The MFTF enforces the presence of both `` and `` if either is present. -- Do not reference in the subsequent tests to data that was persisted in the preconditions. - Referencing to `$stepKey.field$` of these actions is not valid. ## Test writing diff --git a/docs/test/actions.md b/docs/test/actions.md index 9b125d7fc..c5dc83fdb 100644 --- a/docs/test/actions.md +++ b/docs/test/actions.md @@ -1262,6 +1262,7 @@ Attribute|Type|Use|Description ---|---|---|--- `command`|string |optional| CLI command to be executed in Magento environment. `arguments`|string |optional| Unescaped arguments to be passed in with the CLI command. +`timeout`|string|optional| Number of seconds CLI command can run without outputting anything. `stepKey`|string|required| A unique identifier of the action. `before`|string|optional| `stepKey` of action that must be executed next. `after`|string|optional| `stepKey` of preceding action. diff --git a/etc/config/.env.example b/etc/config/.env.example index f25cb3de7..7320d8b8b 100644 --- a/etc/config/.env.example +++ b/etc/config/.env.example @@ -5,7 +5,7 @@ MAGENTO_BASE_URL=http://devdocs.magento.com/ #*** Uncomment if you are running Admin Panel on separate domain (used with MAGENTO_BACKEND_NAME) ***# -# MAGENTO_BACKEND_BASE_HOST=http://admin.example.com/ +# MAGENTO_BACKEND_BASE_URL=http://admin.example.com/ #*** Set the Admin Username and Password for your Magento instance ***# MAGENTO_BACKEND_NAME=admin @@ -53,5 +53,10 @@ MODULE_WHITELIST=Magento_Framework,ConfigurableProductWishlist,ConfigurableProdu #ALLOW_SKIPPED=true #*** Default timeout for wait actions -#WAIT_TIMEOUT=10 +#WAIT_TIMEOUT=30 + +#*** Uncomment and set to enable browser log entries on actions in Allure. Blacklist is used to filter logs of a specific "source" +#ENABLE_BROWSER_LOG=true +#BROWSER_LOG_BLACKLIST=other + #*** End of .env ***# diff --git a/etc/config/command.php b/etc/config/command.php index 7b45a2595..e3b8f1191 100644 --- a/etc/config/command.php +++ b/etc/config/command.php @@ -14,6 +14,7 @@ $tokenPassedIn = urldecode($_POST['token'] ?? ''); $command = urldecode($_POST['command'] ?? ''); $arguments = urldecode($_POST['arguments'] ?? ''); + $timeout = floatval(urldecode($_POST['timeout'] ?? 60)); // Token returned will be null if the token we passed in is invalid $tokenFromMagento = $tokenModel->loadByToken($tokenPassedIn)->getToken(); @@ -24,14 +25,17 @@ if ($valid) { $fullCommand = escapeshellcmd($magentoBinary . " $command" . " $arguments"); $process = new Symfony\Component\Process\Process($fullCommand); - $process->setIdleTimeout(60); + $process->setIdleTimeout($timeout); $process->setTimeout(0); $idleTimeout = false; try { $process->run(); $output = $process->getOutput(); if (!$process->isSuccessful()) { - $output = $process->getErrorOutput(); + $failureOutput = $process->getErrorOutput(); + if (!empty($failureOutput)) { + $output = $failureOutput; + } } if (empty($output)) { $output = "CLI did not return output."; diff --git a/etc/config/functional.suite.dist.yml b/etc/config/functional.suite.dist.yml index 12658515b..5487a3c99 100644 --- a/etc/config/functional.suite.dist.yml +++ b/etc/config/functional.suite.dist.yml @@ -27,7 +27,7 @@ modules: window_size: 1280x1024 username: "%MAGENTO_ADMIN_USERNAME%" password: "%MAGENTO_ADMIN_PASSWORD%" - pageload_timeout: 30 + pageload_timeout: "%WAIT_TIMEOUT%" host: "%SELENIUM_HOST%" port: "%SELENIUM_PORT%" protocol: "%SELENIUM_PROTOCOL%" diff --git a/src/Magento/FunctionalTestingFramework/Allure/AllureHelper.php b/src/Magento/FunctionalTestingFramework/Allure/AllureHelper.php index 0fde0e8e7..58bbc55b8 100644 --- a/src/Magento/FunctionalTestingFramework/Allure/AllureHelper.php +++ b/src/Magento/FunctionalTestingFramework/Allure/AllureHelper.php @@ -5,6 +5,7 @@ */ namespace Magento\FunctionalTestingFramework\Allure; +use Magento\FunctionalTestingFramework\Allure\Event\AddUniqueAttachmentEvent; use Yandex\Allure\Adapter\Allure; use Yandex\Allure\Adapter\Event\AddAttachmentEvent; @@ -19,7 +20,7 @@ class AllureHelper */ public static function addAttachmentToCurrentStep($data, $caption) { - Allure::lifecycle()->fire(new AddAttachmentEvent($data, $caption)); + Allure::lifecycle()->fire(new AddUniqueAttachmentEvent($data, $caption)); } /** @@ -38,8 +39,8 @@ public static function addAttachmentToLastStep($data, $caption) // Nothing to attach to; do not fire off allure event return; } - - $attachmentEvent = new AddAttachmentEvent($data, $caption); + + $attachmentEvent = new AddUniqueAttachmentEvent($data, $caption); $attachmentEvent->process($trueLastStep); } } diff --git a/src/Magento/FunctionalTestingFramework/Allure/Event/AddUniqueAttachmentEvent.php b/src/Magento/FunctionalTestingFramework/Allure/Event/AddUniqueAttachmentEvent.php new file mode 100644 index 000000000..fc4ff64c2 --- /dev/null +++ b/src/Magento/FunctionalTestingFramework/Allure/Event/AddUniqueAttachmentEvent.php @@ -0,0 +1,107 @@ +guessFileMimeType($filePath); + $this->type = $type; + } + + $fileExtension = $this->guessFileExtension($type); + + $fileSha1 = uniqid(sha1_file($filePath)); + $outputPath = parent::getOutputPath($fileSha1, $fileExtension); + if (!$this->copyFile($filePath, $outputPath)) { + throw new AllureException("Failed to copy attachment from $filePath to $outputPath."); + } + + return $this->getOutputFileName($fileSha1, $fileExtension); + } + + /** + * Copies file from one path to another. Wrapper for mocking in unit test. + * @param string $filePath + * @param string $outputPath + * @return boolean + */ + private function copyFile($filePath, $outputPath) + { + return copy($filePath, $outputPath); + } + + /** + * Copy of parent private function + * @param string $filePath + * @return string + */ + private function guessFileMimeType($filePath) + { + $type = MimeTypeGuesser::getInstance()->guess($filePath); + if (!isset($type)) { + return DEFAULT_MIME_TYPE; + } + return $type; + } + + /** + * Copy of parent private function + * @param string $mimeType + * @return string + */ + private function guessFileExtension($mimeType) + { + $candidate = ExtensionGuesser::getInstance()->guess($mimeType); + if (!isset($candidate)) { + return DEFAULT_FILE_EXTENSION; + } + return $candidate; + } + + /** + * Copy of parent private function + * @param string $sha1 + * @param string $extension + * @return string + */ + public function getOutputFileName($sha1, $extension) + { + return $sha1 . '-attachment.' . $extension; + } +} diff --git a/src/Magento/FunctionalTestingFramework/Config/FileResolver/Primary.php b/src/Magento/FunctionalTestingFramework/Config/FileResolver/Primary.php index 2376b3cf1..3a976944b 100644 --- a/src/Magento/FunctionalTestingFramework/Config/FileResolver/Primary.php +++ b/src/Magento/FunctionalTestingFramework/Config/FileResolver/Primary.php @@ -6,8 +6,10 @@ namespace Magento\FunctionalTestingFramework\Config\FileResolver; +use Magento\FunctionalTestingFramework\Exceptions\TestFrameworkException; use Magento\FunctionalTestingFramework\Util\Iterator\File; use Magento\FunctionalTestingFramework\Config\FileResolverInterface; +use Magento\FunctionalTestingFramework\Util\Path\FilePathFormatter; /** * Provides the list of global configuration files. @@ -54,6 +56,7 @@ private function getFilePaths($filename, $scope) * @param string $filename * @param string $scope * @return array + * @throws TestFrameworkException */ private function getPathPatterns($filename, $scope) { @@ -69,8 +72,9 @@ private function getPathPatterns($filename, $scope) $defaultPath . DIRECTORY_SEPARATOR . $scope . DIRECTORY_SEPARATOR . $filename, $defaultPath . DIRECTORY_SEPARATOR . $scope . DIRECTORY_SEPARATOR . '*' . DIRECTORY_SEPARATOR . $filename, - FW_BP . DIRECTORY_SEPARATOR . $scope . DIRECTORY_SEPARATOR . $filename, - FW_BP . DIRECTORY_SEPARATOR . $scope . DIRECTORY_SEPARATOR . '*' . DIRECTORY_SEPARATOR . $filename + FilePathFormatter::format(FW_BP) . $scope . DIRECTORY_SEPARATOR . $filename, + FilePathFormatter::format(FW_BP) . $scope . DIRECTORY_SEPARATOR . '*' . DIRECTORY_SEPARATOR + . $filename ]; } return str_replace(DIRECTORY_SEPARATOR, DIRECTORY_SEPARATOR, $patterns); diff --git a/src/Magento/FunctionalTestingFramework/Config/FileResolver/Root.php b/src/Magento/FunctionalTestingFramework/Config/FileResolver/Root.php index 3b0940b28..4ebb11942 100644 --- a/src/Magento/FunctionalTestingFramework/Config/FileResolver/Root.php +++ b/src/Magento/FunctionalTestingFramework/Config/FileResolver/Root.php @@ -7,7 +7,9 @@ namespace Magento\FunctionalTestingFramework\Config\FileResolver; use Magento\FunctionalTestingFramework\Config\FileResolverInterface; +use Magento\FunctionalTestingFramework\Exceptions\TestFrameworkException; use Magento\FunctionalTestingFramework\Util\Iterator\File; +use Magento\FunctionalTestingFramework\Util\Path\FilePathFormatter; class Root extends Module { @@ -20,12 +22,13 @@ class Root extends Module * @param string $filename * @param string $scope * @return array|\Iterator,\Countable + * @throws TestFrameworkException */ public function get($filename, $scope) { // first pick up the root level test suite dir $paths = glob( - TESTS_BP . DIRECTORY_SEPARATOR . self::ROOT_SUITE_DIR + FilePathFormatter::format(TESTS_BP) . self::ROOT_SUITE_DIR . DIRECTORY_SEPARATOR . $filename ); diff --git a/src/Magento/FunctionalTestingFramework/Config/SchemaLocator.php b/src/Magento/FunctionalTestingFramework/Config/SchemaLocator.php index a92f536a3..869c58373 100644 --- a/src/Magento/FunctionalTestingFramework/Config/SchemaLocator.php +++ b/src/Magento/FunctionalTestingFramework/Config/SchemaLocator.php @@ -6,6 +6,9 @@ namespace Magento\FunctionalTestingFramework\Config; +use Magento\FunctionalTestingFramework\Exceptions\TestFrameworkException; +use Magento\FunctionalTestingFramework\Util\Path\FilePathFormatter; + /** * Configuration schema locator. */ @@ -30,12 +33,14 @@ class SchemaLocator implements \Magento\FunctionalTestingFramework\Config\Schema * * @param string $schemaPath * @param string|null $perFileSchema + * @throws TestFrameworkException */ public function __construct($schemaPath, $perFileSchema = null) { - if (constant('FW_BP') && file_exists(FW_BP . DIRECTORY_SEPARATOR . $schemaPath)) { - $this->schemaPath = FW_BP . DIRECTORY_SEPARATOR . $schemaPath; - $this->perFileSchema = $perFileSchema === null ? null : FW_BP . DIRECTORY_SEPARATOR . $perFileSchema; + if (constant('FW_BP') && file_exists(FilePathFormatter::format(FW_BP) . $schemaPath)) { + $this->schemaPath = FilePathFormatter::format(FW_BP) . $schemaPath; + $this->perFileSchema = $perFileSchema === null ? null : FilePathFormatter::format(FW_BP) + . $perFileSchema; } else { $path = dirname(dirname(dirname(__DIR__))); $path = str_replace('\\', DIRECTORY_SEPARATOR, $path); diff --git a/src/Magento/FunctionalTestingFramework/Console/BaseGenerateCommand.php b/src/Magento/FunctionalTestingFramework/Console/BaseGenerateCommand.php index 87b203d66..0953e9d68 100644 --- a/src/Magento/FunctionalTestingFramework/Console/BaseGenerateCommand.php +++ b/src/Magento/FunctionalTestingFramework/Console/BaseGenerateCommand.php @@ -8,6 +8,9 @@ namespace Magento\FunctionalTestingFramework\Console; +use Magento\FunctionalTestingFramework\Exceptions\TestFrameworkException; +use Magento\FunctionalTestingFramework\Test\Handlers\TestObjectHandler; +use Magento\FunctionalTestingFramework\Util\Path\FilePathFormatter; use Symfony\Component\Console\Command\Command; use Symfony\Component\Console\Input\InputOption; use Symfony\Component\Console\Output\OutputInterface; @@ -56,10 +59,11 @@ protected function configure() * @param OutputInterface $output * @param bool $verbose * @return void + * @throws TestFrameworkException */ protected function removeGeneratedDirectory(OutputInterface $output, bool $verbose) { - $generatedDirectory = TESTS_MODULE_PATH . DIRECTORY_SEPARATOR . TestGenerator::GENERATED_DIR; + $generatedDirectory = FilePathFormatter::format(TESTS_MODULE_PATH) . TestGenerator::GENERATED_DIR; if (file_exists($generatedDirectory)) { DirSetupUtil::rmdirRecursive($generatedDirectory); @@ -75,7 +79,6 @@ protected function removeGeneratedDirectory(OutputInterface $output, bool $verbo * @return false|string * @throws \Magento\FunctionalTestingFramework\Exceptions\XmlException */ - protected function getTestAndSuiteConfiguration(array $tests) { $testConfiguration['tests'] = null; @@ -103,4 +106,72 @@ protected function getTestAndSuiteConfiguration(array $tests) $testConfigurationJson = json_encode($testConfiguration); return $testConfigurationJson; } + + /** + * Returns an array of test configuration to be used as an argument for generation of tests + * This function uses group or suite names for generation + * @return false|string + * @throws \Magento\FunctionalTestingFramework\Exceptions\XmlException + */ + protected function getGroupAndSuiteConfiguration(array $groupOrSuiteNames) + { + $result['tests'] = []; + $result['suites'] = []; + + $groups = []; + $suites = []; + + $allSuites = SuiteObjectHandler::getInstance()->getAllObjects(); + $testsInSuites = SuiteObjectHandler::getInstance()->getAllTestReferences(); + + foreach ($groupOrSuiteNames as $groupOrSuiteName) { + if (array_key_exists($groupOrSuiteName, $allSuites)) { + $suites[] = $groupOrSuiteName; + } else { + $groups[] = $groupOrSuiteName; + } + } + + foreach ($suites as $suite) { + $result['suites'][$suite] = []; + } + + foreach ($groups as $group) { + $testsInGroup = TestObjectHandler::getInstance()->getTestsByGroup($group); + + $testsInGroupAndNotInAnySuite = array_diff( + array_keys($testsInGroup), + array_keys($testsInSuites) + ); + + $testsInGroupAndInAnySuite = array_diff( + array_keys($testsInGroup), + $testsInGroupAndNotInAnySuite + ); + + foreach ($testsInGroupAndInAnySuite as $testInGroupAndInAnySuite) { + $suiteName = $testsInSuites[$testInGroupAndInAnySuite][0]; + if (array_search($suiteName, $suites) !== false) { + // Suite is already being called to run in its entirety, do not filter list + continue; + } + $result['suites'][$suiteName][] = $testInGroupAndInAnySuite; + } + + $result['tests'] = array_merge( + $result['tests'], + $testsInGroupAndNotInAnySuite + ); + } + + if (empty($result['tests'])) { + $result['tests'] = null; + } + if (empty($result['suites'])) { + $result['suites'] = null; + } + + $json = json_encode($result); + return $json; + } } diff --git a/src/Magento/FunctionalTestingFramework/Console/BuildProjectCommand.php b/src/Magento/FunctionalTestingFramework/Console/BuildProjectCommand.php index 5102a3fdf..02dfcda82 100644 --- a/src/Magento/FunctionalTestingFramework/Console/BuildProjectCommand.php +++ b/src/Magento/FunctionalTestingFramework/Console/BuildProjectCommand.php @@ -18,6 +18,7 @@ use Symfony\Component\Process\Process; use Magento\FunctionalTestingFramework\Util\Env\EnvProcessor; use Symfony\Component\Yaml\Yaml; +use Magento\FunctionalTestingFramework\Util\Path\FilePathFormatter; /** * Class BuildProjectCommand @@ -28,7 +29,6 @@ class BuildProjectCommand extends Command { const DEFAULT_YAML_INLINE_DEPTH = 10; - const CREDENTIALS_FILE_PATH = TESTS_BP . DIRECTORY_SEPARATOR . '.credentials.example'; /** * Env processor manages .env files. @@ -41,6 +41,7 @@ class BuildProjectCommand extends Command * Configures the current command. * * @return void + * @throws TestFrameworkException */ protected function configure() { @@ -52,7 +53,7 @@ protected function configure() InputOption::VALUE_NONE, 'upgrade existing MFTF tests according to last major release requirements' ); - $this->envProcessor = new EnvProcessor(TESTS_BP . DIRECTORY_SEPARATOR . '.env'); + $this->envProcessor = new EnvProcessor(FilePathFormatter::format(TESTS_BP) . '.env'); $env = $this->envProcessor->getEnv(); foreach ($env as $key => $value) { $this->addOption($key, null, InputOption::VALUE_REQUIRED, '', $value); @@ -65,8 +66,7 @@ protected function configure() * @param InputInterface $input * @param OutputInterface $output * @return void - * @throws \Symfony\Component\Console\Exception\LogicException - * + * @throws \Exception * @SuppressWarnings(PHPMD.UnusedLocalVariable) */ protected function execute(InputInterface $input, OutputInterface $output) @@ -109,7 +109,7 @@ function ($type, $buffer) use ($output) { if ($input->getOption('upgrade')) { $upgradeCommand = new UpgradeTestsCommand(); - $upgradeOptions = new ArrayInput(['path' => TESTS_MODULE_PATH]); + $upgradeOptions = new ArrayInput(['path' => FilePathFormatter::format(TESTS_MODULE_PATH)]); $upgradeCommand->run($upgradeOptions, $output); } } @@ -119,6 +119,7 @@ function ($type, $buffer) use ($output) { * * @param OutputInterface $output * @return void + * @throws TestFrameworkException */ private function generateConfigFiles(OutputInterface $output) { @@ -126,45 +127,48 @@ private function generateConfigFiles(OutputInterface $output) //Find travel path from codeception.yml to FW_BP $relativePath = $fileSystem->makePathRelative(FW_BP, TESTS_BP); - if (!$fileSystem->exists(TESTS_BP . DIRECTORY_SEPARATOR . 'codeception.yml')) { + if (!$fileSystem->exists(FilePathFormatter::format(TESTS_BP) . 'codeception.yml')) { // read in the codeception.yml file - $configDistYml = Yaml::parse(file_get_contents(realpath(FW_BP . "/etc/config/codeception.dist.yml"))); + $configDistYml = Yaml::parse(file_get_contents( + realpath(FilePathFormatter::format(FW_BP) . "etc/config/codeception.dist.yml") + )); $configDistYml['paths']['support'] = $relativePath . 'src/Magento/FunctionalTestingFramework'; $configDistYml['paths']['envs'] = $relativePath . 'etc/_envs'; $configYmlText = Yaml::dump($configDistYml, self::DEFAULT_YAML_INLINE_DEPTH); // dump output to new codeception.yml file - file_put_contents(TESTS_BP . DIRECTORY_SEPARATOR . 'codeception.yml', $configYmlText); + file_put_contents(FilePathFormatter::format(TESTS_BP) . 'codeception.yml', $configYmlText); $output->writeln("codeception.yml configuration successfully applied."); } - $output->writeln("codeception.yml applied to " . TESTS_BP . DIRECTORY_SEPARATOR . 'codeception.yml'); + $output->writeln("codeception.yml applied to " . FilePathFormatter::format(TESTS_BP) . 'codeception.yml'); // copy the functional suite yml, will only copy if there are differences between the template the destination $fileSystem->copy( - realpath(FW_BP . '/etc/config/functional.suite.dist.yml'), - TESTS_BP . DIRECTORY_SEPARATOR . 'tests' . DIRECTORY_SEPARATOR . 'functional.suite.yml' + realpath(FilePathFormatter::format(FW_BP) . 'etc/config/functional.suite.dist.yml'), + FilePathFormatter::format(TESTS_BP) . 'tests' . DIRECTORY_SEPARATOR . 'functional.suite.yml' ); $output->writeln('functional.suite.yml configuration successfully applied.'); $output->writeln("functional.suite.yml applied to " . - TESTS_BP . DIRECTORY_SEPARATOR . 'tests' . DIRECTORY_SEPARATOR . 'functional.suite.yml'); + FilePathFormatter::format(TESTS_BP) . 'tests' . DIRECTORY_SEPARATOR . 'functional.suite.yml'); $fileSystem->copy( - FW_BP . '/etc/config/.credentials.example', - self::CREDENTIALS_FILE_PATH + FilePathFormatter::format(FW_BP) . 'etc/config/.credentials.example', + FilePathFormatter::format(TESTS_BP) . '.credentials.example' ); // copy command.php into magento instance - if (MAGENTO_BP === FW_BP) { + if (FilePathFormatter::format(MAGENTO_BP, false) + === FilePathFormatter::format(FW_BP, false)) { $output->writeln('MFTF standalone detected, command.php copy not applied.'); } else { $fileSystem->copy( - realpath(FW_BP . '/etc/config/command.php'), - TESTS_BP . DIRECTORY_SEPARATOR . "utils" . DIRECTORY_SEPARATOR .'command.php' + realpath(FilePathFormatter::format(FW_BP) . 'etc/config/command.php'), + FilePathFormatter::format(TESTS_BP) . "utils" . DIRECTORY_SEPARATOR .'command.php' ); $output->writeln('command.php copied to ' . - TESTS_BP . DIRECTORY_SEPARATOR . "utils" . DIRECTORY_SEPARATOR .'command.php'); + FilePathFormatter::format(TESTS_BP) . "utils" . DIRECTORY_SEPARATOR .'command.php'); } // Remove and Create Log File diff --git a/src/Magento/FunctionalTestingFramework/Console/CleanProjectCommand.php b/src/Magento/FunctionalTestingFramework/Console/CleanProjectCommand.php index b2b2c01f3..641cfe1ed 100644 --- a/src/Magento/FunctionalTestingFramework/Console/CleanProjectCommand.php +++ b/src/Magento/FunctionalTestingFramework/Console/CleanProjectCommand.php @@ -7,6 +7,8 @@ namespace Magento\FunctionalTestingFramework\Console; +use Magento\FunctionalTestingFramework\Exceptions\TestFrameworkException; +use Magento\FunctionalTestingFramework\Util\Path\FilePathFormatter; use Symfony\Component\Console\Command\Command; use Symfony\Component\Console\Input\InputInterface; use Symfony\Component\Console\Input\InputOption; @@ -16,21 +18,6 @@ class CleanProjectCommand extends Command { - const CONFIGURATION_FILES = [ - // codeception.yml file for top level config - TESTS_BP . DIRECTORY_SEPARATOR . 'codeception.yml', - // functional.suite.yml for test execution config - TESTS_BP . DIRECTORY_SEPARATOR . 'tests' . DIRECTORY_SEPARATOR . 'functional.suite.yml', - // Acceptance Tester Actions generated by codeception - FW_BP . '/src/Magento/FunctionalTestingFramework/_generated', - // AcceptanceTester Class generated by codeception - FW_BP . '/src/Magento/FunctionalTestingFramework/AcceptanceTester.php' - ]; - - const GENERATED_FILES = [ - TESTS_MODULE_PATH . '/_generated' - ]; - /** * Configures the current command. * @@ -52,22 +39,40 @@ protected function configure() * @param OutputInterface $output * @return void * @throws \Symfony\Component\Console\Exception\LogicException + * @throws TestFrameworkException */ protected function execute(InputInterface $input, OutputInterface $output) { + $configFiles = [ + // codeception.yml file for top level config + FilePathFormatter::format(TESTS_BP) . 'codeception.yml', + // functional.suite.yml for test execution config + FilePathFormatter::format(TESTS_BP) . 'tests' . DIRECTORY_SEPARATOR . 'functional.suite.yml', + // Acceptance Tester Actions generated by codeception + FilePathFormatter::format(FW_BP) . 'src/Magento/FunctionalTestingFramework/_generated', + // AcceptanceTester Class generated by codeception + FilePathFormatter::format(FW_BP) . 'src/Magento/FunctionalTestingFramework/AcceptanceTester.php' + ]; + + $generatedFiles = [ + FilePathFormatter::format(TESTS_MODULE_PATH) . '_generated' + ]; + $isHardReset = $input->getOption('hard'); $fileSystem = new Filesystem(); $finder = new Finder(); - $finder->files()->name('*.php')->in(realpath(FW_BP . '/src/Magento/FunctionalTestingFramework/Group/')); + $finder->files()->name('*.php')->in( + realpath(FilePathFormatter::format(FW_BP) . 'src/Magento/FunctionalTestingFramework/Group/') + ); $filesForRemoval = []; // include config files if user specifies a hard reset for deletion if ($isHardReset) { - $filesForRemoval = array_merge($filesForRemoval, self::CONFIGURATION_FILES); + $filesForRemoval = array_merge($filesForRemoval, $configFiles); } // include the files mftf generates during test execution in TESTS_BP for deletion - $filesForRemoval = array_merge($filesForRemoval, self::GENERATED_FILES); + $filesForRemoval = array_merge($filesForRemoval, $generatedFiles); if ($output->isVerbose()) { $output->writeln('Deleting Files:'); diff --git a/src/Magento/FunctionalTestingFramework/Console/CommandList.php b/src/Magento/FunctionalTestingFramework/Console/CommandList.php index 34d221840..bf9cbd58e 100644 --- a/src/Magento/FunctionalTestingFramework/Console/CommandList.php +++ b/src/Magento/FunctionalTestingFramework/Console/CommandList.php @@ -29,19 +29,20 @@ class CommandList implements CommandListInterface public function __construct(array $commands = []) { $this->commands = [ - 'build:project' => new BuildProjectCommand(), - 'reset' => new CleanProjectCommand(), - 'generate:urn-catalog' => new GenerateDevUrnCommand(), - 'generate:suite' => new GenerateSuiteCommand(), - 'generate:tests' => new GenerateTestsCommand(), - 'run:test' => new RunTestCommand(), - 'run:group' => new RunTestGroupCommand(), - 'run:failed' => new RunTestFailedCommand(), - 'run:manifest' => new RunManifestCommand(), - 'setup:env' => new SetupEnvCommand(), - 'upgrade:tests' => new UpgradeTestsCommand(), - 'generate:docs' => new GenerateDocsCommand(), - 'static-checks' => new StaticChecksCommand() + 'build:project' => new BuildProjectCommand(), + 'doctor' => new DoctorCommand(), + 'generate:docs' => new GenerateDocsCommand(), + 'generate:suite' => new GenerateSuiteCommand(), + 'generate:tests' => new GenerateTestsCommand(), + 'generate:urn-catalog' => new GenerateDevUrnCommand(), + 'reset' => new CleanProjectCommand(), + 'run:failed' => new RunTestFailedCommand(), + 'run:group' => new RunTestGroupCommand(), + 'run:manifest' => new RunManifestCommand(), + 'run:test' => new RunTestCommand(), + 'setup:env' => new SetupEnvCommand(), + 'static-checks' => new StaticChecksCommand(), + 'upgrade:tests' => new UpgradeTestsCommand(), ] + $commands; } diff --git a/src/Magento/FunctionalTestingFramework/Console/DoctorCommand.php b/src/Magento/FunctionalTestingFramework/Console/DoctorCommand.php new file mode 100644 index 000000000..6fc9afa55 --- /dev/null +++ b/src/Magento/FunctionalTestingFramework/Console/DoctorCommand.php @@ -0,0 +1,210 @@ +setName('doctor') + ->setDescription( + 'This command checks environment readiness for generating and running MFTF tests.' + ); + } + + /** + * Executes the current command. + * + * @param InputInterface $input + * @param OutputInterface $output + * @return integer + * @throws TestFrameworkException + * @SuppressWarnings(PHPMD.CyclomaticComplexity) + */ + protected function execute(InputInterface $input, OutputInterface $output) + { + // For output style + $this->ioStyle = new SymfonyStyle($input, $output); + + $cmdStatus = true; + + // Config application + $verbose = $output->isVerbose(); + MftfApplicationConfig::create( + false, + MftfApplicationConfig::GENERATION_PHASE, + $verbose, + MftfApplicationConfig::LEVEL_DEVELOPER, + false + ); + + // Check authentication to Magento Admin + $status = $this->checkAuthenticationToMagentoAdmin(); + $cmdStatus = $cmdStatus && !$status ? false : $cmdStatus; + + // Check connection to Selenium + $status = $this->checkContextOnStep( + MagentoWebDriverDoctor::EXCEPTION_CONTEXT_SELENIUM, + 'Connecting to Selenium Server' + ); + $cmdStatus = $cmdStatus && !$status ? false : $cmdStatus; + + // Check opening Magento Admin in web browser + $status = $this->checkContextOnStep( + MagentoWebDriverDoctor::EXCEPTION_CONTEXT_ADMIN, + 'Loading Admin page' + ); + $cmdStatus = $cmdStatus && !$status ? false : $cmdStatus; + + // Check opening Magento Storefront in web browser + $status = $this->checkContextOnStep( + MagentoWebDriverDoctor::EXCEPTION_CONTEXT_STOREFRONT, + 'Loading Storefront page' + ); + $cmdStatus = $cmdStatus && !$status ? false : $cmdStatus; + + // Check access to Magento CLI + $status = $this->checkContextOnStep( + MagentoWebDriverDoctor::EXCEPTION_CONTEXT_CLI, + 'Running Magento CLI' + ); + $cmdStatus = $cmdStatus && !$status ? false : $cmdStatus; + + return $cmdStatus ? 0 : 1; + } + + /** + * Check admin account authentication + * + * @return boolean + */ + private function checkAuthenticationToMagentoAdmin() + { + $result = false; + try { + $this->ioStyle->text("Requesting API token for admin user through cURL ..."); + ModuleResolver::getInstance()->getAdminToken(); + $this->ioStyle->success('Successful'); + $result = true; + } catch (TestFrameworkException $e) { + if (getenv('MAGENTO_BACKEND_BASE_URL')) { + $urlVar = 'MAGENTO_BACKEND_BASE_URL'; + } else { + $urlVar = 'MAGENTO_BASE_URL'; + } + $this->ioStyle->error( + $e->getMessage() . "\nPlease verify if " . $urlVar . ", " + . "MAGENTO_ADMIN_USERNAME and MAGENTO_ADMIN_PASSWORD in .env are valid." + ); + } + return $result; + } + + /** + * Check exception context after runMagentoWebDriverDoctor + * + * @param string $exceptionType + * @param string $message + * @return boolean + * @throws TestFrameworkException + */ + private function checkContextOnStep($exceptionType, $message) + { + $this->ioStyle->text($message . ' ...'); + $this->runMagentoWebDriverDoctor(); + + if (isset($this->context[$exceptionType])) { + $this->ioStyle->error($this->context[$exceptionType]); + return false; + } else { + $this->ioStyle->success('Successful'); + return true; + } + } + + /** + * Run diagnose through MagentoWebDriverDoctor + * + * @return void + * @throws TestFrameworkException + */ + private function runMagentoWebDriverDoctor() + { + if (!empty($this->context)) { + return; + } + + $magentoWebDriver = '\\' . MagentoWebDriver::class; + $magentoWebDriverDoctor = '\\' . MagentoWebDriverDoctor::class; + + require_once realpath(self::CODECEPTION_AUTOLOAD_FILE); + + $config = Configuration::config(realpath(self::MFTF_CODECEPTION_CONFIG_FILE)); + $settings = Configuration::suiteSettings(self::SUITE, $config); + + // Enable MagentoWebDriverDoctor + $settings['modules']['enabled'][] = $magentoWebDriverDoctor; + $settings['modules']['config'][$magentoWebDriverDoctor] = + $settings['modules']['config'][$magentoWebDriver]; + + // Disable MagentoWebDriver to avoid conflicts + foreach ($settings['modules']['enabled'] as $index => $module) { + if ($module == $magentoWebDriver) { + unset($settings['modules']['enabled'][$index]); + break; + } + } + unset($settings['modules']['config'][$magentoWebDriver]); + + $dispatcher = new EventDispatcher(); + $suiteManager = new SuiteManager($dispatcher, self::SUITE, $settings); + try { + $suiteManager->initialize(); + $this->context = ['Successful']; + } catch (TestFrameworkException $e) { + $this->context = $e->getContext(); + } + } +} diff --git a/src/Magento/FunctionalTestingFramework/Console/GenerateDevUrnCommand.php b/src/Magento/FunctionalTestingFramework/Console/GenerateDevUrnCommand.php index 1147704c0..f953deb16 100644 --- a/src/Magento/FunctionalTestingFramework/Console/GenerateDevUrnCommand.php +++ b/src/Magento/FunctionalTestingFramework/Console/GenerateDevUrnCommand.php @@ -9,6 +9,7 @@ namespace Magento\FunctionalTestingFramework\Console; use Magento\FunctionalTestingFramework\Exceptions\TestFrameworkException; +use Magento\FunctionalTestingFramework\Util\Path\FilePathFormatter; use Symfony\Component\Console\Command\Command; use Symfony\Component\Console\Input\InputArgument; use Symfony\Component\Console\Input\InputInterface; @@ -109,24 +110,32 @@ protected function execute(InputInterface $input, OutputInterface $output) /** * Generates urn => location array for all MFTF schema. * @return array + * @throws TestFrameworkException */ private function generateResourcesArray() { $resourcesArray = [ 'urn:magento:mftf:DataGenerator/etc/dataOperation.xsd' => - realpath(FW_BP . '/src/Magento/FunctionalTestingFramework/DataGenerator/etc/dataOperation.xsd'), + realpath(FilePathFormatter::format(FW_BP) + . 'src/Magento/FunctionalTestingFramework/DataGenerator/etc/dataOperation.xsd'), 'urn:magento:mftf:DataGenerator/etc/dataProfileSchema.xsd' => - realpath(FW_BP . '/src/Magento/FunctionalTestingFramework/DataGenerator/etc/dataProfileSchema.xsd'), + realpath(FilePathFormatter::format(FW_BP) + . 'src/Magento/FunctionalTestingFramework/DataGenerator/etc/dataProfileSchema.xsd'), 'urn:magento:mftf:Page/etc/PageObject.xsd' => - realpath(FW_BP . '/src/Magento/FunctionalTestingFramework/Page/etc/PageObject.xsd'), + realpath(FilePathFormatter::format(FW_BP) + . 'src/Magento/FunctionalTestingFramework/Page/etc/PageObject.xsd'), 'urn:magento:mftf:Page/etc/SectionObject.xsd' => - realpath(FW_BP . '/src/Magento/FunctionalTestingFramework/Page/etc/SectionObject.xsd'), + realpath(FilePathFormatter::format(FW_BP) + . 'src/Magento/FunctionalTestingFramework/Page/etc/SectionObject.xsd'), 'urn:magento:mftf:Test/etc/actionGroupSchema.xsd' => - realpath(FW_BP . '/src/Magento/FunctionalTestingFramework/Test/etc/actionGroupSchema.xsd'), + realpath(FilePathFormatter::format(FW_BP) + . 'src/Magento/FunctionalTestingFramework/Test/etc/actionGroupSchema.xsd'), 'urn:magento:mftf:Test/etc/testSchema.xsd' => - realpath(FW_BP . '/src/Magento/FunctionalTestingFramework/Test/etc/testSchema.xsd'), + realpath(FilePathFormatter::format(FW_BP) + . 'src/Magento/FunctionalTestingFramework/Test/etc/testSchema.xsd'), 'urn:magento:mftf:Suite/etc/suiteSchema.xsd' => - realpath(FW_BP . '/src/Magento/FunctionalTestingFramework/Suite/etc/suiteSchema.xsd') + realpath(FilePathFormatter::format(FW_BP) + . 'src/Magento/FunctionalTestingFramework/Suite/etc/suiteSchema.xsd') ]; return $resourcesArray; } diff --git a/src/Magento/FunctionalTestingFramework/Console/RunManifestCommand.php b/src/Magento/FunctionalTestingFramework/Console/RunManifestCommand.php index e487d16cf..37a90bb77 100644 --- a/src/Magento/FunctionalTestingFramework/Console/RunManifestCommand.php +++ b/src/Magento/FunctionalTestingFramework/Console/RunManifestCommand.php @@ -13,6 +13,7 @@ use Symfony\Component\Console\Input\InputInterface; use Symfony\Component\Console\Output\OutputInterface; use Symfony\Component\Process\Process; +use Magento\FunctionalTestingFramework\Util\Path\FilePathFormatter; class RunManifestCommand extends Command { @@ -31,6 +32,13 @@ class RunManifestCommand extends Command */ private $failedTests = []; + /** + * Path for a failed test + * + * @var string + */ + private $testsFailedFile; + /** * Configure the run:manifest command. * @@ -53,6 +61,14 @@ protected function configure() */ protected function execute(InputInterface $input, OutputInterface $output): int { + $testsOutputDir = FilePathFormatter::format(TESTS_BP) . + "tests" . + DIRECTORY_SEPARATOR . + "_output" . + DIRECTORY_SEPARATOR; + + $this->testsFailedFile = $testsOutputDir . "failed"; + $path = $input->getArgument("path"); if (!file_exists($path)) { @@ -117,8 +133,8 @@ private function runManifestLine(string $manifestLine, OutputInterface $output) */ private function aggregateFailed() { - if (file_exists(RunTestFailedCommand::TESTS_FAILED_FILE)) { - $currentFile = file(RunTestFailedCommand::TESTS_FAILED_FILE, FILE_IGNORE_NEW_LINES); + if (file_exists($this->testsFailedFile)) { + $currentFile = file($this->testsFailedFile, FILE_IGNORE_NEW_LINES); $this->failedTests = array_merge( $this->failedTests, $currentFile @@ -133,8 +149,8 @@ private function aggregateFailed() */ private function deleteFailedFile() { - if (file_exists(RunTestFailedCommand::TESTS_FAILED_FILE)) { - unlink(RunTestFailedCommand::TESTS_FAILED_FILE); + if (file_exists($this->testsFailedFile)) { + unlink($this->testsFailedFile); } } @@ -146,7 +162,7 @@ private function deleteFailedFile() private function writeFailedFile() { foreach ($this->failedTests as $test) { - file_put_contents(RunTestFailedCommand::TESTS_FAILED_FILE, $test . "\n", FILE_APPEND); + file_put_contents($this->testsFailedFile, $test . "\n", FILE_APPEND); } } } diff --git a/src/Magento/FunctionalTestingFramework/Console/RunTestCommand.php b/src/Magento/FunctionalTestingFramework/Console/RunTestCommand.php index 77b8be513..373256cfc 100644 --- a/src/Magento/FunctionalTestingFramework/Console/RunTestCommand.php +++ b/src/Magento/FunctionalTestingFramework/Console/RunTestCommand.php @@ -8,6 +8,7 @@ namespace Magento\FunctionalTestingFramework\Console; use Magento\FunctionalTestingFramework\Config\MftfApplicationConfig; +use Magento\FunctionalTestingFramework\Util\Path\FilePathFormatter; use Magento\FunctionalTestingFramework\Util\TestGenerator; use Symfony\Component\Console\Input\ArrayInput; use Symfony\Component\Console\Input\InputArgument; @@ -122,8 +123,7 @@ protected function execute(InputInterface $input, OutputInterface $output): int private function runTests(array $tests, OutputInterface $output) { $codeceptionCommand = realpath(PROJECT_ROOT . '/vendor/bin/codecept') . ' run functional '; - $testsDirectory = TESTS_MODULE_PATH . - DIRECTORY_SEPARATOR . + $testsDirectory = FilePathFormatter::format(TESTS_MODULE_PATH) . TestGenerator::GENERATED_DIR . DIRECTORY_SEPARATOR . TestGenerator::DEFAULT_DIR . diff --git a/src/Magento/FunctionalTestingFramework/Console/RunTestFailedCommand.php b/src/Magento/FunctionalTestingFramework/Console/RunTestFailedCommand.php index 336fd5d87..5f0596a7f 100644 --- a/src/Magento/FunctionalTestingFramework/Console/RunTestFailedCommand.php +++ b/src/Magento/FunctionalTestingFramework/Console/RunTestFailedCommand.php @@ -8,6 +8,7 @@ namespace Magento\FunctionalTestingFramework\Console; use Magento\FunctionalTestingFramework\Config\MftfApplicationConfig; +use Magento\FunctionalTestingFramework\Util\Path\FilePathFormatter; use Symfony\Component\Console\Input\ArrayInput; use Symfony\Component\Console\Input\InputInterface; use Symfony\Component\Console\Output\OutputInterface; @@ -22,20 +23,20 @@ class RunTestFailedCommand extends BaseGenerateCommand */ const DEFAULT_TEST_GROUP = 'default'; - const TESTS_OUTPUT_DIR = TESTS_BP . - DIRECTORY_SEPARATOR . - "tests" . - DIRECTORY_SEPARATOR . - "_output" . - DIRECTORY_SEPARATOR; - - const TESTS_FAILED_FILE = self::TESTS_OUTPUT_DIR . "failed"; - const TESTS_RERUN_FILE = self::TESTS_OUTPUT_DIR . "rerun_tests"; - const TESTS_MANIFEST_FILE= TESTS_MODULE_PATH . - DIRECTORY_SEPARATOR . - "_generated" . - DIRECTORY_SEPARATOR . - "testManifest.txt"; + /** + * @var string + */ + private $testsFailedFile; + + /** + * @var string + */ + private $testsReRunFile; + + /** + * @var string + */ + private $testsManifestFile; /** * @var array @@ -68,6 +69,19 @@ protected function configure() */ protected function execute(InputInterface $input, OutputInterface $output): int { + $testsOutputDir = FilePathFormatter::format(TESTS_BP) . + "tests" . + DIRECTORY_SEPARATOR . + "_output" . + DIRECTORY_SEPARATOR; + + $this->testsFailedFile = $testsOutputDir . "failed"; + $this->testsReRunFile = $testsOutputDir . "rerun_tests"; + $this->testsManifestFile= FilePathFormatter::format(TESTS_MODULE_PATH) . + "_generated" . + DIRECTORY_SEPARATOR . + "testManifest.txt"; + $force = $input->getOption('force'); $debug = $input->getOption('debug') ?? MftfApplicationConfig::LEVEL_DEVELOPER; // for backward compatibility $allowSkipped = $input->getOption('allow-skipped'); @@ -115,15 +129,15 @@ function ($type, $buffer) use ($output) { $output->write($buffer); } )); - if (file_exists(self::TESTS_FAILED_FILE)) { + if (file_exists($this->testsFailedFile)) { $this->failedList = array_merge( $this->failedList, - $this->readFailedTestFile(self::TESTS_FAILED_FILE) + $this->readFailedTestFile($this->testsFailedFile) ); } } foreach ($this->failedList as $test) { - $this->writeFailedTestToFile($test, self::TESTS_FAILED_FILE); + $this->writeFailedTestToFile($test, $this->testsFailedFile); } return $returnCode; @@ -138,12 +152,12 @@ private function getFailedTestList() { $failedTestDetails = ['tests' => [], 'suites' => []]; - if (realpath(self::TESTS_FAILED_FILE)) { - $testList = $this->readFailedTestFile(self::TESTS_FAILED_FILE); + if (realpath($this->testsFailedFile)) { + $testList = $this->readFailedTestFile($this->testsFailedFile); foreach ($testList as $test) { if (!empty($test)) { - $this->writeFailedTestToFile($test, self::TESTS_RERUN_FILE); + $this->writeFailedTestToFile($test, $this->testsReRunFile); $testInfo = explode(DIRECTORY_SEPARATOR, $test); $testName = explode(":", $testInfo[count($testInfo) - 1])[1]; $suiteName = $testInfo[count($testInfo) - 2]; @@ -184,7 +198,7 @@ private function getFailedTestList() */ private function readTestManifestFile() { - return file(self::TESTS_MANIFEST_FILE, FILE_IGNORE_NEW_LINES); + return file($this->testsManifestFile, FILE_IGNORE_NEW_LINES); } /** @@ -202,6 +216,7 @@ private function readFailedTestFile($filePath) * Writes the test name to a file if it does not already exist * * @param string $test + * @param string $filePath * @return void */ private function writeFailedTestToFile($test, $filePath) diff --git a/src/Magento/FunctionalTestingFramework/Console/RunTestGroupCommand.php b/src/Magento/FunctionalTestingFramework/Console/RunTestGroupCommand.php index 7f954c8fe..98d121d40 100644 --- a/src/Magento/FunctionalTestingFramework/Console/RunTestGroupCommand.php +++ b/src/Magento/FunctionalTestingFramework/Console/RunTestGroupCommand.php @@ -111,32 +111,4 @@ function ($type, $buffer) use ($output) { } ); } - - /** - * Returns a json string to be used as an argument for generation of a group or suite - * - * @param array $groups - * @return string - * @throws \Magento\FunctionalTestingFramework\Exceptions\XmlException - */ - private function getGroupAndSuiteConfiguration(array $groups) - { - $testConfiguration['tests'] = []; - $testConfiguration['suites'] = null; - $availableSuites = SuiteObjectHandler::getInstance()->getAllObjects(); - - foreach ($groups as $group) { - if (array_key_exists($group, $availableSuites)) { - $testConfiguration['suites'][$group] = []; - } - - $testConfiguration['tests'] = array_merge( - $testConfiguration['tests'], - array_keys(TestObjectHandler::getInstance()->getTestsByGroup($group)) - ); - } - - $testConfigurationJson = json_encode($testConfiguration); - return $testConfigurationJson; - } } diff --git a/src/Magento/FunctionalTestingFramework/Console/SetupEnvCommand.php b/src/Magento/FunctionalTestingFramework/Console/SetupEnvCommand.php index 4d1ce0334..65e8c61e7 100644 --- a/src/Magento/FunctionalTestingFramework/Console/SetupEnvCommand.php +++ b/src/Magento/FunctionalTestingFramework/Console/SetupEnvCommand.php @@ -7,6 +7,8 @@ namespace Magento\FunctionalTestingFramework\Console; +use Magento\FunctionalTestingFramework\Exceptions\TestFrameworkException; +use Magento\FunctionalTestingFramework\Util\Path\FilePathFormatter; use Symfony\Component\Console\Command\Command; use Symfony\Component\Console\Input\InputOption; use Symfony\Component\Console\Exception\InvalidOptionException; @@ -27,12 +29,13 @@ class SetupEnvCommand extends Command * Configures the current command. * * @return void + * @throws TestFrameworkException */ protected function configure() { $this->setName('setup:env') ->setDescription("Generate .env file."); - $this->envProcessor = new EnvProcessor(TESTS_BP . DIRECTORY_SEPARATOR . '.env'); + $this->envProcessor = new EnvProcessor(FilePathFormatter::format(TESTS_BP) . '.env'); $env = $this->envProcessor->getEnv(); foreach ($env as $key => $value) { $this->addOption($key, null, InputOption::VALUE_REQUIRED, '', $value); diff --git a/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/CredentialStore.php b/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/CredentialStore.php index 0514cfe57..94ff40069 100644 --- a/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/CredentialStore.php +++ b/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/CredentialStore.php @@ -9,6 +9,7 @@ use Magento\FunctionalTestingFramework\Exceptions\TestFrameworkException; use Magento\FunctionalTestingFramework\DataGenerator\Handlers\SecretStorage\FileStorage; use Magento\FunctionalTestingFramework\DataGenerator\Handlers\SecretStorage\VaultStorage; +use Magento\FunctionalTestingFramework\Util\Path\UrlFormatter; class CredentialStore { @@ -63,7 +64,7 @@ private function __construct() if ($cvAddress !== false && $cvSecretPath !== false) { try { $this->credStorage[self::ARRAY_KEY_FOR_VAULT] = new VaultStorage( - rtrim($cvAddress, '/'), + UrlFormatter::format($cvAddress, false), '/' . trim($cvSecretPath, '/') ); } catch (TestFrameworkException $e) { diff --git a/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/SecretStorage/FileStorage.php b/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/SecretStorage/FileStorage.php index 064610c79..be77a6de2 100644 --- a/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/SecretStorage/FileStorage.php +++ b/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/SecretStorage/FileStorage.php @@ -6,10 +6,10 @@ namespace Magento\FunctionalTestingFramework\DataGenerator\Handlers\SecretStorage; -use Magento\FunctionalTestingFramework\Console\BuildProjectCommand; use Magento\FunctionalTestingFramework\Exceptions\TestFrameworkException; use Magento\FunctionalTestingFramework\Config\MftfApplicationConfig; use Magento\FunctionalTestingFramework\Util\Logger\LoggingUtil; +use Magento\FunctionalTestingFramework\Util\Path\FilePathFormatter; class FileStorage extends BaseStorage { @@ -71,7 +71,7 @@ private function readInCredentialsFile() $credsFilePath = str_replace( '.credentials.example', '.credentials', - BuildProjectCommand::CREDENTIALS_FILE_PATH + FilePathFormatter::format(TESTS_BP) . '.credentials.example' ); if (!file_exists($credsFilePath)) { diff --git a/src/Magento/FunctionalTestingFramework/DataGenerator/Persist/Curl/AbstractExecutor.php b/src/Magento/FunctionalTestingFramework/DataGenerator/Persist/Curl/AbstractExecutor.php index b6c3f29ec..e27da3718 100644 --- a/src/Magento/FunctionalTestingFramework/DataGenerator/Persist/Curl/AbstractExecutor.php +++ b/src/Magento/FunctionalTestingFramework/DataGenerator/Persist/Curl/AbstractExecutor.php @@ -6,6 +6,8 @@ namespace Magento\FunctionalTestingFramework\DataGenerator\Persist\Curl; +use Magento\FunctionalTestingFramework\Exceptions\TestFrameworkException; +use Magento\FunctionalTestingFramework\Util\Path\UrlFormatter; use Magento\FunctionalTestingFramework\Util\Protocol\CurlInterface; /** @@ -23,9 +25,10 @@ abstract class AbstractExecutor implements CurlInterface /** * Returns base URL for Magento instance * @return string + * @throws TestFrameworkException */ public function getBaseUrl(): string { - return getenv('MAGENTO_BASE_URL'); + return UrlFormatter::format(getenv('MAGENTO_BASE_URL')); } } diff --git a/src/Magento/FunctionalTestingFramework/DataGenerator/Persist/Curl/AdminExecutor.php b/src/Magento/FunctionalTestingFramework/DataGenerator/Persist/Curl/AdminExecutor.php index 4e11f5cec..86b3b742f 100644 --- a/src/Magento/FunctionalTestingFramework/DataGenerator/Persist/Curl/AdminExecutor.php +++ b/src/Magento/FunctionalTestingFramework/DataGenerator/Persist/Curl/AdminExecutor.php @@ -6,6 +6,7 @@ namespace Magento\FunctionalTestingFramework\DataGenerator\Persist\Curl; +use Magento\FunctionalTestingFramework\Util\Path\UrlFormatter; use Magento\FunctionalTestingFramework\Util\Protocol\CurlInterface; use Magento\FunctionalTestingFramework\Util\Protocol\CurlTransport; use Magento\FunctionalTestingFramework\Exceptions\TestFrameworkException; @@ -59,11 +60,20 @@ public function __construct($removeBackend) /** * Returns base URL for Magento backend instance * @return string + * @throws TestFrameworkException */ public function getBaseUrl(): string { - $backendHost = getenv('MAGENTO_BACKEND_BASE_URL') ?: parent::getBaseUrl(); - return $backendHost . getenv('MAGENTO_BACKEND_NAME') . '/'; + $backendHost = getenv('MAGENTO_BACKEND_BASE_URL') + ? + UrlFormatter::format(getenv('MAGENTO_BACKEND_BASE_URL')) + : + parent::getBaseUrl(); + return empty(getenv('MAGENTO_BACKEND_NAME')) + ? + $backendHost + : + $backendHost . getenv('MAGENTO_BACKEND_NAME') . '/'; } /** diff --git a/src/Magento/FunctionalTestingFramework/DataGenerator/Persist/Curl/WebapiExecutor.php b/src/Magento/FunctionalTestingFramework/DataGenerator/Persist/Curl/WebapiExecutor.php index 281eaa24d..494ce8f97 100644 --- a/src/Magento/FunctionalTestingFramework/DataGenerator/Persist/Curl/WebapiExecutor.php +++ b/src/Magento/FunctionalTestingFramework/DataGenerator/Persist/Curl/WebapiExecutor.php @@ -7,6 +7,7 @@ namespace Magento\FunctionalTestingFramework\DataGenerator\Persist\Curl; use Magento\FunctionalTestingFramework\Exceptions\TestFrameworkException; +use Magento\FunctionalTestingFramework\Util\Path\UrlFormatter; use Magento\FunctionalTestingFramework\Util\Protocol\CurlInterface; use Magento\FunctionalTestingFramework\Util\Protocol\CurlTransport; @@ -75,24 +76,32 @@ public function __construct($storeCode = null) /** * Returns base URL for Magento Web API instance * @return string + * @throws TestFrameworkException */ public function getBaseUrl(): string { - $baseUrl = parent::getBaseUrl(); - $webapiHost = getenv('MAGENTO_RESTAPI_SERVER_HOST'); $webapiPort = getenv("MAGENTO_RESTAPI_SERVER_PORT"); $webapiProtocol = getenv("MAGENTO_RESTAPI_SERVER_PROTOCOL"); - if ($webapiHost) { - $baseUrl = sprintf('%s://%s/', $webapiProtocol, $webapiHost); + if ($webapiHost && $webapiProtocol) { + $baseUrl = UrlFormatter::format( + sprintf('%s://%s', $webapiProtocol, $webapiHost), + false + ); + } elseif ($webapiHost) { + $baseUrl = UrlFormatter::format(sprintf('%s', $webapiProtocol, $webapiHost), false); + } + + if (!isset($baseUrl)) { + $baseUrl = rtrim(parent::getBaseUrl(), '/'); } if ($webapiPort) { - $baseUrl = rtrim($baseUrl, '/') . ':' . $webapiPort . '/'; + $baseUrl .= ':' . $webapiPort; } - return $baseUrl; + return $baseUrl . '/'; } /** @@ -175,22 +184,23 @@ public function close() * Builds and returns URL for request, appending storeCode if needed. * @param string $resource * @return string + * @throws TestFrameworkException */ public function getFormattedUrl($resource) { $urlResult = $this->getBaseUrl() . 'rest/'; if ($this->storeCode != null) { - $urlResult .= $this->storeCode . "/"; + $urlResult .= $this->storeCode . '/'; } - $urlResult .= trim($resource, "/"); + $urlResult .= trim($resource, '/'); return $urlResult; } /** * Return admin auth token. * - * @throws TestFrameworkException * @return string + * @throws TestFrameworkException */ public function getAuthToken() { diff --git a/src/Magento/FunctionalTestingFramework/DataGenerator/Persist/CurlHandler.php b/src/Magento/FunctionalTestingFramework/DataGenerator/Persist/CurlHandler.php index 691ce3606..16584e7a3 100644 --- a/src/Magento/FunctionalTestingFramework/DataGenerator/Persist/CurlHandler.php +++ b/src/Magento/FunctionalTestingFramework/DataGenerator/Persist/CurlHandler.php @@ -123,6 +123,8 @@ public function executeRequest($dependentEntities) $returnRegex = $this->operationDefinition->getReturnRegex(); $returnIndex = $this->operationDefinition->getReturnIndex(); $method = $this->operationDefinition->getApiMethod(); + AllureHelper::addAttachmentToLastStep($apiUrl, 'API Endpoint'); + AllureHelper::addAttachmentToLastStep(json_encode($headers, JSON_PRETTY_PRINT), 'Request Headers'); $operationDataResolver = new OperationDataArrayResolver($dependentEntities); $this->requestData = $operationDataResolver->resolveOperationDataArray( diff --git a/src/Magento/FunctionalTestingFramework/DataGenerator/Persist/OperationDataArrayResolver.php b/src/Magento/FunctionalTestingFramework/DataGenerator/Persist/OperationDataArrayResolver.php index b2b4fb0e4..60b6e73c2 100644 --- a/src/Magento/FunctionalTestingFramework/DataGenerator/Persist/OperationDataArrayResolver.php +++ b/src/Magento/FunctionalTestingFramework/DataGenerator/Persist/OperationDataArrayResolver.php @@ -67,6 +67,10 @@ public function __construct($dependentEntities = null) * @param boolean $fromArray * @return array * @throws \Exception + * + * @SuppressWarnings(PHPMD.CyclomaticComplexity) + * I suppressed this warning because I was in a hurry to deliver a community PR. That PR modified this function and + * introduced a new conditional, bumping the complexity to 11. */ public function resolveOperationDataArray($entityObject, $operationMetadata, $operation, $fromArray = false) { @@ -109,6 +113,26 @@ public function resolveOperationDataArray($entityObject, $operationMetadata, $op $operationElementType, $operationDataArray ); + } elseif (is_array($operationElementType)) { + foreach ($operationElementType as $currentElementType) { + if (in_array($currentElementType, self::PRIMITIVE_TYPES)) { + $this->resolvePrimitiveReferenceElement( + $entityObject, + $operationElement, + $currentElementType, + $operationDataArray + ); + } else { + $this->resolveNonPrimitiveReferenceElement( + $entityObject, + $operation, + $fromArray, + $currentElementType, + $operationElement, + $operationDataArray + ); + } + } } else { $this->resolveNonPrimitiveReferenceElement( $entityObject, @@ -248,7 +272,8 @@ private function resolveNonPrimitiveElement($entityName, $operationElement, $ope $linkedEntityObj = $this->resolveLinkedEntityObject($entityName); // in array case - if (!empty($operationElement->getNestedOperationElement($operationElement->getValue())) + if (!is_array($operationElement->getValue()) + && !empty($operationElement->getNestedOperationElement($operationElement->getValue())) && $operationElement->getType() == OperationDefinitionObjectHandler::ENTITY_OPERATION_ARRAY ) { $operationSubArray = $this->resolveOperationDataArray( diff --git a/src/Magento/FunctionalTestingFramework/DataGenerator/Util/OperationElementExtractor.php b/src/Magento/FunctionalTestingFramework/DataGenerator/Util/OperationElementExtractor.php index 35d188f5b..c86e27972 100644 --- a/src/Magento/FunctionalTestingFramework/DataGenerator/Util/OperationElementExtractor.php +++ b/src/Magento/FunctionalTestingFramework/DataGenerator/Util/OperationElementExtractor.php @@ -112,9 +112,18 @@ private function extractOperationField(&$operationElements, $operationFieldArray private function extractOperationArray(&$operationArrayData, $operationArrayArray) { foreach ($operationArrayArray as $operationFieldType) { - $operationElementValue = - $operationFieldType[OperationDefinitionObjectHandler::ENTITY_OPERATION_ARRAY_VALUE][0] - [OperationElementExtractor::OPERATION_OBJECT_ARRAY_VALUE] ?? null; + $operationElementValue = []; + if (isset($operationFieldType[OperationDefinitionObjectHandler::ENTITY_OPERATION_ARRAY_VALUE])) { + foreach ($operationFieldType[OperationDefinitionObjectHandler::ENTITY_OPERATION_ARRAY_VALUE] as + $operationFieldValue) { + $operationElementValue[] = + $operationFieldValue[OperationElementExtractor::OPERATION_OBJECT_ARRAY_VALUE] ?? null; + } + } + + if (count($operationElementValue) === 1) { + $operationElementValue = array_pop($operationElementValue); + } $nestedOperationElements = []; if (array_key_exists(OperationElementExtractor::OPERATION_OBJECT_OBJ_NAME, $operationFieldType)) { diff --git a/src/Magento/FunctionalTestingFramework/Exceptions/TestFrameworkException.php b/src/Magento/FunctionalTestingFramework/Exceptions/TestFrameworkException.php index 5e9e594c0..a82eddef0 100644 --- a/src/Magento/FunctionalTestingFramework/Exceptions/TestFrameworkException.php +++ b/src/Magento/FunctionalTestingFramework/Exceptions/TestFrameworkException.php @@ -13,6 +13,13 @@ */ class TestFrameworkException extends \Exception { + /** + * Exception context + * + * @var array + */ + protected $context; + /** * TestFrameworkException constructor. * @param string $message @@ -27,6 +34,17 @@ public function __construct($message, $context = []) $context ); + $this->context = $context; parent::__construct($message); } + + /** + * Return exception context + * + * @return array + */ + public function getContext() + { + return $this->context; + } } diff --git a/src/Magento/FunctionalTestingFramework/Extension/BrowserLogUtil.php b/src/Magento/FunctionalTestingFramework/Extension/BrowserLogUtil.php new file mode 100644 index 000000000..ca9d2be9e --- /dev/null +++ b/src/Magento/FunctionalTestingFramework/Extension/BrowserLogUtil.php @@ -0,0 +1,84 @@ +setJsError("ERROR({$entry["level"]}) - " . $entry["message"]); + } + } + + /** + * Loops through given log and returns entries of the given type. + * + * @param array $log + * @param string $type + * @return array + */ + public static function getLogsOfType($log, $type) + { + $errors = []; + foreach ($log as $entry) { + if (array_key_exists("source", $entry) && $entry["source"] === $type) { + $errors[] = $entry; + } + } + return $errors; + } + + /** + * Loops through given log and filters entries of the given type. + * + * @param array $log + * @param string $type + * @return array + */ + public static function filterLogsOfType($log, $type) + { + $errors = []; + foreach ($log as $entry) { + if (array_key_exists("source", $entry) && $entry["source"] !== $type) { + $errors[] = $entry; + } + } + return $errors; + } + + /** + * Logs errors to console/report. + * @param string $type + * @param \Codeception\Event\StepEvent $stepEvent + * @param array $entry + * @return void + */ + private static function logError($type, $stepEvent, $entry) + { + //TODO Add to overall log + $stepEvent->getTest()->getScenario()->comment("{$type} ERROR({$entry["level"]}) - " . $entry["message"]); + } +} diff --git a/src/Magento/FunctionalTestingFramework/Extension/ErrorLogger.php b/src/Magento/FunctionalTestingFramework/Extension/ErrorLogger.php deleted file mode 100644 index b0621df1b..000000000 --- a/src/Magento/FunctionalTestingFramework/Extension/ErrorLogger.php +++ /dev/null @@ -1,76 +0,0 @@ -webDriver->manage()->getAvailableLogTypes())) { - $browserLogEntries = $module->webDriver->manage()->getLog("browser"); - foreach ($browserLogEntries as $entry) { - if (array_key_exists("source", $entry) && $entry["source"] === "javascript") { - $this->logError("javascript", $stepEvent, $entry); - //Set javascript error in MagentoWebDriver internal array - $module->setJsError("ERROR({$entry["level"]}) - " . $entry["message"]); - } - } - } - } - - /** - * Logs errors to console/report. - * @param string $type - * @param \Codeception\Event\StepEvent $stepEvent - * @param array $entry - * @return void - */ - private function logError($type, $stepEvent, $entry) - { - //TODO Add to overall log - $stepEvent->getTest()->getScenario()->comment("{$type} ERROR({$entry["level"]}) - " . $entry["message"]); - } -} diff --git a/src/Magento/FunctionalTestingFramework/Extension/TestContextExtension.php b/src/Magento/FunctionalTestingFramework/Extension/TestContextExtension.php index 467074097..c50156f08 100644 --- a/src/Magento/FunctionalTestingFramework/Extension/TestContextExtension.php +++ b/src/Magento/FunctionalTestingFramework/Extension/TestContextExtension.php @@ -7,17 +7,24 @@ namespace Magento\FunctionalTestingFramework\Extension; use Codeception\Events; +use Magento\FunctionalTestingFramework\Allure\AllureHelper; use Magento\FunctionalTestingFramework\DataGenerator\Handlers\PersistedObjectHandler; /** * Class TestContextExtension * @SuppressWarnings(PHPMD.UnusedPrivateField) + * @SuppressWarnings(PHPMD.CouplingBetweenObjects) */ class TestContextExtension extends BaseExtension { const TEST_PHASE_AFTER = "_after"; - const CODECEPT_AFTER_VERSION = "2.3.9"; + const TEST_PHASE_BEFORE = "_before"; + const TEST_FAILED_FILE = 'failed'; + const TEST_HOOKS = [ + self::TEST_PHASE_AFTER => 'AfterHook', + self::TEST_PHASE_BEFORE => 'BeforeHook' + ]; /** * Codeception Events Mapping to methods @@ -35,7 +42,6 @@ public function _initialize() { $events = [ Events::TEST_START => 'testStart', - Events::TEST_FAIL => 'testFail', Events::STEP_AFTER => 'afterStep', Events::TEST_END => 'testEnd', Events::RESULT_PRINT_AFTER => 'saveFailed' @@ -56,23 +62,7 @@ public function testStart() } /** - * Codeception event listener function, triggered on test failure. - * @param \Codeception\Event\FailEvent $e - * @return void - */ - public function testFail(\Codeception\Event\FailEvent $e) - { - $cest = $e->getTest(); - $context = $this->extractContext($e->getFail()->getTrace(), $cest->getTestMethod()); - // Do not attempt to run _after if failure was in the _after block - // Try to run _after but catch exceptions to prevent them from overwriting original failure. - if ($context != TestContextExtension::TEST_PHASE_AFTER) { - $this->runAfterBlock($e, $cest); - } - } - - /** - * Codeception event listener function, triggered on test ending (naturally or by error). + * Codeception event listener function, triggered on test ending naturally or by errors/failures. * @param \Codeception\Event\TestEvent $e * @return void * @throws \Exception @@ -81,55 +71,33 @@ public function testEnd(\Codeception\Event\TestEvent $e) { $cest = $e->getTest(); - //Access private TestResultObject to find stack and if there are any errors (as opposed to failures) + //Access private TestResultObject to find stack and if there are any errors/failures $testResultObject = call_user_func(\Closure::bind( function () use ($cest) { return $cest->getTestResultObject(); }, $cest )); - $errors = $testResultObject->errors(); - if (!empty($errors)) { - foreach ($errors as $error) { - if ($error->failedTest()->getTestMethod() == $cest->getName()) { - $stack = $errors[0]->thrownException()->getTrace(); - $context = $this->extractContext($stack, $cest->getTestMethod()); - // Do not attempt to run _after if failure was in the _after block - // Try to run _after but catch exceptions to prevent them from overwriting original failure. - if ($context != TestContextExtension::TEST_PHASE_AFTER) { - $this->runAfterBlock($e, $cest); - } - continue; + + // check for errors in all test hooks and attach in allure + if (!empty($testResultObject->errors())) { + foreach ($testResultObject->errors() as $error) { + if ($error->failedTest()->getTestMethod() == $cest->getTestMethod()) { + $this->attachExceptionToAllure($error->thrownException(), $cest->getTestMethod()); } } } - // Reset Session and Cookies after all Test Runs, workaround due to functional.suite.yml restart: true - $this->getDriver()->_runAfter($e->getTest()); - } - /** - * Runs cest's after block, if necessary. - * @param \Symfony\Component\EventDispatcher\Event $e - * @param \Codeception\TestInterface $cest - * @return void - */ - private function runAfterBlock($e, $cest) - { - try { - $actorClass = $e->getTest()->getMetadata()->getCurrent('actor'); - $I = new $actorClass($cest->getScenario()); - if (version_compare(\Codeception\Codecept::VERSION, TestContextExtension::CODECEPT_AFTER_VERSION, "<=")) { - call_user_func(\Closure::bind( - function () use ($cest, $I) { - $cest->executeHook($I, 'after'); - }, - null, - $cest - )); + // check for failures in all test hooks and attach in allure + if (!empty($testResultObject->failures())) { + foreach ($testResultObject->failures() as $failure) { + if ($failure->failedTest()->getTestMethod() == $cest->getTestMethod()) { + $this->attachExceptionToAllure($failure->thrownException(), $cest->getTestMethod()); + } } - } catch (\Exception $e) { - // Do not rethrow Exception } + // Reset Session and Cookies after all Test Runs, workaround due to functional.suite.yml restart: true + $this->getDriver()->_runAfter($e->getTest()); } /** @@ -149,6 +117,46 @@ public function extractContext($trace, $class) return null; } + /** + * Attach stack trace of exceptions thrown in each test hook to allure. + * @param \Exception $exception + * @param string $testMethod + * @return mixed + */ + public function attachExceptionToAllure($exception, $testMethod) + { + if (is_subclass_of($exception, \PHPUnit\Framework\Exception::class)) { + $trace = $exception->getSerializableTrace(); + } else { + $trace = $exception->getTrace(); + } + + $context = $this->extractContext($trace, $testMethod); + + if (isset(self::TEST_HOOKS[$context])) { + $context = self::TEST_HOOKS[$context]; + } else { + $context = 'TestMethod'; + } + + AllureHelper::addAttachmentToCurrentStep($exception, $context . 'Exception'); + + //pop suppressed exceptions and attach to allure + $change = function () { + if ($this instanceof \PHPUnit\Framework\ExceptionWrapper) { + return $this->previous; + } else { + return $this->getPrevious(); + } + }; + + $previousException = $change->call($exception); + + if ($previousException !== null) { + $this->attachExceptionToAllure($previousException, $testMethod); + } + } + /** * Codeception event listener function, triggered before step. * Check if it's a new page. @@ -173,7 +181,16 @@ public function beforeStep(\Codeception\Event\StepEvent $e) */ public function afterStep(\Codeception\Event\StepEvent $e) { - ErrorLogger::getInstance()->logErrors($this->getDriver(), $e); + $browserLog = $this->getDriver()->webDriver->manage()->getLog("browser"); + if (getenv('ENABLE_BROWSER_LOG') === 'true') { + foreach (explode(',', getenv('BROWSER_LOG_BLACKLIST')) as $source) { + $browserLog = BrowserLogUtil::filterLogsOfType($browserLog, $source); + } + if (!empty($browserLog)) { + AllureHelper::addAttachmentToCurrentStep(json_encode($browserLog, JSON_PRETTY_PRINT), "Browser Log"); + } + } + BrowserLogUtil::logErrors($browserLog, $this->getDriver(), $e); } /** diff --git a/src/Magento/FunctionalTestingFramework/Module/MagentoWebDriver.php b/src/Magento/FunctionalTestingFramework/Module/MagentoWebDriver.php index ac6e86c0a..ff0c621dd 100644 --- a/src/Magento/FunctionalTestingFramework/Module/MagentoWebDriver.php +++ b/src/Magento/FunctionalTestingFramework/Module/MagentoWebDriver.php @@ -16,12 +16,16 @@ use Codeception\Util\Uri; use Magento\FunctionalTestingFramework\DataGenerator\Handlers\CredentialStore; use Magento\FunctionalTestingFramework\DataGenerator\Persist\Curl\WebapiExecutor; -use Magento\FunctionalTestingFramework\Util\Protocol\CurlTransport; +use Magento\FunctionalTestingFramework\Util\Path\UrlFormatter; use Magento\FunctionalTestingFramework\Util\Protocol\CurlInterface; use Magento\FunctionalTestingFramework\Util\ConfigSanitizerUtil; use Yandex\Allure\Adapter\AllureException; +use Magento\FunctionalTestingFramework\Util\Protocol\CurlTransport; use Yandex\Allure\Adapter\Support\AttachmentSupport; use Magento\FunctionalTestingFramework\Exceptions\TestFrameworkException; +use Magento\FunctionalTestingFramework\Config\MftfApplicationConfig; +use Facebook\WebDriver\Remote\RemoteWebDriver; +use Facebook\WebDriver\Exception\WebDriverCurlException; /** * MagentoWebDriver module provides common Magento web actions through Selenium WebDriver. @@ -49,6 +53,7 @@ class MagentoWebDriver extends WebDriver /** * List of known magento loading masks by selector + * * @var array */ public static $loadingMasksLocators = [ @@ -56,7 +61,7 @@ class MagentoWebDriver extends WebDriver '//div[contains(@class, "admin_data-grid-loading-mask")]', '//div[contains(@class, "admin__data-grid-loading-mask")]', '//div[contains(@class, "admin__form-loading-mask")]', - '//div[@data-role="spinner"]' + '//div[@data-role="spinner"]', ]; /** @@ -69,7 +74,7 @@ class MagentoWebDriver extends WebDriver 'backend_name', 'username', 'password', - 'browser' + 'browser', ]; /** @@ -116,6 +121,7 @@ class MagentoWebDriver extends WebDriver /** * Sanitizes config, then initializes using parent. + * * @return void */ public function _initialize() @@ -139,6 +145,7 @@ public function _resetConfig() /** * Remap parent::_after, called in TestContextExtension + * * @param TestInterface $test * @return void */ @@ -149,9 +156,10 @@ public function _runAfter(TestInterface $test) /** * Override parent::_after to do nothing. - * @return void + * * @param TestInterface $test * @SuppressWarnings(PHPMD) + * @return void */ public function _after(TestInterface $test) { @@ -161,9 +169,9 @@ public function _after(TestInterface $test) /** * Returns URL of a host. * - * @api * @return mixed * @throws ModuleConfigException + * @api */ public function _getUrl() { @@ -173,6 +181,7 @@ public function _getUrl() "Module connection failure. The URL for client can't bre retrieved" ); } + return $this->config['url']; } @@ -180,8 +189,8 @@ public function _getUrl() * Uri of currently opened page. * * @return string - * @api * @throws ModuleException + * @api */ public function _getCurrentUri() { @@ -189,6 +198,7 @@ public function _getCurrentUri() if ($url == 'about:blank') { throw new ModuleException($this, 'Current url is blank, no page was opened'); } + return Uri::retrieveUri($url); } @@ -257,6 +267,7 @@ public function grabFromCurrentUrl($regex = null) if (!isset($matches[1])) { $this->fail("Nothing to grab. A regex parameter with a capture group is required. Ex: '/(foo)(bar)/'"); } + return $matches[1]; } @@ -326,13 +337,13 @@ public function closeAdminNotification() * @param string $select * @param array $options * @param boolean $requireAction - * @throws \Exception * @return void + * @throws \Exception */ public function searchAndMultiSelectOption($select, array $options, $requireAction = false) { - $selectDropdown = $select . ' .action-select.admin__action-multiselect'; - $selectSearchText = $select + $selectDropdown = $select . ' .action-select.admin__action-multiselect'; + $selectSearchText = $select . ' .admin__action-multiselect-search-wrap>input[data-role="advanced-select-text"]'; $selectSearchResult = $select . ' .admin__action-multiselect-label>span'; @@ -355,8 +366,8 @@ public function searchAndMultiSelectOption($select, array $options, $requireActi * @param string $selectSearchTextField * @param string $selectSearchResult * @param string[] $options - * @throws \Exception * @return void + * @throws \Exception */ public function selectMultipleOptions($selectSearchTextField, $selectSearchResult, array $options) { @@ -393,8 +404,8 @@ public function waitForAjaxLoad($timeout = null) * Wait for all JavaScript to finish executing. * * @param integer $timeout - * @throws \Exception * @return void + * @throws \Exception */ public function waitForPageLoad($timeout = null) { @@ -409,8 +420,8 @@ public function waitForPageLoad($timeout = null) * Wait for all visible loading masks to disappear. Gets all elements by mask selector, then loops over them. * * @param integer $timeout - * @throws \Exception * @return void + * @throws \Exception */ public function waitForLoadingMaskToDisappear($timeout = null) { @@ -438,6 +449,7 @@ public function formatMoney(float $money, $locale = 'en_US.UTF-8') $this->mResetLocale(); $prefix = substr($money, 0, 1); $number = substr($money, 1); + return ['prefix' => $prefix, 'number' => $number]; } @@ -450,6 +462,7 @@ public function formatMoney(float $money, $locale = 'en_US.UTF-8') public function parseFloat($floatString) { $floatString = str_replace(',', '', $floatString); + return floatval($floatString); } @@ -471,6 +484,7 @@ public function mSetLocale(int $category, $locale) /** * Reset Locale setting. + * * @return void */ public function mResetLocale() @@ -485,6 +499,7 @@ public function mResetLocale() /** * Scroll to the Top of the Page. + * * @return void */ public function scrollToTopOfPage() @@ -493,20 +508,27 @@ public function scrollToTopOfPage() } /** - * Takes given $command and executes it against exposed MTF CLI entry point. Returns response from server. - * @param string $command - * @param string $arguments - * @throws TestFrameworkException + * Takes given $command and executes it against bin/magento or custom exposed entrypoint. Returns command output. + * + * @param string $command + * @param integer $timeout + * @param string $arguments * @return string + * + * @throws TestFrameworkException */ - public function magentoCLI($command, $arguments = null) + public function magentoCLI($command, $timeout = null, $arguments = null) { // Remove index.php if it's present in url $baseUrl = rtrim( str_replace('index.php', '', rtrim($this->config['url'], '/')), '/' ); - $apiURL = $baseUrl . '/' . ltrim(getenv('MAGENTO_CLI_COMMAND_PATH'), '/'); + + $apiURL = UrlFormatter::format( + $baseUrl . '/' . ltrim(getenv('MAGENTO_CLI_COMMAND_PATH'), '/'), + false + ); $restExecutor = new WebapiExecutor(); $executor = new CurlTransport(); @@ -515,7 +537,8 @@ public function magentoCLI($command, $arguments = null) [ 'token' => $restExecutor->getAuthToken(), getenv('MAGENTO_CLI_COMMAND_PARAMETER') => $command, - 'arguments' => $arguments + 'arguments' => $arguments, + 'timeout' => $timeout, ], CurlInterface::POST, [] @@ -523,14 +546,16 @@ public function magentoCLI($command, $arguments = null) $response = $executor->read(); $restExecutor->close(); $executor->close(); + return $response; } /** * Runs DELETE request to delete a Magento entity against the url given. + * * @param string $url - * @throws TestFrameworkException * @return string + * @throws TestFrameworkException */ public function deleteEntityByUrl($url) { @@ -538,6 +563,7 @@ public function deleteEntityByUrl($url) $executor->write($url, [], CurlInterface::DELETE, []); $response = $executor->read(); $executor->close(); + return $response; } @@ -547,8 +573,8 @@ public function deleteEntityByUrl($url) * @param string $selector * @param string $dependentSelector * @param boolean $visible - * @throws \Exception * @return void + * @throws \Exception */ public function conditionalClick($selector, $dependentSelector, $visible) { @@ -603,6 +629,7 @@ public function assertElementContainsAttribute($selector, $attribute, $value) /** * Sets current test to the given test, and resets test failure artifacts to null + * * @param TestInterface $test * @return void */ @@ -617,6 +644,7 @@ public function _before(TestInterface $test) /** * Override for codeception's default dragAndDrop to include offset options. + * * @param string $source * @param string $target * @param integer $xOffset @@ -668,17 +696,18 @@ public function fillSecretField($field, $value) * The data is decrypted immediately prior to data creation to avoid exposure in console or log. * * @param string $command + * @param null $timeout * @param null $arguments * @throws TestFrameworkException * @return string */ - public function magentoCLISecret($command, $arguments = null) + public function magentoCLISecret($command, $timeout = null, $arguments = null) { // to protect any secrets from being printed to console the values are executed only at the webdriver level as a // decrypted value $decryptedCommand = CredentialStore::getInstance()->decryptAllSecretsInString($command); - return $this->magentoCLI($decryptedCommand, $arguments); + return $this->magentoCLI($decryptedCommand, $timeout, $arguments); } /** @@ -711,6 +740,7 @@ public function _failed(TestInterface $test, $fail) /** * Function which saves a screenshot of the current stat of the browser + * * @return void */ public function saveScreenshot() @@ -730,8 +760,8 @@ public function saveScreenshot() * Go to a page and wait for ajax requests to finish * * @param string $page - * @throws \Exception * @return void + * @throws \Exception */ public function amOnPage($page) { @@ -743,8 +773,8 @@ public function amOnPage($page) * Turn Readiness check on or off * * @param boolean $check - * @throws \Exception * @return void + * @throws \Exception */ public function skipReadinessCheck($check) { @@ -787,6 +817,7 @@ private function getJsErrors() $errors .= "\n" . $jsError; } } + return $errors; } diff --git a/src/Magento/FunctionalTestingFramework/Module/MagentoWebDriverDoctor.php b/src/Magento/FunctionalTestingFramework/Module/MagentoWebDriverDoctor.php new file mode 100644 index 000000000..1407c957b --- /dev/null +++ b/src/Magento/FunctionalTestingFramework/Module/MagentoWebDriverDoctor.php @@ -0,0 +1,168 @@ +connectToSeleniumServer(); + } catch (TestFrameworkException $e) { + $context[self::EXCEPTION_CONTEXT_SELENIUM] = $e->getMessage(); + } + + try { + $adminUrl = rtrim(getenv('MAGENTO_BACKEND_BASE_URL'), '/') + ?: rtrim(getenv('MAGENTO_BASE_URL'), '/') + . '/' . getenv('MAGENTO_BACKEND_NAME') . '/admin'; + $this->loadPageAtUrl($adminUrl); + } catch (\Exception $e) { + $context[self::EXCEPTION_CONTEXT_ADMIN] = $e->getMessage(); + } + + try { + $storeUrl = getenv('MAGENTO_BASE_URL'); + $this->loadPageAtUrl($storeUrl); + } catch (\Exception $e) { + $context[self::EXCEPTION_CONTEXT_STOREFRONT] = $e->getMessage(); + } + + try { + $this->runMagentoCLI(); + } catch (\Exception $e) { + $context[self::EXCEPTION_CONTEXT_CLI] = $e->getMessage(); + } + + if (null !== $this->remoteWebDriver) { + $this->remoteWebDriver->close(); + } + + if (!empty($context)) { + throw new TestFrameworkException('Exception occurred in MagentoWebDriverDoctor', $context); + } + } + + /** + * Check connecting to running selenium server + * + * @return void + * @throws TestFrameworkException + */ + private function connectToSeleniumServer() + { + try { + $this->remoteWebDriver = RemoteWebDriver::create( + $this->wdHost, + $this->capabilities, + $this->connectionTimeoutInMs, + $this->requestTimeoutInMs, + $this->httpProxy, + $this->httpProxyPort + ); + if (null !== $this->remoteWebDriver) { + return; + } + } catch (\Exception $e) { + } + + throw new TestFrameworkException( + "Failed to connect Selenium WebDriver at: {$this->wdHost}.\n" + . "Please make sure that Selenium Server is running." + ); + } + + /** + * Validate loading a web page at url in the browser controlled by selenium + * + * @param string $url + * @return void + * @throws TestFrameworkException + */ + private function loadPageAtUrl($url) + { + try { + if (null !== $this->remoteWebDriver) { + // Open the web page at url first + $this->remoteWebDriver->get($url); + + // Execute Javascript to retrieve HTTP response code + $script = '' + . 'var xhr = new XMLHttpRequest();' + . "xhr.open('GET', '" . $url . "', false);" + . 'xhr.send(null); ' + . 'return xhr.status'; + $status = $this->remoteWebDriver->executeScript($script); + + if ($status === 200) { + return; + } + } + } catch (\Exception $e) { + } + + throw new TestFrameworkException( + "Failed to load page at url: $url\n" + . "Please check Selenium Browser session have access to Magento instance." + ); + } + + /** + * Check running Magento CLI command + * + * @return void + * @throws TestFrameworkException + */ + private function runMagentoCLI() + { + try { + $regex = '~^.*[\r\n]+.*(?Currency).*(?Code).*~'; + $output = parent::magentoCLI(self::MAGENTO_CLI_COMMAND); + preg_match($regex, $output, $matches); + + if (isset($matches['name']) && isset($matches['code'])) { + return; + } + } catch (\Exception $e) { + } + + throw new TestFrameworkException( + "Failed to run Magento CLI command\n" + . "Please reference Magento DevDoc to setup command.php and .htaccess files." + ); + } +} diff --git a/src/Magento/FunctionalTestingFramework/Suite/SuiteGenerator.php b/src/Magento/FunctionalTestingFramework/Suite/SuiteGenerator.php index 9f0045d19..4df7daa75 100644 --- a/src/Magento/FunctionalTestingFramework/Suite/SuiteGenerator.php +++ b/src/Magento/FunctionalTestingFramework/Suite/SuiteGenerator.php @@ -6,6 +6,7 @@ namespace Magento\FunctionalTestingFramework\Suite; +use Magento\FunctionalTestingFramework\Exceptions\TestFrameworkException; use Magento\FunctionalTestingFramework\Exceptions\TestReferenceException; use Magento\FunctionalTestingFramework\Exceptions\XmlException; use Magento\FunctionalTestingFramework\Suite\Generators\GroupClassGenerator; @@ -15,9 +16,14 @@ use Magento\FunctionalTestingFramework\Util\Filesystem\DirSetupUtil; use Magento\FunctionalTestingFramework\Util\Logger\LoggingUtil; use Magento\FunctionalTestingFramework\Util\Manifest\BaseTestManifest; +use Magento\FunctionalTestingFramework\Util\Path\FilePathFormatter; use Magento\FunctionalTestingFramework\Util\TestGenerator; use Symfony\Component\Yaml\Yaml; +/** + * Class SuiteGenerator + * @SuppressWarnings(PHPMD.CouplingBetweenObjects) + */ class SuiteGenerator { const YAML_CODECEPTION_DIST_FILENAME = 'codeception.dist.yml'; @@ -128,11 +134,12 @@ public function generateSuite($suiteName) * @return void * @throws TestReferenceException * @throws XmlException + * @throws TestFrameworkException */ private function generateSuiteFromTest($suiteName, $tests = [], $originalSuiteName = null) { $relativePath = TestGenerator::GENERATED_DIR . DIRECTORY_SEPARATOR . $suiteName; - $fullPath = TESTS_MODULE_PATH . DIRECTORY_SEPARATOR . $relativePath . DIRECTORY_SEPARATOR; + $fullPath = FilePathFormatter::format(TESTS_MODULE_PATH) . $relativePath . DIRECTORY_SEPARATOR; DirSetupUtil::createGroupDir($fullPath); @@ -348,9 +355,10 @@ private static function getYamlFileContents() * Static getter for the Config yml filepath (as path cannot be stored in a const) * * @return string + * @throws TestFrameworkException */ private static function getYamlConfigFilePath() { - return TESTS_BP . DIRECTORY_SEPARATOR; + return FilePathFormatter::format(TESTS_BP); } } diff --git a/src/Magento/FunctionalTestingFramework/Suite/Util/SuiteObjectExtractor.php b/src/Magento/FunctionalTestingFramework/Suite/Util/SuiteObjectExtractor.php index 7ef3d5e8c..2e4d7f9dd 100644 --- a/src/Magento/FunctionalTestingFramework/Suite/Util/SuiteObjectExtractor.php +++ b/src/Magento/FunctionalTestingFramework/Suite/Util/SuiteObjectExtractor.php @@ -13,6 +13,7 @@ use Magento\FunctionalTestingFramework\Test\Util\BaseObjectExtractor; use Magento\FunctionalTestingFramework\Test\Util\TestHookObjectExtractor; use Magento\FunctionalTestingFramework\Test\Util\TestObjectExtractor; +use Magento\FunctionalTestingFramework\Util\Path\FilePathFormatter; use Magento\FunctionalTestingFramework\Util\Validation\NameValidationUtil; class SuiteObjectExtractor extends BaseObjectExtractor @@ -259,8 +260,7 @@ private function resolveFilePathTestNames($filename, $moduleName = null) { $filepath = $filename; if (!strstr($filepath, DIRECTORY_SEPARATOR)) { - $filepath = TESTS_MODULE_PATH . - DIRECTORY_SEPARATOR . + $filepath = FilePathFormatter::format(TESTS_MODULE_PATH) . $moduleName . DIRECTORY_SEPARATOR . 'Test' . @@ -293,8 +293,7 @@ private function resolveModulePathTestNames($moduleName) { $testObjects = []; $xmlFiles = glob( - TESTS_MODULE_PATH . - DIRECTORY_SEPARATOR . + FilePathFormatter::format(TESTS_MODULE_PATH) . $moduleName . DIRECTORY_SEPARATOR . 'Test' . diff --git a/src/Magento/FunctionalTestingFramework/Test/Objects/ActionObject.php b/src/Magento/FunctionalTestingFramework/Test/Objects/ActionObject.php index 9b1cdf1af..ad5ca0c33 100644 --- a/src/Magento/FunctionalTestingFramework/Test/Objects/ActionObject.php +++ b/src/Magento/FunctionalTestingFramework/Test/Objects/ActionObject.php @@ -63,6 +63,7 @@ class ActionObject const DELETE_DATA_MUTUAL_EXCLUSIVE_ATTRIBUTES = ["url", "createDataKey"]; const EXTERNAL_URL_AREA_INVALID_ACTIONS = ['amOnPage']; const FUNCTION_CLOSURE_ACTIONS = ['waitForElementChange', 'performOn', 'executeInSelenium']; + const COMMAND_ACTION_ATTRIBUTES = ['magentoCLI', 'magentoCLISecret']; const MERGE_ACTION_ORDER_AFTER = 'after'; const MERGE_ACTION_ORDER_BEFORE = 'before'; const ACTION_ATTRIBUTE_TIMEZONE = 'timezone'; @@ -71,7 +72,7 @@ class ActionObject const ACTION_ATTRIBUTE_VARIABLE_REGEX_PARAMETER = '/\(.+\)/'; const ACTION_ATTRIBUTE_VARIABLE_REGEX_PATTERN = '/({{[\w]+\.[\w\[\]]+}})|({{[\w]+\.[\w]+\((?(?!}}).)+\)}})/'; const STRING_PARAMETER_REGEX = "/'[^']+'/"; - const DEFAULT_WAIT_TIMEOUT = 10; + const DEFAULT_COMMAND_WAIT_TIMEOUT = 60; const ACTION_ATTRIBUTE_USERINPUT = 'userInput'; const ACTION_TYPE_COMMENT = 'comment'; @@ -167,7 +168,7 @@ public function __construct( */ public static function getDefaultWaitTimeout() { - return getenv('WAIT_TIMEOUT') ?: self::DEFAULT_WAIT_TIMEOUT; + return getenv('WAIT_TIMEOUT'); } /** diff --git a/src/Magento/FunctionalTestingFramework/Test/etc/Actions/customActions.xsd b/src/Magento/FunctionalTestingFramework/Test/etc/Actions/customActions.xsd index 3a002e126..2424d0d31 100644 --- a/src/Magento/FunctionalTestingFramework/Test/etc/Actions/customActions.xsd +++ b/src/Magento/FunctionalTestingFramework/Test/etc/Actions/customActions.xsd @@ -50,6 +50,13 @@ + + + + Idle timeout in seconds, defaulted to 60s when not specified. + + + diff --git a/src/Magento/FunctionalTestingFramework/Util/ConfigSanitizerUtil.php b/src/Magento/FunctionalTestingFramework/Util/ConfigSanitizerUtil.php index ce5740493..3b92b48e4 100644 --- a/src/Magento/FunctionalTestingFramework/Util/ConfigSanitizerUtil.php +++ b/src/Magento/FunctionalTestingFramework/Util/ConfigSanitizerUtil.php @@ -7,6 +7,8 @@ namespace Magento\FunctionalTestingFramework\Util; use Magento\FunctionalTestingFramework\Config\MftfApplicationConfig; +use Magento\FunctionalTestingFramework\Exceptions\TestFrameworkException; +use Magento\FunctionalTestingFramework\Util\Path\UrlFormatter; /** * Class ConfigSanitizerUtil @@ -24,7 +26,7 @@ public static function sanitizeWebDriverConfig($config, $params = ['url', 'selen self::validateConfigBasedVars($config); if (in_array('url', $params)) { - $config['url'] = self::sanitizeUrl($config['url']); + $config['url'] = UrlFormatter::format($config['url']); } if (in_array('selenium', $params)) { @@ -80,71 +82,4 @@ private static function validateConfigBasedVars($config) } } } - - /** - * Sanitizes and returns given URL. - * @param string $url - * @return string - */ - public static function sanitizeUrl($url) - { - if (strlen($url) == 0 && !MftfApplicationConfig::getConfig()->forceGenerateEnabled()) { - trigger_error("MAGENTO_BASE_URL must be defined in .env", E_USER_ERROR); - } - - if (filter_var($url, FILTER_VALIDATE_URL) === true) { - return rtrim($url, "/") . "/"; - } - - $urlParts = parse_url($url); - - if (!isset($urlParts['scheme'])) { - $urlParts['scheme'] = "http"; - } - if (!isset($urlParts['host'])) { - $urlParts['host'] = rtrim($urlParts['path'], "/"); - $urlParts['host'] = str_replace("//", "/", $urlParts['host']); - unset($urlParts['path']); - } - - if (!isset($urlParts['path'])) { - $urlParts['path'] = "/"; - } else { - $urlParts['path'] = rtrim($urlParts['path'], "/") . "/"; - } - - return str_replace("///", "//", self::buildUrl($urlParts)); - } - - /** - * Returns url from $parts given, used with parse_url output for convenience. - * This only exists because of deprecation of http_build_url, which does the exact same thing as the code below. - * @param array $parts - * @return string - */ - private static function buildUrl(array $parts) - { - $get = function ($key) use ($parts) { - return isset($parts[$key]) ? $parts[$key] : null; - }; - - $pass = $get('pass'); - $user = $get('user'); - $userinfo = $pass !== null ? "$user:$pass" : $user; - $port = $get('port'); - $scheme = $get('scheme'); - $query = $get('query'); - $fragment = $get('fragment'); - $authority = - ($userinfo !== null ? "$userinfo@" : '') . - $get('host') . - ($port ? ":$port" : ''); - - return - (strlen($scheme) ? "$scheme:" : '') . - (strlen($authority) ? "//$authority" : '') . - $get('path') . - (strlen($query) ? "?$query" : '') . - (strlen($fragment) ? "#$fragment" : ''); - } } diff --git a/src/Magento/FunctionalTestingFramework/Util/Env/EnvProcessor.php b/src/Magento/FunctionalTestingFramework/Util/Env/EnvProcessor.php index f09ab63fa..86f7fa89d 100644 --- a/src/Magento/FunctionalTestingFramework/Util/Env/EnvProcessor.php +++ b/src/Magento/FunctionalTestingFramework/Util/Env/EnvProcessor.php @@ -7,6 +7,9 @@ namespace Magento\FunctionalTestingFramework\Util\Env; +use Magento\FunctionalTestingFramework\Exceptions\TestFrameworkException; +use Magento\FunctionalTestingFramework\Util\Path\FilePathFormatter; + /** * Helper class EnvProcessor for reading and writing .env files. * @@ -45,13 +48,14 @@ class EnvProcessor /** * EnvProcessor constructor. * @param string $envFile + * @throws TestFrameworkException */ public function __construct( string $envFile = '' ) { $this->envFile = $envFile; $this->envExists = file_exists($envFile); - $this->envExampleFile = realpath(FW_BP . "/etc/config/.env.example"); + $this->envExampleFile = realpath(FilePathFormatter::format(FW_BP) . "etc/config/.env.example"); } /** diff --git a/src/Magento/FunctionalTestingFramework/Util/Logger/LoggingUtil.php b/src/Magento/FunctionalTestingFramework/Util/Logger/LoggingUtil.php index 7ac128f28..97fdada21 100644 --- a/src/Magento/FunctionalTestingFramework/Util/Logger/LoggingUtil.php +++ b/src/Magento/FunctionalTestingFramework/Util/Logger/LoggingUtil.php @@ -6,6 +6,8 @@ namespace Magento\FunctionalTestingFramework\Util\Logger; +use Magento\FunctionalTestingFramework\Exceptions\TestFrameworkException; +use Magento\FunctionalTestingFramework\Util\Path\FilePathFormatter; use Monolog\Handler\StreamHandler; use Monolog\Logger; @@ -82,9 +84,10 @@ public function getLogger($className): MftfLogger * Function which returns a static path to the the log file. * * @return string + * @throws TestFrameworkException */ public function getLoggingPath(): string { - return TESTS_BP . DIRECTORY_SEPARATOR . "mftf.log"; + return FilePathFormatter::format(TESTS_BP) . "mftf.log"; } } diff --git a/src/Magento/FunctionalTestingFramework/Util/Manifest/TestManifestFactory.php b/src/Magento/FunctionalTestingFramework/Util/Manifest/TestManifestFactory.php index 1cfa6559e..fd7d885bf 100644 --- a/src/Magento/FunctionalTestingFramework/Util/Manifest/TestManifestFactory.php +++ b/src/Magento/FunctionalTestingFramework/Util/Manifest/TestManifestFactory.php @@ -6,7 +6,9 @@ namespace Magento\FunctionalTestingFramework\Util\Manifest; +use Magento\FunctionalTestingFramework\Exceptions\TestFrameworkException; use Magento\FunctionalTestingFramework\Test\Handlers\TestObjectHandler; +use Magento\FunctionalTestingFramework\Util\Path\FilePathFormatter; use Magento\FunctionalTestingFramework\Util\TestGenerator; class TestManifestFactory @@ -26,11 +28,11 @@ private function __construct() * @param array $suiteConfiguration * @param string $testPath * @return BaseTestManifest + * @throws TestFrameworkException */ public static function makeManifest($runConfig, $suiteConfiguration, $testPath = TestGenerator::DEFAULT_DIR) { - $testDirFullPath = TESTS_MODULE_PATH - . DIRECTORY_SEPARATOR + $testDirFullPath = FilePathFormatter::format(TESTS_MODULE_PATH) . TestGenerator::GENERATED_DIR . DIRECTORY_SEPARATOR . $testPath; diff --git a/src/Magento/FunctionalTestingFramework/Util/ModuleResolver.php b/src/Magento/FunctionalTestingFramework/Util/ModuleResolver.php index d776603d1..aa9a08ddb 100644 --- a/src/Magento/FunctionalTestingFramework/Util/ModuleResolver.php +++ b/src/Magento/FunctionalTestingFramework/Util/ModuleResolver.php @@ -9,6 +9,8 @@ use Magento\FunctionalTestingFramework\Config\MftfApplicationConfig; use Magento\FunctionalTestingFramework\Exceptions\TestFrameworkException; use Magento\FunctionalTestingFramework\Util\Logger\LoggingUtil; +use Magento\FunctionalTestingFramework\Util\Path\FilePathFormatter; +use Magento\FunctionalTestingFramework\Util\Path\UrlFormatter; use Symfony\Component\HttpFoundation\Response; /** @@ -201,7 +203,7 @@ public function getEnabledModules() $token = $this->getAdminToken(); - $url = ConfigSanitizerUtil::sanitizeUrl(getenv('MAGENTO_BASE_URL')) . $this->moduleUrl; + $url = UrlFormatter::format(getenv('MAGENTO_BASE_URL')) . $this->moduleUrl; $headers = [ 'Authorization: Bearer ' . $token, @@ -313,11 +315,11 @@ private function aggregateTestModulePaths() $allModulePaths = []; // Define the Module paths from magento bp - $magentoBaseCodePath = MAGENTO_BP; + $magentoBaseCodePath = FilePathFormatter::format(MAGENTO_BP, false); // Define the Module paths from default TESTS_MODULE_PATH $modulePath = defined('TESTS_MODULE_PATH') ? TESTS_MODULE_PATH : TESTS_BP; - $modulePath = rtrim($modulePath, DIRECTORY_SEPARATOR); + $modulePath = FilePathFormatter::format($modulePath, false); $vendorCodePath = DIRECTORY_SEPARATOR . self::VENDOR; $appCodePath = DIRECTORY_SEPARATOR . self::APP_CODE; @@ -415,15 +417,16 @@ private static function globRelevantWrapper($testPath, $pattern) * Aggregate all code paths with test module composer json files * * @return array + * @throws TestFrameworkException */ private function aggregateTestModulePathsFromComposerJson() { // Define the module paths - $magentoBaseCodePath = MAGENTO_BP; + $magentoBaseCodePath = FilePathFormatter::format(MAGENTO_BP, false); // Define the module paths from default TESTS_MODULE_PATH $modulePath = defined('TESTS_MODULE_PATH') ? TESTS_MODULE_PATH : TESTS_BP; - $modulePath = rtrim($modulePath, DIRECTORY_SEPARATOR); + $modulePath = FilePathFormatter::format($modulePath, false); $searchCodePaths = [ $magentoBaseCodePath . DIRECTORY_SEPARATOR . self::DEV_TESTS, @@ -677,7 +680,7 @@ private function printMagentoVersionInfo() if (MftfApplicationConfig::getConfig()->forceGenerateEnabled()) { return; } - $url = ConfigSanitizerUtil::sanitizeUrl(getenv('MAGENTO_BASE_URL')) . $this->versionUrl; + $url = UrlFormatter::format(getenv('MAGENTO_BASE_URL')) . $this->versionUrl; LoggingUtil::getInstance()->getLogger(ModuleResolver::class)->info( "Fetching version information.", ['url' => $url] @@ -703,7 +706,7 @@ private function printMagentoVersionInfo() * * @return string|boolean */ - protected function getAdminToken() + public function getAdminToken() { $login = $_ENV['MAGENTO_ADMIN_USERNAME'] ?? null; $password = $_ENV['MAGENTO_ADMIN_PASSWORD'] ?? null; @@ -718,7 +721,7 @@ protected function getAdminToken() throw new TestFrameworkException($message, $context); } - $url = ConfigSanitizerUtil::sanitizeUrl($this->getBackendUrl()) . $this->adminTokenUrl; + $url = $this->getBackendUrl() . $this->adminTokenUrl; $data = [ 'username' => $login, 'password' => $password @@ -886,7 +889,15 @@ private function getRegisteredModuleList() */ private function getBackendUrl() { - return getenv('MAGENTO_BACKEND_BASE_URL') ?: getenv('MAGENTO_BASE_URL'); + try { + if (getenv('MAGENTO_BACKEND_BASE_URL')) { + return UrlFormatter::format(getenv('MAGENTO_BACKEND_BASE_URL')); + } else { + return UrlFormatter::format(getenv('MAGENTO_BASE_URL')); + } + } catch (TestFrameworkException $e) { + return null; + } } /** diff --git a/src/Magento/FunctionalTestingFramework/Util/Path/FilePathFormatter.php b/src/Magento/FunctionalTestingFramework/Util/Path/FilePathFormatter.php new file mode 100644 index 000000000..8b496e739 --- /dev/null +++ b/src/Magento/FunctionalTestingFramework/Util/Path/FilePathFormatter.php @@ -0,0 +1,31 @@ +#%";/?:@&= + $sanitizedUrl = filter_var($sanitizedUrl, FILTER_SANITIZE_URL); + + if (false === $sanitizedUrl) { + throw new TestFrameworkException("Invalid url: $url\n"); + } + + // Validate URL according to http://www.faqs.org/rfcs/rfc2396 + $validUrl = filter_var($sanitizedUrl, FILTER_VALIDATE_URL); + + if (false !== $validUrl) { + return $withTrailingSeparator ? $validUrl . '/' : $validUrl; + } + + // Validation might be failed due to missing URL scheme or host, attempt to build them and re-validate + $validUrl = filter_var(self::buildUrl($sanitizedUrl), FILTER_VALIDATE_URL); + + if (false !== $validUrl) { + return $withTrailingSeparator ? $validUrl . '/' : $validUrl; + } + + throw new TestFrameworkException("Invalid url: $url\n"); + } + + /** + * Try to build missing url scheme and host + * + * @param string $url + * @return string + */ + private static function buildUrl($url) + { + $urlParts = parse_url($url); + + if (!isset($urlParts['scheme'])) { + $urlParts['scheme'] = 'http'; + } + + if (!isset($urlParts['host'])) { + $urlParts['host'] = rtrim($urlParts['path'], '/'); + $urlParts['host'] = str_replace("//", '/', $urlParts['host']); + unset($urlParts['path']); + } + + if (isset($urlParts['path'])) { + $urlParts['path'] = rtrim($urlParts['path'], '/'); + } + + return str_replace("///", "//", self::merge($urlParts)); + } + + /** + * Returns url from $parts given, used with parse_url output for convenience. + * This only exists because of deprecation of http_build_url, which does the exact same thing as the code below. + * @param array $parts + * @return string + */ + private static function merge(array $parts) + { + $get = function ($key) use ($parts) { + return isset($parts[$key]) ? $parts[$key] : null; + }; + + $pass = $get('pass'); + $user = $get('user'); + $userinfo = $pass !== null ? "$user:$pass" : $user; + $port = $get('port'); + $scheme = $get('scheme'); + $query = $get('query'); + $fragment = $get('fragment'); + $authority = + ($userinfo !== null ? "$userinfo@" : '') . + $get('host') . + ($port ? ":$port" : ''); + + return + (strlen($scheme) ? "$scheme:" : '') . + (strlen($authority) ? "//$authority" : '') . + $get('path') . + (strlen($query) ? "?$query" : '') . + (strlen($fragment) ? "#$fragment" : ''); + } +} diff --git a/src/Magento/FunctionalTestingFramework/Util/TestGenerator.php b/src/Magento/FunctionalTestingFramework/Util/TestGenerator.php index ce45af963..fcbb49ac9 100644 --- a/src/Magento/FunctionalTestingFramework/Util/TestGenerator.php +++ b/src/Magento/FunctionalTestingFramework/Util/TestGenerator.php @@ -10,6 +10,7 @@ use Magento\FunctionalTestingFramework\DataGenerator\Handlers\CredentialStore; use Magento\FunctionalTestingFramework\DataGenerator\Handlers\PersistedObjectHandler; use Magento\FunctionalTestingFramework\DataGenerator\Objects\EntityDataObject; +use Magento\FunctionalTestingFramework\Exceptions\TestFrameworkException; use Magento\FunctionalTestingFramework\Exceptions\TestReferenceException; use Magento\FunctionalTestingFramework\Suite\Handlers\SuiteObjectHandler; use Magento\FunctionalTestingFramework\Test\Handlers\ActionGroupObjectHandler; @@ -26,6 +27,7 @@ use Magento\FunctionalTestingFramework\Test\Util\TestObjectExtractor; use Magento\FunctionalTestingFramework\Util\Filesystem\DirSetupUtil; use Magento\FunctionalTestingFramework\Test\Util\ActionMergeUtil; +use Magento\FunctionalTestingFramework\Util\Path\FilePathFormatter; /** * Class TestGenerator @@ -103,14 +105,14 @@ class TestGenerator * @param string $exportDir * @param array $tests * @param boolean $debug + * @throws TestFrameworkException */ private function __construct($exportDir, $tests, $debug = false) { // private constructor for factory $this->exportDirName = $exportDir ?? self::DEFAULT_DIR; $exportDir = $exportDir ?? self::DEFAULT_DIR; - $this->exportDirectory = TESTS_MODULE_PATH - . DIRECTORY_SEPARATOR + $this->exportDirectory = FilePathFormatter::format(TESTS_MODULE_PATH) . self::GENERATED_DIR . DIRECTORY_SEPARATOR . $exportDir; @@ -611,7 +613,12 @@ public function generateStepsPhp($actionObjects, $generationScope = TestGenerato if (isset($customActionAttributes['timeout'])) { $time = $customActionAttributes['timeout']; } - $time = $time ?? ActionObject::getDefaultWaitTimeout(); + + if (in_array($actionObject->getType(), ActionObject::COMMAND_ACTION_ATTRIBUTES)) { + $time = $time ?? ActionObject::DEFAULT_COMMAND_WAIT_TIMEOUT; + } else { + $time = $time ?? ActionObject::getDefaultWaitTimeout(); + } if (isset($customActionAttributes['parameterArray']) && $actionObject->getType() != 'pressKey') { // validate the param array is in the correct format @@ -1280,6 +1287,7 @@ public function generateStepsPhp($actionObjects, $generationScope = TestGenerato $actor, $actionObject, $command, + $time, $arguments ); $testSteps .= sprintf(self::STEP_KEY_ANNOTATION, $stepKey) . PHP_EOL; @@ -1291,6 +1299,7 @@ public function generateStepsPhp($actionObjects, $generationScope = TestGenerato break; case "field": $fieldKey = $actionObject->getCustomActionAttributes()['key']; + $input = $this->resolveStepKeyReferences($input, $actionObject->getActionOrigin()); $input = $this->resolveTestVariable( [$input], $actionObject->getActionOrigin() diff --git a/src/Magento/FunctionalTestingFramework/_bootstrap.php b/src/Magento/FunctionalTestingFramework/_bootstrap.php index 34807d2b9..a5ce1f931 100644 --- a/src/Magento/FunctionalTestingFramework/_bootstrap.php +++ b/src/Magento/FunctionalTestingFramework/_bootstrap.php @@ -15,11 +15,13 @@ return; } defined('PROJECT_ROOT') || define('PROJECT_ROOT', $projectRootPath); -$envFilepath = realpath($projectRootPath . '/dev/tests/acceptance/'); +$envFilePath = realpath($projectRootPath . '/dev/tests/acceptance/') . DIRECTORY_SEPARATOR; +defined('ENV_FILE_PATH') || define('ENV_FILE_PATH', $envFilePath); -if (file_exists($envFilepath . DIRECTORY_SEPARATOR . '.env')) { - $env = new \Dotenv\Loader($envFilepath . DIRECTORY_SEPARATOR . '.env'); +//Load constants from .env file +if (file_exists(ENV_FILE_PATH . '.env')) { + $env = new \Dotenv\Loader(ENV_FILE_PATH . '.env'); $env->load(); if (array_key_exists('TESTS_MODULE_PATH', $_ENV) xor array_key_exists('TESTS_BP', $_ENV)) { @@ -48,6 +50,9 @@ defined('DEFAULT_TIMEZONE') || define('DEFAULT_TIMEZONE', 'America/Los_Angeles'); $env->setEnvironmentVariable('DEFAULT_TIMEZONE', DEFAULT_TIMEZONE); + defined('WAIT_TIMEOUT') || define('WAIT_TIMEOUT', 30); + $env->setEnvironmentVariable('WAIT_TIMEOUT', 30); + try { new DateTimeZone(DEFAULT_TIMEZONE); } catch (\Exception $e) { @@ -59,7 +64,7 @@ defined('MAGENTO_BP') || define('MAGENTO_BP', realpath(PROJECT_ROOT)); // TODO REMOVE THIS CODE ONCE WE HAVE STOPPED SUPPORTING dev/tests/acceptance PATH // define TEST_PATH and TEST_MODULE_PATH -defined('TESTS_BP') || define('TESTS_BP', realpath(MAGENTO_BP . DIRECTORY_SEPARATOR . 'dev/tests/acceptance/')); +defined('TESTS_BP') || define('TESTS_BP', realpath(MAGENTO_BP . DIRECTORY_SEPARATOR . 'dev/tests/acceptance')); $RELATIVE_TESTS_MODULE_PATH = '/tests/functional/Magento/FunctionalTest'; defined('TESTS_MODULE_PATH') || define(