diff --git a/CHANGELOG.md b/CHANGELOG.md index d3f82a2f9..fd57fff10 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,47 @@ Magento Functional Testing Framework Changelog ================================================ +2.5.0 +----- +* Traceability + * Allure output has been enhanced to contain new test artifacts created and used per MFTF step: + * `makeScreenshot` will contain an attachment under its Allure step. + * `seeInCurrentUrl` and all other `Url` asserts now contain an attachment with the expects vs actual comparison. + * `createData` and all other `Data` actions now contain attachments with `Request Body` and `Response Body`. +* Modularity + * Added a new `mftf run:manifest` command to run testManifest files generated by `generate:tests`. + * See DevDocs for details + * `mftf generate/run:test` commands now implicitly generates the `suite` the test exists in. + * If a test exists in multiple suites, it will generate it in all suite contexts. + * `mftf run:test ` will now only run the exact test provided, regardless of what is generated. +* Maintainability + * Added an `--allow-skipped` flag that allows MFTF to ignore the `` annotation. This was added to the following commands: + * `generate:test` + * `run:test` + * `run:group` + * `run:failed` +* Customizability + * `` defined in data.xml can now reference other `` directly. + * See DevDocs for details + * Added vault as an alternative credential storage. + * See DevDocs for details + +### Fixes +* Fixed an issue where `grab` action variables were not substituting correctly when used as an element parameter. +* Framework will not throw a descriptive error when referencing a `$persisted.field$` that does not exist. +* MFTF test materials that `extends=""` itself will no longer cause infinite recursion. +* Fixed an issue where a test could not reference a `$data.field$` whose casing was modified by the API that it used. +* Fixed an issue with the default `functional.suite.yml` where it was incompatible with `symfony/yaml 4.0.0`. +* Improved test generation performance via class refactors (`~10%` faster). + +### GitHub Issues/Pull requests: +* [#377](https://github.com/magento/magento2-functional-testing-framework/pull/377) -- Non-API operations fixes + +2.4.5 +----- +### Fixes +* Fixed an issue where `.credentials` was required when using `` actions with field overrides. + 2.4.4 ----- ### Fixes diff --git a/bin/mftf b/bin/mftf index 0f2bf274d..7f9db3524 100755 --- a/bin/mftf +++ b/bin/mftf @@ -29,7 +29,7 @@ try { try { $application = new Symfony\Component\Console\Application(); $application->setName('Magento Functional Testing Framework CLI'); - $application->setVersion('2.4.4'); + $application->setVersion('2.5.0'); /** @var \Magento\FunctionalTestingFramework\Console\CommandListInterface $commandList */ $commandList = new \Magento\FunctionalTestingFramework\Console\CommandList; foreach ($commandList->getCommands() as $command) { diff --git a/composer.json b/composer.json index 66936938c..b33567303 100755 --- a/composer.json +++ b/composer.json @@ -2,7 +2,7 @@ "name": "magento/magento2-functional-testing-framework", "description": "Magento2 Functional Testing Framework", "type": "library", - "version": "2.4.4", + "version": "2.5.0", "license": "AGPL-3.0", "keywords": ["magento", "automation", "functional", "testing"], "config": { diff --git a/composer.lock b/composer.lock index 10169fb08..dd7d6ad3f 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "1b7f01a11ff57f3b64f08352c33a506f", + "content-hash": "01cbd9e237e76de7070a3c0de4ee8b9f", "packages": [ { "name": "allure-framework/allure-codeception", @@ -250,7 +250,7 @@ { "name": "Tobias Nyholm", "email": "tobias.nyholm@gmail.com", - "homepage": "https://github.com/nyholm" + "homepage": "https://github.com/Nyholm" } ], "description": "Library of all the php-cache adapters", diff --git a/docs/actiongroup-list.md b/docs/actiongroup-list.md new file mode 100644 index 000000000..7b469f4ed --- /dev/null +++ b/docs/actiongroup-list.md @@ -0,0 +1,6 @@ +# MFTF action group reference + +Action groups are important building blocks for quickly creating tests for the Magento Functional Testing Framework. +This page lists all current action groups so developers can see what is available to them. + +{% include mftf/actiongroup_data.md %} diff --git a/docs/configuration.md b/docs/configuration.md index 9d87e0735..c741e2f60 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1,7 +1,7 @@ # Configuration The `*.env` file provides additional configuration for the Magento Functional Testing Framework (MFTF). -To run the MFTF on your Magento testing instance, specify the basic configuration values. +To run the MFTF on your Magento instance, specify the basic configuration values. Advanced users can create custom configurations based on requirements and environment. ## Basic configuration @@ -204,7 +204,7 @@ Example: FW_BP=~/magento/magento2-functional-testing-framework ``` -#### TESTS_MODULE_PATH +### TESTS_MODULE_PATH The path to where the MFTF modules mirror Magento modules. @@ -234,8 +234,8 @@ It points to `MAGENTO_BASE_URL` + `dev/tests/acceptance/utils/command.php` Modify the default value: -- for non-default Magento installation -- when use a subdirectory in the `MAGENTO_BASE_URL` +- for non-default Magento installation +- when use a subdirectory in the `MAGENTO_BASE_URL` Example: `dev/tests/acceptance/utils/command.php` @@ -251,9 +251,35 @@ Example: BROWSER=firefox ``` +### CREDENTIAL_VAULT_ADDRESS + +The Api address for a vault server. + +Default: http://127.0.0.1:8200 + +Example: + +```conf +# Default api address for local vault dev server +CREDENTIAL_VAULT_ADDRESS=http://127.0.0.1:8200 +``` + +### CREDENTIAL_VAULT_SECRET_BASE_PATH + +Vault secret engine base path. + +Default: secret + +Example: + +```conf +# Default base path for kv secret engine in local vault dev server +CREDENTIAL_VAULT_SECRET_BASE_PATH=secret +``` + [`MAGENTO_CLI_COMMAND_PATH`]: #magento_cli_command_path [generateDate]: test/actions.md#generatedate [mftf]: commands/mftf.md -[timezones]: http://php.net/manual/en/timezones.php \ No newline at end of file +[timezones]: http://php.net/manual/en/timezones.php diff --git a/docs/credentials.md b/docs/credentials.md index 7afa9b805..4b1e2cc2d 100644 --- a/docs/credentials.md +++ b/docs/credentials.md @@ -1,15 +1,21 @@ # Credentials -When you test functionality that involves external services such as UPS, FedEx, PayPal, or SignifyD, use the MFTF credentials feature to hide sensitive [data][] like integration tokens and API keys. +When you test functionality that involves external services such as UPS, FedEx, PayPal, or SignifyD, +use the MFTF credentials feature to hide sensitive [data][] like integration tokens and API keys. -## Define sensitive data in `.credentials` +Currently the MFTF supports two types of credential storage: + +- **.credentials file** +- **HashiCorp vault** + +## Configure File Storage The MFTF creates a sample file for credentials during [initial setup][]: `magento2/dev/tests/acceptance/.credentials.example`. The file contains an example list of keys for fields that can require credentials. ### Create `.credentials` -To make the MFTF process the file with credentials, change directories to `magento2/dev/tests/acceptance/` and copy `.credentials.example` to `.credentials`. +To make the MFTF process the file with credentials, in the command line, navigate to `magento2/dev/tests/acceptance/` and rename `.credentials.example` to `.credentials`. ```bash cd dev/tests/acceptance/ @@ -33,49 +39,119 @@ The command outputs the path if the file is excluded: .credentials ``` -### Define sensitive data +### Define sensitive data in the `.credentials` file -Open the `.credentials` file, uncomment the fields you want to use, and add your values: +Open the `.credentials` file and, for Magento core credentials, uncomment the fields you want to use and add your values: -```config +```conf ... # Credentials for the USPS service -carriers_usps_userid=test_user -carriers_usps_password=Lmgxvrq89uPwECeV +magento/carriers_usps_userid=usps_test_user +magento/carriers_usps_password=Lmgxvrq89uPwECeV # Credentials for the DHL service -#carriers/dhl/id_us= -#carriers/dhl/password_us= +#magento/carriers_dhl_id_us=dhl_test_user +#magento/carriers_dhl_password_us=Mlgxv3dsagVeG .... +``` + +Or add new key & value pairs for your own credentials. The keys use the following format: + +```conf +/= ```
-The `/` symbol is not supported in a key name. +The `/` symbol is not supported in a `key_name` other than the one after your vendor or extension name.
- -You are free to use any other keys you like, as they are merely the keys to reference from your tests. + +Otherwise you are free to use any other `key_name` you like, as they are merely the keys to reference from your tests. ```conf # Credentials for the MyAwesome service -my_awesome_service_token=rRVSVnh3cbDsVG39oTMz4A +vendor/my_awesome_service_token=rRVSVnh3cbDsVG39oTMz4A +``` -# Credentials for the USPS service -carriers_usps_userid=test_user -carriers_usps_password=Lmgxvrq89uPwECeV -.... +## Configure Vault Storage + +Hashicorp vault secures, stores, and tightly controls access to data in modern computing. +It provides advanced data protection for your testing credentials. + +The MFTF works with both `vault enterprise` and `vault open source` that use `KV Version 2` secret engine. + +### Install vault CLI + +Download and install vault CLI tool if you want to run or develop MFTF tests locally. [Download Vault][Download Vault] + +### Authenticate to vault via vault CLI + +Authenticate to vault server via the vault CLI tool: [Login Vault][Login Vault]. + +```bash +vault login -method -path +``` + +**Do not** use `-no-store` command option, as the MFTF will rely on the persisted token in the token helper (usually the local filesystem) for future API requests. + +### Store secrets in vault + +The MFTF uses the `KV Version 2` secret engine for secret storage. +More information for working with `KV Version 2` can be found in [Vault KV2][Vault KV2]. + +#### Secrets path and key convention + +The path and key for secret data must follow the format: + +```conf +/mftf// +``` + +```conf +# Secret path and key for carriers_usps_userid +secret/mftf/magento/carriers_usps_userid + +# Secret path and key for carriers_usps_password +secret/mftf/magento/carriers_usps_password +``` + +#### Write secrets to vault + +You can use vault CLI or API to write secret data (credentials, etc) to vault. Here is a CLI example: + +```bash +vault kv put secret/mftf/magento/carriers_usps_userid carriers_usps_userid=usps_test_user +vault kv put secret/mftf/magento/carriers_usps_password carriers_usps_password=Lmgxvrq89uPwECeV ``` +### Setup MFTF to use vault + +Add vault configuration environment variables [`CREDENTIAL_VAULT_ADDRESS`][] and [`CREDENTIAL_VAULT_SECRET_BASE_PATH`][] +from `etc/config/.env.example` in `.env`. +Set values according to your vault server configuration. + +```conf +# Default vault dev server +CREDENTIAL_VAULT_ADDRESS=http://127.0.0.1:8200 +CREDENTIAL_VAULT_SECRET_BASE_PATH=secret +``` + +## Configure both File Storage and Vault Storage + +It is possible and sometimes useful to setup and use both `.credentials` file and vault for secret storage at the same time. +In this case, the MFTF tests are able to read secret data at runtime from both storage options, but the local `.credentials` file will take precedence. + ## Use credentials in a test -Access the data defined in the `.credentials` file using the [`fillField`][] action with the `userInput` attribute. -Define the value as a reference to the corresponding key in the credentials file such as `{{_CREDS.my_data_key}}`: +Credentials can be used in actions: [`fillField`][], [`magentoCLI`][], and [`createData`][]. + +Define the value as a reference to the corresponding key in the credentials file or vault such as `{{_CREDS.my_data_key}}`: - `_CREDS` is an environment constant pointing to the `.credentials` file -- `my_data_key` is a key in the the `.credentials` file that contains the value to be used in a test step +- `my_data_key` is a key in the the `.credentials` file or vault that contains the value to be used in a test step -For example: +For example, reference secret data in the [`fillField`][] action with the `userInput` attribute. ```xml @@ -88,13 +164,21 @@ For example: The generated tests do not contain credentials values. The MFTF dynamically retrieves, encrypts, and decrypts the sensitive data during test execution. Decrypted credentials do not appear in the console, error logs, or [test reports][]. -The decrypted values are only available in the `.credentials` file. +The decrypted values are only available in the `.credentials` file or within vault.
-The MFTF tests delivered with Magento application do not use credentials and do not cover external services, because of sensitivity of the data.
+The MFTF tests delivered with Magento application do not use credentials and do not cover external services, because of sensitivity of the data. + [`fillField`]: test/actions.md#fillfield +[`magentoCLI`]: test/actions.md#magentocli +[`createData`]: test/actions.md#createdata [data]: data.md [initial setup]: getting-started.md [test reports]: reporting.md +[Download Vault]: https://www.hashicorp.com/products/vault/ +[Login Vault]: https://www.vaultproject.io/docs/commands/login.html +[Vault KV2]: https://www.vaultproject.io/docs/secrets/kv/kv-v2.html +[`CREDENTIAL_VAULT_ADDRESS`]: configuration.md#CREDENTIAL_VAULT_ADDRESS +[`CREDENTIAL_VAULT_SECRET_BASE_PATH`]: configuration.md#CREDENTIAL_VAULT_SECRET_BASE_PATH diff --git a/docs/data.md b/docs/data.md index 875df0198..4c618ae45 100644 --- a/docs/data.md +++ b/docs/data.md @@ -173,6 +173,16 @@ The following is an example of a call in test: This action inputs data from the `name` of the `_defaultCategory` entity (for example, `simpleCategory598742365`) into the field with the locator defined in the selector of the `categoryNameInput` element of the `AdminCategoryBasicFieldSection`. +You can also call data from the xml definition of a `data` tag directly: + +```xml + + admin + {{AnotherUser.current_password}} + {{_ENV.MAGENTO_ADMIN_PASSWORD}} + +``` + ## Reference ### entities {#entities-tag} diff --git a/docs/selectors.md b/docs/selectors.md new file mode 100644 index 000000000..870072e15 --- /dev/null +++ b/docs/selectors.md @@ -0,0 +1,35 @@ +## Selectors + +These guidelines should help you to write high quality selectors. + +### Selectors SHOULD be written in CSS instead of XPath whenever possible + +CSS is generally easier to read than XPath. For example, `//*[@id="foo"]` in XPath can be expressed as simply as `#foo` in CSS. +See this [XPath Cheatsheet](https://devhints.io/xpath) for more examples. + +### XPath selectors SHOULD NOT use `@attribute="foo"`. + +This would fail if the attribute was `attribute="foo bar"`. +Instead you SHOULD use `contains(@attribute, "foo")` where `@attribute` is any valid attribute such as `@text` or `@class`. + +### CSS and XPath selectors SHOULD be implemented in their most simple form + +* GOOD: `#foo` +* BAD: `button[contains(@id, "foo")]` + +### CSS and XPath selectors SHOULD avoid making use of hardcoded indices + +Instead you SHOULD parameterize the selector. + +* GOOD: `.foo:nth-of-type({{index}})` +* BAD: `.foo:nth-of-type(1)` + +* GOOD: `button[contains(@id, "foo")][{{index}}]` +* BAD: `button[contains(@id, "foo")][1]` + +* GOOD: `#actions__{{index}}__aggregator` +* BAD: `#actions__1__aggregator` + +### CSS and XPath selectors MUST NOT reference the `@data-bind` attribute + +The `@data-bind` attribute is used by KnockoutJS, a framework Magento uses to create dynamic Javascript pages. Since this `@data-bind` attribute is tied to a specific framework, it should not be used for selectors. If Magento decides to use a different framework then these `@data-bind` selectors would break. diff --git a/etc/config/.credentials.example b/etc/config/.credentials.example index d9c73ac66..429e9d19f 100644 --- a/etc/config/.credentials.example +++ b/etc/config/.credentials.example @@ -1,4 +1,4 @@ -#magento/magento/carriers_fedex_account= +#magento/carriers_fedex_account= #magento/carriers_fedex_meter_number= #magento/carriers_fedex_key= #magento/carriers_fedex_password= diff --git a/etc/config/command.php b/etc/config/command.php index f018e3014..047af324a 100644 --- a/etc/config/command.php +++ b/etc/config/command.php @@ -12,8 +12,8 @@ $tokenModel = $magentoObjectManager->get(\Magento\Integration\Model\Oauth\Token::class); $tokenPassedIn = urldecode($_POST['token']); - $command = str_replace([';', '&', '|'], '', urldecode($_POST['command'])); - $arguments = str_replace([';', '&', '|'], '', urldecode($_POST['arguments'])); + $command = urldecode($_POST['command']); + $arguments = urldecode($_POST['arguments']); // Token returned will be null if the token we passed in is invalid $tokenFromMagento = $tokenModel->loadByToken($tokenPassedIn)->getToken(); diff --git a/src/Magento/FunctionalTestingFramework/Console/BaseGenerateCommand.php b/src/Magento/FunctionalTestingFramework/Console/BaseGenerateCommand.php index 40582db86..87b203d66 100644 --- a/src/Magento/FunctionalTestingFramework/Console/BaseGenerateCommand.php +++ b/src/Magento/FunctionalTestingFramework/Console/BaseGenerateCommand.php @@ -36,7 +36,7 @@ protected function configure() InputOption::VALUE_NONE, 'force generation and running of tests regardless of Magento Instance Configuration' )->addOption( - "allowSkipped", + "allow-skipped", 'a', InputOption::VALUE_NONE, 'Allows MFTF to generate and run skipped tests.' diff --git a/src/Magento/FunctionalTestingFramework/Console/GenerateSuiteCommand.php b/src/Magento/FunctionalTestingFramework/Console/GenerateSuiteCommand.php index 7dd3b8cb4..cd798f420 100644 --- a/src/Magento/FunctionalTestingFramework/Console/GenerateSuiteCommand.php +++ b/src/Magento/FunctionalTestingFramework/Console/GenerateSuiteCommand.php @@ -46,7 +46,7 @@ protected function execute(InputInterface $input, OutputInterface $output) $debug = $input->getOption('debug') ?? MftfApplicationConfig::LEVEL_DEVELOPER; // for backward compatibility $remove = $input->getOption('remove'); $verbose = $output->isVerbose(); - $allowSkipped = $input->getOption('allowSkipped'); + $allowSkipped = $input->getOption('allow-skipped'); // Set application configuration so we can references the user options in our framework MftfApplicationConfig::create( diff --git a/src/Magento/FunctionalTestingFramework/Console/GenerateTestsCommand.php b/src/Magento/FunctionalTestingFramework/Console/GenerateTestsCommand.php index 64c7c01d5..621f29d03 100644 --- a/src/Magento/FunctionalTestingFramework/Console/GenerateTestsCommand.php +++ b/src/Magento/FunctionalTestingFramework/Console/GenerateTestsCommand.php @@ -71,7 +71,7 @@ protected function execute(InputInterface $input, OutputInterface $output) $debug = $input->getOption('debug') ?? MftfApplicationConfig::LEVEL_DEVELOPER; // for backward compatibility $remove = $input->getOption('remove'); $verbose = $output->isVerbose(); - $allowSkipped = $input->getOption('allowSkipped'); + $allowSkipped = $input->getOption('allow-skipped'); // Set application configuration so we can references the user options in our framework MftfApplicationConfig::create( diff --git a/src/Magento/FunctionalTestingFramework/Console/RunTestCommand.php b/src/Magento/FunctionalTestingFramework/Console/RunTestCommand.php index b83953dce..77b8be513 100644 --- a/src/Magento/FunctionalTestingFramework/Console/RunTestCommand.php +++ b/src/Magento/FunctionalTestingFramework/Console/RunTestCommand.php @@ -19,6 +19,13 @@ class RunTestCommand extends BaseGenerateCommand { + /** + * The return code. Determined by all tests that run. + * + * @var integer + */ + private $returnCode = 0; + /** * Configures the current command. * @@ -49,8 +56,6 @@ protected function configure() * @param OutputInterface $output * @return integer * @throws \Exception - * - * @SuppressWarnings(PHPMD.UnusedLocalVariable) */ protected function execute(InputInterface $input, OutputInterface $output): int { @@ -59,7 +64,7 @@ protected function execute(InputInterface $input, OutputInterface $output): int $force = $input->getOption('force'); $remove = $input->getOption('remove'); $debug = $input->getOption('debug') ?? MftfApplicationConfig::LEVEL_DEVELOPER; // for backward compatibility - $allowSkipped = $input->getOption('allowSkipped'); + $allowSkipped = $input->getOption('allow-skipped'); $verbose = $output->isVerbose(); if ($skipGeneration and $remove) { @@ -87,65 +92,89 @@ protected function execute(InputInterface $input, OutputInterface $output): int '--force' => $force, '--remove' => $remove, '--debug' => $debug, - '--allowSkipped' => $allowSkipped, + '--allow-skipped' => $allowSkipped, '-v' => $verbose ]; $command->run(new ArrayInput($args), $output); } - // tests with resolved suite references - $resolvedTests = $this->resolveSuiteReferences($testConfiguration); + $testConfigArray = json_decode($testConfiguration, true); + + if (isset($testConfigArray['tests'])) { + $this->runTests($testConfigArray['tests'], $output); + } + + if (isset($testConfigArray['suites'])) { + $this->runTestsInSuite($testConfigArray['suites'], $output); + } + + return $this->returnCode; + } + + /** + * Run tests not referenced in suites + * + * @param array $tests + * @param OutputInterface $output + * @return void + * @throws TestFrameworkException + */ + private function runTests(array $tests, OutputInterface $output) + { $codeceptionCommand = realpath(PROJECT_ROOT . '/vendor/bin/codecept') . ' run functional '; - $testsDirectory = TESTS_MODULE_PATH . DIRECTORY_SEPARATOR . TestGenerator::GENERATED_DIR . DIRECTORY_SEPARATOR; - $returnCode = 0; - //execute only tests specified as arguments in run command - foreach ($resolvedTests as $test) { - //set directory as suite name for tests in suite, if not set to "default" - if (strpos($test, ':')) { - list($testGroup, $testName) = explode(":", $test); - } else { - list($testGroup, $testName) = [TestGenerator::DEFAULT_DIR, $test]; - } - $testGroup = $testGroup . DIRECTORY_SEPARATOR; - $testName = $testName . 'Cest.php'; - if (!realpath($testsDirectory . $testGroup . $testName)) { + $testsDirectory = TESTS_MODULE_PATH . + DIRECTORY_SEPARATOR . + TestGenerator::GENERATED_DIR . + DIRECTORY_SEPARATOR . + TestGenerator::DEFAULT_DIR . + DIRECTORY_SEPARATOR ; + + foreach ($tests as $test) { + $testName = $test . 'Cest.php'; + if (!realpath($testsDirectory . $testName)) { throw new TestFrameworkException( - $testName . " is not available under " . $testsDirectory . $testGroup + $testName . " is not available under " . $testsDirectory ); } - $fullCommand = $codeceptionCommand . $testsDirectory . $testGroup . $testName . ' --verbose --steps'; - $process = new Process($fullCommand); - $process->setWorkingDirectory(TESTS_BP); - $process->setIdleTimeout(600); - $process->setTimeout(0); - - $returnCode = max($returnCode, $process->run( - function ($type, $buffer) use ($output) { - $output->write($buffer); - } - )); + $fullCommand = $codeceptionCommand . $testsDirectory . $testName . ' --verbose --steps'; + $this->returnCode = max($this->returnCode, $this->executeTestCommand($fullCommand, $output)); } - return $returnCode; } /** - * Get an array of tests with resolved suite references from $testConfiguration - * eg: if test is referenced in a suite, it'll be stored in format suite:test - * @param string $testConfigurationJson - * @return array + * Run tests referenced in suites within suites' context. + * + * @param array $suitesConfig + * @param OutputInterface $output + * @return void */ - private function resolveSuiteReferences($testConfigurationJson) + private function runTestsInSuite(array $suitesConfig, OutputInterface $output) { - $testConfiguration = json_decode($testConfigurationJson, true); - $testsArray = $testConfiguration['tests'] ?? []; - $suitesArray = $testConfiguration['suites'] ?? []; - $testArrayBuilder = []; - - foreach ($suitesArray as $suite => $tests) { - foreach ($tests as $test) { - $testArrayBuilder[] = "$suite:$test"; - } + $codeceptionCommand = realpath(PROJECT_ROOT . '/vendor/bin/codecept') . ' run functional --verbose --steps '; + //for tests in suites, run them as a group to run before and after block + foreach (array_keys($suitesConfig) as $suite) { + $fullCommand = $codeceptionCommand . " -g {$suite}"; + $this->returnCode = max($this->returnCode, $this->executeTestCommand($fullCommand, $output)); } - return array_merge($testArrayBuilder, $testsArray); + } + + /** + * Runs the codeception test command and returns exit code + * + * @param string $command + * @param OutputInterface $output + * @return integer + * + * @SuppressWarnings(PHPMD.UnusedLocalVariable) + */ + private function executeTestCommand(string $command, OutputInterface $output) + { + $process = new Process($command); + $process->setWorkingDirectory(TESTS_BP); + $process->setIdleTimeout(600); + $process->setTimeout(0); + return $process->run(function ($type, $buffer) use ($output) { + $output->write($buffer); + }); } } diff --git a/src/Magento/FunctionalTestingFramework/Console/RunTestFailedCommand.php b/src/Magento/FunctionalTestingFramework/Console/RunTestFailedCommand.php index 15ba723e6..336fd5d87 100644 --- a/src/Magento/FunctionalTestingFramework/Console/RunTestFailedCommand.php +++ b/src/Magento/FunctionalTestingFramework/Console/RunTestFailedCommand.php @@ -70,7 +70,7 @@ protected function execute(InputInterface $input, OutputInterface $output): int { $force = $input->getOption('force'); $debug = $input->getOption('debug') ?? MftfApplicationConfig::LEVEL_DEVELOPER; // for backward compatibility - $allowSkipped = $input->getOption('allowSkipped'); + $allowSkipped = $input->getOption('allow-skipped'); $verbose = $output->isVerbose(); // Create Mftf Configuration @@ -95,7 +95,7 @@ protected function execute(InputInterface $input, OutputInterface $output): int '--force' => $force, '--remove' => true, '--debug' => $debug, - '--allowSkipped' => $allowSkipped, + '--allow-skipped' => $allowSkipped, '-v' => $verbose ]; $command->run(new ArrayInput($args), $output); diff --git a/src/Magento/FunctionalTestingFramework/Console/RunTestGroupCommand.php b/src/Magento/FunctionalTestingFramework/Console/RunTestGroupCommand.php index 20d290af9..7f954c8fe 100644 --- a/src/Magento/FunctionalTestingFramework/Console/RunTestGroupCommand.php +++ b/src/Magento/FunctionalTestingFramework/Console/RunTestGroupCommand.php @@ -60,7 +60,7 @@ protected function execute(InputInterface $input, OutputInterface $output): int $groups = $input->getArgument('groups'); $remove = $input->getOption('remove'); $debug = $input->getOption('debug') ?? MftfApplicationConfig::LEVEL_DEVELOPER; // for backward compatibility - $allowSkipped = $input->getOption('allowSkipped'); + $allowSkipped = $input->getOption('allow-skipped'); $verbose = $output->isVerbose(); if ($skipGeneration and $remove) { @@ -87,7 +87,7 @@ protected function execute(InputInterface $input, OutputInterface $output): int '--force' => $force, '--remove' => $remove, '--debug' => $debug, - '--allowSkipped' => $allowSkipped, + '--allow-skipped' => $allowSkipped, '-v' => $verbose ]; diff --git a/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/PersistedObjectHandler.php b/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/PersistedObjectHandler.php index ea3aa919a..42286564f 100644 --- a/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/PersistedObjectHandler.php +++ b/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/PersistedObjectHandler.php @@ -89,7 +89,12 @@ public function createEntity( } foreach ($overrideFields as $index => $field) { - $overrideFields[$index] = CredentialStore::getInstance()->decryptAllSecretsInString($field); + try { + $overrideFields[$index] = CredentialStore::getInstance()->decryptAllSecretsInString($field); + } catch (TestFrameworkException $e) { + //do not rethrow if Credentials are not defined + $overrideFields[$index] = $field; + } } $retrievedEntity = DataObjectHandler::getInstance()->getObject($entity); diff --git a/src/Magento/FunctionalTestingFramework/DataGenerator/Persist/CurlHandler.php b/src/Magento/FunctionalTestingFramework/DataGenerator/Persist/CurlHandler.php index 4b184c206..691ce3606 100644 --- a/src/Magento/FunctionalTestingFramework/DataGenerator/Persist/CurlHandler.php +++ b/src/Magento/FunctionalTestingFramework/DataGenerator/Persist/CurlHandler.php @@ -167,8 +167,6 @@ public function executeRequest($dependentEntities) $response = $executor->read($successRegex, $returnRegex, $returnIndex); $executor->close(); - AllureHelper::addAttachmentToLastStep($apiUrl, 'API Endpoint'); - AllureHelper::addAttachmentToLastStep(json_encode($headers, JSON_PRETTY_PRINT), 'Request Headers'); AllureHelper::addAttachmentToLastStep(json_encode($this->requestData, JSON_PRETTY_PRINT), 'Request Body'); AllureHelper::addAttachmentToLastStep( json_encode(json_decode($response, true), JSON_PRETTY_PRINT+JSON_UNESCAPED_UNICODE+JSON_UNESCAPED_SLASHES),