From fa579d1fa07806259d995310439e0e5b4a8215d5 Mon Sep 17 00:00:00 2001 From: Soumya Unnikrishnan Date: Tue, 10 Sep 2019 16:32:33 -0500 Subject: [PATCH 1/5] MQE-1755: mftf run:test Test1 Test2 does not run before/after hooks correctly --- .../Console/RunTestCommand.php | 104 +++++++++++------- 1 file changed, 64 insertions(+), 40 deletions(-) diff --git a/src/Magento/FunctionalTestingFramework/Console/RunTestCommand.php b/src/Magento/FunctionalTestingFramework/Console/RunTestCommand.php index e1e30589d..10f3defc6 100644 --- a/src/Magento/FunctionalTestingFramework/Console/RunTestCommand.php +++ b/src/Magento/FunctionalTestingFramework/Console/RunTestCommand.php @@ -19,6 +19,13 @@ class RunTestCommand extends BaseGenerateCommand { + /** + * The return code. Determined by all tests that run. + * + * @var integer + */ + private $returnCode = 0; + /** * Configures the current command. * @@ -76,60 +83,77 @@ protected function execute(InputInterface $input, OutputInterface $output): int ]; $command->run(new ArrayInput($args), $output); } - // tests with resolved suite references - $resolvedTests = $this->resolveSuiteReferences($testConfiguration); + $testConfigArray = json_decode($testConfiguration, true); + + // run tests not referenced in suites + $this->runTests($testConfigArray['tests'], $output); + + // run tests in suites + $this->runTestsInSuite($testConfigArray['suites'], $output); + + return $this->returnCode; + + } + + /** + * Run tests not referenced in suites + * @param array $testsConfig + * @param OutputInterface $output + * @throws TestFrameworkException + */ + private function runTests($testsConfig, OutputInterface $output) { + + + $tests = $testsConfig ?? []; $codeceptionCommand = realpath(PROJECT_ROOT . '/vendor/bin/codecept') . ' run functional '; - $testsDirectory = TESTS_MODULE_PATH . DIRECTORY_SEPARATOR . TestGenerator::GENERATED_DIR . DIRECTORY_SEPARATOR; - $returnCode = 0; - //execute only tests specified as arguments in run command - foreach ($resolvedTests as $test) { - //set directory as suite name for tests in suite, if not set to "default" - if (strpos($test, ':')) { - list($testGroup, $testName) = explode(":", $test); - } else { - list($testGroup, $testName) = [TestGenerator::DEFAULT_DIR, $test]; - } - $testGroup = $testGroup . DIRECTORY_SEPARATOR; - $testName = $testName . 'Cest.php'; - if (!realpath($testsDirectory . $testGroup . $testName)) { + $testsDirectory = TESTS_MODULE_PATH . + DIRECTORY_SEPARATOR . + TestGenerator::GENERATED_DIR . + DIRECTORY_SEPARATOR . + TestGenerator::DEFAULT_DIR . + DIRECTORY_SEPARATOR ; + + foreach ($tests as $test) { + $testName = $test . 'Cest.php'; + if (!realpath($testsDirectory . $testName)) { throw new TestFrameworkException( - $testName . " is not available under " . $testsDirectory . $testGroup + $testName . " is not available under " . $testsDirectory ); } - $fullCommand = $codeceptionCommand . $testsDirectory . $testGroup . $testName . ' --verbose --steps'; + $fullCommand = $codeceptionCommand . $testsDirectory . $testName . ' --verbose --steps'; $process = new Process($fullCommand); $process->setWorkingDirectory(TESTS_BP); $process->setIdleTimeout(600); $process->setTimeout(0); - - $returnCode = max($returnCode, $process->run( - function ($type, $buffer) use ($output) { - $output->write($buffer); - } - )); + $subReturnCode = $process->run(function ($type, $buffer) use ($output) { + $output->write($buffer); + }); + $this->returnCode = max($this->returnCode, $subReturnCode); } - return $returnCode; } /** - * Get an array of tests with resolved suite references from $testConfiguration - * eg: if test is referenced in a suite, it'll be stored in format suite:test - * @param string $testConfigurationJson - * @return array + * Run tests referenced in suites within suites' context. + * @param array $suitesConfig + * @param OutputInterface $output */ - private function resolveSuiteReferences($testConfigurationJson) - { - $testConfiguration = json_decode($testConfigurationJson, true); - $testsArray = $testConfiguration['tests'] ?? []; - $suitesArray = $testConfiguration['suites'] ?? []; - $testArrayBuilder = []; - - foreach ($suitesArray as $suite => $tests) { - foreach ($tests as $test) { - $testArrayBuilder[] = "$suite:$test"; - } + private function runTestsInSuite($suitesConfig, OutputInterface $output) { + + $suites = $suitesConfig ?? []; + $codeceptionCommand = realpath(PROJECT_ROOT . '/vendor/bin/codecept') . ' run functional --verbose --steps '; + $testGroups = array_keys($suites); + //for tests in suites, run them as a group to run before and after block + foreach ($testGroups as $testGroup) { + $fullCommand = $codeceptionCommand . " -g {$testGroup}"; + $process = new Process($fullCommand); + $process->setWorkingDirectory(TESTS_BP); + $process->setIdleTimeout(600); + $process->setTimeout(0); + $subReturnCode = $process->run(function ($type, $buffer) use ($output) { + $output->write($buffer); + }); + $this->returnCode = max($this->returnCode, $subReturnCode); } - return array_merge($testArrayBuilder, $testsArray); } } From 3b84f12a963a789bded006226d0dfd46463de279 Mon Sep 17 00:00:00 2001 From: Soumya Unnikrishnan Date: Tue, 10 Sep 2019 19:08:02 -0500 Subject: [PATCH 2/5] MQE-1755: mftf run:test Test1 Test2 does not run before/after hooks correctly fixed unit tests --- .../Console/RunTestCommand.php | 39 ++++++++++--------- 1 file changed, 21 insertions(+), 18 deletions(-) diff --git a/src/Magento/FunctionalTestingFramework/Console/RunTestCommand.php b/src/Magento/FunctionalTestingFramework/Console/RunTestCommand.php index 10f3defc6..668d5152c 100644 --- a/src/Magento/FunctionalTestingFramework/Console/RunTestCommand.php +++ b/src/Magento/FunctionalTestingFramework/Console/RunTestCommand.php @@ -51,8 +51,6 @@ protected function configure() * @param OutputInterface $output * @return integer * @throws \Exception - * - * @SuppressWarnings(PHPMD.UnusedLocalVariable) */ protected function execute(InputInterface $input, OutputInterface $output): int { @@ -86,26 +84,29 @@ protected function execute(InputInterface $input, OutputInterface $output): int $testConfigArray = json_decode($testConfiguration, true); - // run tests not referenced in suites - $this->runTests($testConfigArray['tests'], $output); + if (isset($testConfigArray['tests'])) { + $this->runTests($testConfigArray['tests'], $output); + } - // run tests in suites - $this->runTestsInSuite($testConfigArray['suites'], $output); + if (isset($testConfigArray['suites'])) { + $this->runTestsInSuite($testConfigArray['suites'], $output); + } return $this->returnCode; - } /** * Run tests not referenced in suites - * @param array $testsConfig + * + * @param array $tests * @param OutputInterface $output + * @return void * @throws TestFrameworkException + * + * @SuppressWarnings(PHPMD.UnusedLocalVariable) */ - private function runTests($testsConfig, OutputInterface $output) { - - - $tests = $testsConfig ?? []; + private function runTests(array $tests, OutputInterface $output) + { $codeceptionCommand = realpath(PROJECT_ROOT . '/vendor/bin/codecept') . ' run functional '; $testsDirectory = TESTS_MODULE_PATH . DIRECTORY_SEPARATOR . @@ -135,17 +136,19 @@ private function runTests($testsConfig, OutputInterface $output) { /** * Run tests referenced in suites within suites' context. + * * @param array $suitesConfig * @param OutputInterface $output + * @return void + * + * @SuppressWarnings(PHPMD.UnusedLocalVariable) */ - private function runTestsInSuite($suitesConfig, OutputInterface $output) { - - $suites = $suitesConfig ?? []; + private function runTestsInSuite(array $suitesConfig, OutputInterface $output) + { $codeceptionCommand = realpath(PROJECT_ROOT . '/vendor/bin/codecept') . ' run functional --verbose --steps '; - $testGroups = array_keys($suites); //for tests in suites, run them as a group to run before and after block - foreach ($testGroups as $testGroup) { - $fullCommand = $codeceptionCommand . " -g {$testGroup}"; + foreach (array_keys($suitesConfig) as $suite) { + $fullCommand = $codeceptionCommand . " -g {$suite}"; $process = new Process($fullCommand); $process->setWorkingDirectory(TESTS_BP); $process->setIdleTimeout(600); From 8597fcb8313a022f83c9ff47a20dd9c421fbdb4c Mon Sep 17 00:00:00 2001 From: Soumya Unnikrishnan Date: Tue, 10 Sep 2019 19:33:09 -0500 Subject: [PATCH 3/5] MQE-1755: mftf run:test Test1 Test2 does not run before/after hooks correctly fix unit tests --- .../FunctionalTestingFramework/Console/RunTestCommand.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Magento/FunctionalTestingFramework/Console/RunTestCommand.php b/src/Magento/FunctionalTestingFramework/Console/RunTestCommand.php index 668d5152c..dc763b9b0 100644 --- a/src/Magento/FunctionalTestingFramework/Console/RunTestCommand.php +++ b/src/Magento/FunctionalTestingFramework/Console/RunTestCommand.php @@ -98,7 +98,7 @@ protected function execute(InputInterface $input, OutputInterface $output): int /** * Run tests not referenced in suites * - * @param array $tests + * @param array $tests * @param OutputInterface $output * @return void * @throws TestFrameworkException @@ -137,7 +137,7 @@ private function runTests(array $tests, OutputInterface $output) /** * Run tests referenced in suites within suites' context. * - * @param array $suitesConfig + * @param array $suitesConfig * @param OutputInterface $output * @return void * From 9cfeb9c71b573482f18ce5f51ebe342bdd861e90 Mon Sep 17 00:00:00 2001 From: Soumya Unnikrishnan Date: Wed, 11 Sep 2019 13:42:23 -0500 Subject: [PATCH 4/5] MQE-1755: mftf run:test Test1 Test2 does not run before/after hooks correctly --- .../Console/RunTestCommand.php | 42 ++++++++++--------- 1 file changed, 22 insertions(+), 20 deletions(-) diff --git a/src/Magento/FunctionalTestingFramework/Console/RunTestCommand.php b/src/Magento/FunctionalTestingFramework/Console/RunTestCommand.php index dc763b9b0..b65409b70 100644 --- a/src/Magento/FunctionalTestingFramework/Console/RunTestCommand.php +++ b/src/Magento/FunctionalTestingFramework/Console/RunTestCommand.php @@ -102,8 +102,6 @@ protected function execute(InputInterface $input, OutputInterface $output): int * @param OutputInterface $output * @return void * @throws TestFrameworkException - * - * @SuppressWarnings(PHPMD.UnusedLocalVariable) */ private function runTests(array $tests, OutputInterface $output) { @@ -123,14 +121,7 @@ private function runTests(array $tests, OutputInterface $output) ); } $fullCommand = $codeceptionCommand . $testsDirectory . $testName . ' --verbose --steps'; - $process = new Process($fullCommand); - $process->setWorkingDirectory(TESTS_BP); - $process->setIdleTimeout(600); - $process->setTimeout(0); - $subReturnCode = $process->run(function ($type, $buffer) use ($output) { - $output->write($buffer); - }); - $this->returnCode = max($this->returnCode, $subReturnCode); + $this->returnCode = max($this->returnCode, $this->executeTestCommand($fullCommand, $output)); } } @@ -140,8 +131,6 @@ private function runTests(array $tests, OutputInterface $output) * @param array $suitesConfig * @param OutputInterface $output * @return void - * - * @SuppressWarnings(PHPMD.UnusedLocalVariable) */ private function runTestsInSuite(array $suitesConfig, OutputInterface $output) { @@ -149,14 +138,27 @@ private function runTestsInSuite(array $suitesConfig, OutputInterface $output) //for tests in suites, run them as a group to run before and after block foreach (array_keys($suitesConfig) as $suite) { $fullCommand = $codeceptionCommand . " -g {$suite}"; - $process = new Process($fullCommand); - $process->setWorkingDirectory(TESTS_BP); - $process->setIdleTimeout(600); - $process->setTimeout(0); - $subReturnCode = $process->run(function ($type, $buffer) use ($output) { - $output->write($buffer); - }); - $this->returnCode = max($this->returnCode, $subReturnCode); + $this->returnCode = max($this->returnCode, $this->executeTestCommand($fullCommand, $output)); } } + + /** + * Runs the codeception test command and returns exit code + * + * @param String $command + * @param OutputInterface $output + * @return int + * + * @SuppressWarnings(PHPMD.UnusedLocalVariable) + */ + private function executeTestCommand(String $command, OutputInterface $output) + { + $process = new Process($command); + $process->setWorkingDirectory(TESTS_BP); + $process->setIdleTimeout(600); + $process->setTimeout(0); + return $process->run(function ($type, $buffer) use ($output) { + $output->write($buffer); + }); + } } From 6a5798ed082a73372cf06b37f3d2168eb4835333 Mon Sep 17 00:00:00 2001 From: Soumya Unnikrishnan Date: Wed, 11 Sep 2019 13:57:27 -0500 Subject: [PATCH 5/5] MQE-1755: mftf run:test Test1 Test2 does not run before/after hooks correctly fixed unit test failures --- .../FunctionalTestingFramework/Console/RunTestCommand.php | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Magento/FunctionalTestingFramework/Console/RunTestCommand.php b/src/Magento/FunctionalTestingFramework/Console/RunTestCommand.php index b65409b70..c848a3010 100644 --- a/src/Magento/FunctionalTestingFramework/Console/RunTestCommand.php +++ b/src/Magento/FunctionalTestingFramework/Console/RunTestCommand.php @@ -145,13 +145,13 @@ private function runTestsInSuite(array $suitesConfig, OutputInterface $output) /** * Runs the codeception test command and returns exit code * - * @param String $command + * @param string $command * @param OutputInterface $output - * @return int + * @return integer * * @SuppressWarnings(PHPMD.UnusedLocalVariable) */ - private function executeTestCommand(String $command, OutputInterface $output) + private function executeTestCommand(string $command, OutputInterface $output) { $process = new Process($command); $process->setWorkingDirectory(TESTS_BP);