Skip to content

Commit 261ea1d

Browse files
aljcalandraaljcalandra
authored
Merge branch 'develop' into MQE-1031
2 parents c7200d3 + 22965f6 commit 261ea1d

File tree

6 files changed

+113
-58
lines changed

6 files changed

+113
-58
lines changed

‎dev/tests/unit/Magento/FunctionalTestFramework/Util/Sorter/ParallelGroupSorterTest.php

Lines changed: 18 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -36,9 +36,9 @@ public function testBasicTestGroupSort()
3636
$expectedResult = [
3737
1 => ['test2'],
3838
2 => ['test7'],
39-
3 => ['test6', 'test9'],
40-
4 => ['test1', 'test4', 'test3'],
41-
5 => ['test5', 'test10', 'test8']
39+
3 => ['test6', 'test4', 'test8'],
40+
4 => ['test1', 'test9'],
41+
5 => ['test3', 'test5', 'test10']
4242
];
4343

4444
$testSorter = new ParallelGroupSorter();
@@ -59,13 +59,16 @@ public function testSortWithSuites()
5959
{
6060
// mock tests for test object handler.
6161
$numberOfCalls = 0;
62-
$mockTest1 = AspectMock::double(TestObject::class, ['getTestActionCount' => function () use (&$numberOfCalls) {
63-
$actionCount = [200, 275];
64-
$result = $actionCount[$numberOfCalls];
65-
$numberOfCalls++;
66-
67-
return $result;
68-
}])->make();
62+
$mockTest1 = AspectMock::double(
63+
TestObject::class,
64+
['getEstimatedDuration' => function () use (&$numberOfCalls) {
65+
$actionCount = [300, 275];
66+
$result = $actionCount[$numberOfCalls];
67+
$numberOfCalls++;
68+
69+
return $result;
70+
}]
71+
)->make();
6972

7073
$mockHandler = AspectMock::double(
7174
TestObjectHandler::class,
@@ -92,17 +95,16 @@ public function testSortWithSuites()
9295

9396
// perform sort
9497
$testSorter = new ParallelGroupSorter();
95-
$actualResult = $testSorter->getTestsGroupedBySize($sampleSuiteArray, $sampleTestArray, 200);
98+
$actualResult = $testSorter->getTestsGroupedBySize($sampleSuiteArray, $sampleTestArray, 500);
9699

97100
// verify the resulting groups
98-
$this->assertCount(5, $actualResult);
101+
$this->assertCount(4, $actualResult);
99102

100103
$expectedResults = [
101104
1 => ['test3'],
102-
2 => ['test2'],
103-
3 => ['mockSuite1_0'],
104-
4 => ['mockSuite1_1'],
105-
5 => ['test5', 'test4', 'test1']
105+
2 => ['test2','test5', 'test4'],
106+
3 => ['mockSuite1_0', 'test1'],
107+
4 => ['mockSuite1_1']
106108
];
107109

108110
foreach ($actualResult as $groupNum => $group) {

‎src/Magento/FunctionalTestingFramework/Console/GenerateTestsCommand.php

Lines changed: 10 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,7 @@ protected function configure()
3535
->addArgument('name', InputArgument::OPTIONAL | InputArgument::IS_ARRAY, 'name(s) of specific tests to generate')
3636
->addOption("config", 'c', InputOption::VALUE_REQUIRED, 'default, singleRun, or parallel', 'default')
3737
->addOption("force", 'f',InputOption::VALUE_NONE, 'force generation of tests regardless of Magento Instance Configuration')
38-
->addOption('lines', 'l', InputOption::VALUE_REQUIRED, 'Used in combination with a parallel configuration, determines desired group size', 500)
38+
->addOption('time', 'i', InputOption::VALUE_REQUIRED, 'Used in combination with a parallel configuration, determines desired group size (in minutes)', 10)
3939
->addOption('tests', 't', InputOption::VALUE_REQUIRED, 'A parameter accepting a JSON string used to determine the test configuration');
4040
}
4141

@@ -45,22 +45,27 @@ protected function configure()
4545
* @param InputInterface $input
4646
* @param OutputInterface $output
4747
* @return void
48-
* @throws \Symfony\Component\Console\Exception\LogicException
48+
* @throws \Symfony\Component\Console\Exception\LogicException|TestFrameworkException
4949
*/
5050
protected function execute(InputInterface $input, OutputInterface $output)
5151
{
5252
$tests = $input->getArgument('name');
5353
$config = $input->getOption('config');
5454
$json = $input->getOption('tests');
5555
$force = $input->getOption('force');
56-
$lines = $input->getOption('lines');
56+
$time = $input->getOption('time') * 60 * 1000; // convert from minutes to milliseconds
5757
$verbose = $output->isVerbose();
5858

5959
if ($json !== null && !json_decode($json)) {
6060
// stop execution if we have failed to properly parse any json passed in by the user
6161
throw new TestFrameworkException("JSON could not be parsed: " . json_last_error_msg());
6262
}
6363

64+
if ($config === 'parallel' && $time <= 0) {
65+
// stop execution if the user has given us an invalid argument for time argument during parallel generation
66+
throw new TestFrameworkException("time option cannot be less than or equal to 0");
67+
}
68+
6469
$testConfiguration = $this->createTestConfiguration($json, $tests, $force, $verbose);
6570

6671
// create our manifest file here
@@ -69,13 +74,13 @@ protected function execute(InputInterface $input, OutputInterface $output)
6974

7075
if ($config == 'parallel') {
7176
/** @var ParallelTestManifest $testManifest */
72-
$testManifest->createTestGroups($lines);
77+
$testManifest->createTestGroups($time);
7378
}
7479

7580
SuiteGenerator::getInstance()->generateAllSuites($testManifest);
7681
$testManifest->generate();
7782

78-
print "Generate Tests Command Run" . PHP_EOL;
83+
$output->writeln("Generate Tests Command Run");
7984
}
8085

8186
/**

‎src/Magento/FunctionalTestingFramework/Test/Objects/TestObject.php

Lines changed: 48 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,18 @@
1515
*/
1616
class TestObject
1717
{
18+
const WAIT_TIME_ATTRIBUTE = 'time';
19+
20+
const TEST_ACTION_WEIGHT = [
21+
'waitForPageLoad' => 1500,
22+
'amOnPage' => 1000,
23+
'waitForLoadingMaskToDisappear' => 500,
24+
'wait' => self::WAIT_TIME_ATTRIBUTE,
25+
'comment' => 5,
26+
'assertCount' => 5,
27+
'closeAdminNotification' => 10
28+
];
29+
1830
/**
1931
* Name of the test
2032
*
@@ -159,28 +171,57 @@ public function getHooks()
159171
}
160172

161173
/**
162-
* Returns the number of a test actions contained within a single test (including before/after actions).
174+
* Returns the estimated duration of a single test (including before/after actions).
163175
*
164176
* @return int
165177
*/
166-
public function getTestActionCount()
178+
public function getEstimatedDuration()
167179
{
168180
// a skipped action results in a single skip being appended to the beginning of the test and no execution
169181
if ($this->isSkipped()) {
170182
return 1;
171183
}
172184

173-
$hookActions = 0;
185+
$hookTime = 0;
174186
if (array_key_exists('before', $this->hooks)) {
175-
$hookActions += count($this->hooks['before']->getActions());
187+
$hookTime += $this->calculateWeightedActionTimes($this->hooks['before']->getActions());
176188
}
177189

178190
if (array_key_exists('after', $this->hooks)) {
179-
$hookActions += count($this->hooks['after']->getActions());
191+
$hookTime += $this->calculateWeightedActionTimes($this->hooks['after']->getActions());
192+
}
193+
194+
$testTime = $this->calculateWeightedActionTimes($this->getOrderedActions());
195+
196+
return $hookTime + $testTime;
197+
}
198+
199+
/**
200+
* Function which takes a set of actions and estimates time for completion based on action type.
201+
*
202+
* @param ActionObject[] $actions
203+
* @return int
204+
*/
205+
private function calculateWeightedActionTimes($actions)
206+
{
207+
$actionTime = 0;
208+
// search for any actions of special type
209+
foreach ($actions as $action) {
210+
/** @var ActionObject $action */
211+
if (array_key_exists($action->getType(), self::TEST_ACTION_WEIGHT)) {
212+
$weight = self::TEST_ACTION_WEIGHT[$action->getType()];
213+
if ($weight === self::WAIT_TIME_ATTRIBUTE) {
214+
$weight = intval($action->getCustomActionAttributes()[$weight]) * 1000;
215+
}
216+
217+
$actionTime += $weight;
218+
continue;
219+
}
220+
221+
$actionTime += 50;
180222
}
181223

182-
$testActions = count($this->getOrderedActions());
183-
return $hookActions + $testActions;
224+
return $actionTime;
184225
}
185226

186227
/**

‎src/Magento/FunctionalTestingFramework/Util/Env/EnvProcessor.php

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -77,7 +77,7 @@ private function parseEnvFile(): array
7777
$envContents = $this->parseEnvFileLines($envFile);
7878
}
7979

80-
return array_diff_key($this->parseEnvFileLines($envExampleFile), $envContents);
80+
return array_merge($this->parseEnvFileLines($envExampleFile), $envContents);
8181
}
8282

8383
private function parseEnvFileLines(array $file): array

‎src/Magento/FunctionalTestingFramework/Util/Manifest/ParallelTestManifest.php

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -70,22 +70,22 @@ public function __construct($suiteConfiguration, $testPath)
7070
*/
7171
public function addTest($testObject)
7272
{
73-
$this->testNameToSize[$testObject->getCodeceptionName()] = $testObject->getTestActionCount();
73+
$this->testNameToSize[$testObject->getCodeceptionName()] = $testObject->getEstimatedDuration();
7474
}
7575

7676
/**
7777
* Function which generates test groups based on arg passed. The function builds groups using the args as an upper
7878
* limit.
7979
*
80-
* @param int $lines
80+
* @param int $time
8181
* @return void
8282
*/
83-
public function createTestGroups($lines)
83+
public function createTestGroups($time)
8484
{
8585
$this->testGroups = $this->parallelGroupSorter->getTestsGroupedBySize(
8686
$this->getSuiteConfig(),
8787
$this->testNameToSize,
88-
$lines
88+
$time
8989
);
9090

9191
$this->suiteConfiguration = $this->parallelGroupSorter->getResultingSuiteConfig();

‎src/Magento/FunctionalTestingFramework/Util/Sorter/ParallelGroupSorter.php

Lines changed: 32 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -30,22 +30,22 @@ public function __construct()
3030
*
3131
* @param array $suiteConfiguration
3232
* @param array $testNameToSize
33-
* @param integer $lines
33+
* @param integer $time
3434
* @return array
3535
* @throws TestFrameworkException
3636
*/
37-
public function getTestsGroupedBySize($suiteConfiguration, $testNameToSize, $lines)
37+
public function getTestsGroupedBySize($suiteConfiguration, $testNameToSize, $time)
3838
{
3939
// we must have the lines argument in order to create the test groups
40-
if ($lines == 0) {
40+
if ($time == 0) {
4141
throw new TestFrameworkException(
42-
"Please provide the argument '--lines' to the robo command in order to".
42+
"Please provide the argument '--time' to the robo command in order to".
4343
" generate grouped tests manifests for a parallel execution"
4444
);
4545
}
4646

4747
$testGroups = [];
48-
$splitSuiteNamesToTests = $this->createGroupsWithinSuites($suiteConfiguration, $lines);
48+
$splitSuiteNamesToTests = $this->createGroupsWithinSuites($suiteConfiguration, $time);
4949
$splitSuiteNamesToSize = $this->getSuiteToSize($splitSuiteNamesToTests);
5050
$entriesForGeneration = array_merge($testNameToSize, $splitSuiteNamesToSize);
5151
arsort($entriesForGeneration);
@@ -58,7 +58,7 @@ public function getTestsGroupedBySize($suiteConfiguration, $testNameToSize, $lin
5858
continue;
5959
}
6060

61-
$testGroup = $this->createTestGroup($lines, $testName, $testSize, $testNameToSizeForUse);
61+
$testGroup = $this->createTestGroup($time, $testName, $testSize, $testNameToSizeForUse);
6262
$testGroups[$nodeNumber] = $testGroup;
6363

6464
// unset the test which have been used.
@@ -88,26 +88,29 @@ public function getResultingSuiteConfig()
8888
* a test to be used as a starting point, the size of a starting test, an array of tests available to be added to
8989
* the group.
9090
*
91-
* @param integer $lineMaximum
91+
* @param integer $timeMaximum
9292
* @param string $testName
9393
* @param integer $testSize
9494
* @param array $testNameToSizeForUse
9595
* @return array
9696
*/
97-
private function createTestGroup($lineMaximum, $testName, $testSize, $testNameToSizeForUse)
97+
private function createTestGroup($timeMaximum, $testName, $testSize, $testNameToSizeForUse)
9898
{
9999
$group[$testName] = $testSize;
100100

101-
if ($testSize < $lineMaximum) {
102-
while (array_sum($group) < $lineMaximum && !empty($testNameToSizeForUse)) {
101+
if ($testSize < $timeMaximum) {
102+
while (array_sum($group) < $timeMaximum && !empty($testNameToSizeForUse)) {
103103
$groupSize = array_sum($group);
104-
$lineGoal = $lineMaximum - $groupSize;
104+
$lineGoal = $timeMaximum - $groupSize;
105105

106106
$testNameForUse = $this->getClosestLineCount($testNameToSizeForUse, $lineGoal);
107-
$testSizeForUse = $testNameToSizeForUse[$testNameForUse];
107+
if ($testNameToSizeForUse[$testNameForUse] < $lineGoal) {
108+
$testSizeForUse = $testNameToSizeForUse[$testNameForUse];
109+
$group[$testNameForUse] = $testSizeForUse;
110+
}
111+
108112
unset($testNameToSizeForUse[$testNameForUse]);
109113

110-
$group[$testNameForUse] = $testSizeForUse;
111114
}
112115
}
113116

@@ -127,8 +130,12 @@ private function getClosestLineCount($testGroup, $desiredValue)
127130
$winner = key($testGroup);
128131
$closestThreshold = $desiredValue;
129132
foreach ($testGroup as $testName => $testValue) {
130-
$testThreshold = abs($desiredValue - $testValue);
131-
if ($closestThreshold > $testThreshold) {
133+
// find the difference between the desired value and test candidate for the group
134+
$testThreshold = $desiredValue - $testValue;
135+
136+
// if we see that the gap between the desired value is non-negative and lower than the current closest make
137+
// the test the winner.
138+
if ($closestThreshold > $testThreshold && $testThreshold > 0) {
132139
$closestThreshold = $testThreshold;
133140
$winner = $testName;
134141
}
@@ -187,7 +194,7 @@ private function getSuiteNameToTestSize($suiteConfiguration)
187194
foreach ($test as $testName) {
188195
$suiteNameToTestSize[$suite][$testName] = TestObjectHandler::getInstance()
189196
->getObject($testName)
190-
->getTestActionCount();
197+
->getEstimatedDuration();
191198
}
192199
}
193200

@@ -224,30 +231,30 @@ private function getSuiteToSize($suiteNamesToTests)
224231
*
225232
* @param string $suiteName
226233
* @param array $tests
227-
* @param integer $lineLimit
234+
* @param integer $maxTime
228235
* @return array
229236
*/
230-
private function splitTestSuite($suiteName, $tests, $lineLimit)
237+
private function splitTestSuite($suiteName, $tests, $maxTime)
231238
{
232239
arsort($tests);
233-
$split_suites = [];
240+
$splitSuites = [];
234241
$availableTests = $tests;
235-
$split_count = 0;
242+
$splitCount = 0;
236243

237244
foreach ($tests as $test => $size) {
238245
if (!array_key_exists($test, $availableTests)) {
239246
continue;
240247
}
241248

242-
$group = $this->createTestGroup($lineLimit, $test, $size, $availableTests);
243-
$split_suites["{$suiteName}_${split_count}"] = $group;
244-
$this->addSuiteToConfig($suiteName, "{$suiteName}_${split_count}", $group);
249+
$group = $this->createTestGroup($maxTime, $test, $size, $availableTests);
250+
$splitSuites["{$suiteName}_${splitCount}"] = $group;
251+
$this->addSuiteToConfig($suiteName, "{$suiteName}_${splitCount}", $group);
245252

246253
$availableTests = array_diff_key($availableTests, $group);
247-
$split_count++;
254+
$splitCount++;
248255
}
249256

250-
return $split_suites;
257+
return $splitSuites;
251258
}
252259

253260
/**

0 commit comments

Comments
 (0)