Skip to content

Commit 9bafd56

Browse files
aljcalandraaljcalandra
authored
Merge pull request #167 from magento/MQE-1068
MQE-1068: Require Issue ID for Skipped Test
2 parents bf22ecb + cb17300 commit 9bafd56

15 files changed

+316
-3
lines changed

‎dev/tests/verification/Resources/ChildExtendedTestNoParent.txt

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,6 @@ class ChildExtendedTestNoParentCest
3434
*/
3535
public function ChildExtendedTestNoParent(AcceptanceTester $I, \Codeception\Scenario $scenario)
3636
{
37-
$scenario->skip("This test is skipped");
37+
$scenario->skip("This test is skipped due to the following issues:\nNo issues have been specified.");
3838
}
3939
}

‎dev/tests/verification/Resources/MergeSkip.txt

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
<?php
2+
namespace Magento\AcceptanceTest\_default\Backend;
3+
4+
use Magento\FunctionalTestingFramework\AcceptanceTester;
5+
use Magento\FunctionalTestingFramework\DataGenerator\Handlers\DataObjectHandler;
6+
use Magento\FunctionalTestingFramework\DataGenerator\Persist\DataPersistenceHandler;
7+
use Magento\FunctionalTestingFramework\DataGenerator\Objects\EntityDataObject;
8+
use Magento\FunctionalTestingFramework\DataGenerator\Handlers\CredentialStore;
9+
use \Codeception\Util\Locator;
10+
use Yandex\Allure\Adapter\Annotation\Features;
11+
use Yandex\Allure\Adapter\Annotation\Stories;
12+
use Yandex\Allure\Adapter\Annotation\Title;
13+
use Yandex\Allure\Adapter\Annotation\Description;
14+
use Yandex\Allure\Adapter\Annotation\Parameter;
15+
use Yandex\Allure\Adapter\Annotation\Severity;
16+
use Yandex\Allure\Adapter\Model\SeverityLevel;
17+
use Yandex\Allure\Adapter\Annotation\TestCaseId;
18+
19+
/**
20+
*/
21+
class MergeSkipCest
22+
{
23+
/**
24+
* @Features({"TestModule"})
25+
* @Parameter(name = "AcceptanceTester", value="$I")
26+
* @param AcceptanceTester $I
27+
* @return void
28+
* @throws \Exception
29+
*/
30+
public function MergeSkip(AcceptanceTester $I, \Codeception\Scenario $scenario)
31+
{
32+
$scenario->skip("This test is skipped due to the following issues:\nIssue5");
33+
}
34+
}

‎dev/tests/verification/Resources/SkippedTest.txt

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
<?php
2+
namespace Magento\AcceptanceTest\_default\Backend;
3+
4+
use Magento\FunctionalTestingFramework\AcceptanceTester;
5+
use Magento\FunctionalTestingFramework\DataGenerator\Handlers\DataObjectHandler;
6+
use Magento\FunctionalTestingFramework\DataGenerator\Persist\DataPersistenceHandler;
7+
use Magento\FunctionalTestingFramework\DataGenerator\Objects\EntityDataObject;
8+
use Magento\FunctionalTestingFramework\DataGenerator\Handlers\CredentialStore;
9+
use \Codeception\Util\Locator;
10+
use Yandex\Allure\Adapter\Annotation\Features;
11+
use Yandex\Allure\Adapter\Annotation\Stories;
12+
use Yandex\Allure\Adapter\Annotation\Title;
13+
use Yandex\Allure\Adapter\Annotation\Description;
14+
use Yandex\Allure\Adapter\Annotation\Parameter;
15+
use Yandex\Allure\Adapter\Annotation\Severity;
16+
use Yandex\Allure\Adapter\Model\SeverityLevel;
17+
use Yandex\Allure\Adapter\Annotation\TestCaseId;
18+
19+
/**
20+
* @Title("skippedTest")
21+
* @Description("")
22+
*/
23+
class SkippedTestCest
24+
{
25+
/**
26+
* @Stories({"skipped"})
27+
* @Severity(level = SeverityLevel::MINOR)
28+
* @Features({"TestModule"})
29+
* @Parameter(name = "AcceptanceTester", value="$I")
30+
* @param AcceptanceTester $I
31+
* @return void
32+
* @throws \Exception
33+
*/
34+
public function SkippedTest(AcceptanceTester $I, \Codeception\Scenario $scenario)
35+
{
36+
$scenario->skip("This test is skipped due to the following issues:\nSkippedValue");
37+
}
38+
}

‎dev/tests/verification/Resources/SkippedTestNoIssues.txt

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
<?php
2+
namespace Magento\AcceptanceTest\_default\Backend;
3+
4+
use Magento\FunctionalTestingFramework\AcceptanceTester;
5+
use Magento\FunctionalTestingFramework\DataGenerator\Handlers\DataObjectHandler;
6+
use Magento\FunctionalTestingFramework\DataGenerator\Persist\DataPersistenceHandler;
7+
use Magento\FunctionalTestingFramework\DataGenerator\Objects\EntityDataObject;
8+
use Magento\FunctionalTestingFramework\DataGenerator\Handlers\CredentialStore;
9+
use \Codeception\Util\Locator;
10+
use Yandex\Allure\Adapter\Annotation\Features;
11+
use Yandex\Allure\Adapter\Annotation\Stories;
12+
use Yandex\Allure\Adapter\Annotation\Title;
13+
use Yandex\Allure\Adapter\Annotation\Description;
14+
use Yandex\Allure\Adapter\Annotation\Parameter;
15+
use Yandex\Allure\Adapter\Annotation\Severity;
16+
use Yandex\Allure\Adapter\Model\SeverityLevel;
17+
use Yandex\Allure\Adapter\Annotation\TestCaseId;
18+
19+
/**
20+
* @Title("skippedNoIssuesTest")
21+
* @Description("")
22+
* @group skip
23+
*/
24+
class SkippedTestNoIssuesCest
25+
{
26+
/**
27+
* @Stories({"skippedNo"})
28+
* @Severity(level = SeverityLevel::MINOR)
29+
* @Features({"TestModule"})
30+
* @Parameter(name = "AcceptanceTester", value="$I")
31+
* @param AcceptanceTester $I
32+
* @return void
33+
* @throws \Exception
34+
*/
35+
public function SkippedTestNoIssues(AcceptanceTester $I, \Codeception\Scenario $scenario)
36+
{
37+
$scenario->skip("This test is skipped due to the following issues:\nNo issues have been specified.");
38+
}
39+
}

‎dev/tests/verification/Resources/SkippedTestTwoIssues.txt

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
<?php
2+
namespace Magento\AcceptanceTest\_default\Backend;
3+
4+
use Magento\FunctionalTestingFramework\AcceptanceTester;
5+
use Magento\FunctionalTestingFramework\DataGenerator\Handlers\DataObjectHandler;
6+
use Magento\FunctionalTestingFramework\DataGenerator\Persist\DataPersistenceHandler;
7+
use Magento\FunctionalTestingFramework\DataGenerator\Objects\EntityDataObject;
8+
use Magento\FunctionalTestingFramework\DataGenerator\Handlers\CredentialStore;
9+
use \Codeception\Util\Locator;
10+
use Yandex\Allure\Adapter\Annotation\Features;
11+
use Yandex\Allure\Adapter\Annotation\Stories;
12+
use Yandex\Allure\Adapter\Annotation\Title;
13+
use Yandex\Allure\Adapter\Annotation\Description;
14+
use Yandex\Allure\Adapter\Annotation\Parameter;
15+
use Yandex\Allure\Adapter\Annotation\Severity;
16+
use Yandex\Allure\Adapter\Model\SeverityLevel;
17+
use Yandex\Allure\Adapter\Annotation\TestCaseId;
18+
19+
/**
20+
* @Title("skippedMultipleIssuesTest")
21+
* @Description("")
22+
*/
23+
class SkippedTestTwoIssuesCest
24+
{
25+
/**
26+
* @Stories({"skippedMultiple"})
27+
* @Severity(level = SeverityLevel::MINOR)
28+
* @Features({"TestModule"})
29+
* @Parameter(name = "AcceptanceTester", value="$I")
30+
* @param AcceptanceTester $I
31+
* @return void
32+
* @throws \Exception
33+
*/
34+
public function SkippedTestTwoIssues(AcceptanceTester $I, \Codeception\Scenario $scenario)
35+
{
36+
$scenario->skip("This test is skipped due to the following issues:\nSkippedValue\nSecondSkippedValue");
37+
}
38+
}

‎dev/tests/verification/TestModule/Test/MergeFunctionalTest.xml

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -55,4 +55,7 @@
5555
<click stepKey="clickTwo" selector="#mergeTwo"/>
5656
<click stepKey="clickThree" selector="#mergeThree"/>
5757
</test>
58+
<test name="MergeSkip">
59+
<comment userInput="ThisTestShouldBeSkipped" stepKey="skipComment"/>
60+
</test>
5861
</tests>

‎dev/tests/verification/TestModule/Test/SkippedTest.xml

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
<?xml version="1.0" encoding="UTF-8"?>
2+
<!--
3+
/**
4+
* Copyright © Magento, Inc. All rights reserved.
5+
* See COPYING.txt for license details.
6+
*/
7+
-->
8+
9+
<tests xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
10+
xsi:noNamespaceSchemaLocation="../../../../../src/Magento/FunctionalTestingFramework/Test/etc/testSchema.xsd">
11+
<test name="SkippedTest">
12+
<annotations>
13+
<stories value="skipped"/>
14+
<title value="skippedTest"/>
15+
<description value=""/>
16+
<severity value="AVERAGE"/>
17+
<skip>
18+
<issueId value="SkippedValue"/>
19+
</skip>
20+
</annotations>
21+
</test>
22+
<test name="SkippedTestTwoIssues">
23+
<annotations>
24+
<stories value="skippedMultiple"/>
25+
<title value="skippedMultipleIssuesTest"/>
26+
<description value=""/>
27+
<severity value="AVERAGE"/>
28+
<skip>
29+
<issueId value="SkippedValue"/>
30+
<issueId value="SecondSkippedValue"/>
31+
</skip>
32+
</annotations>
33+
</test>
34+
<test name="SkippedTestNoIssues">
35+
<annotations>
36+
<stories value="skippedNo"/>
37+
<title value="skippedNoIssuesTest"/>
38+
<description value=""/>
39+
<severity value="AVERAGE"/>
40+
<group value="skip"/>
41+
</annotations>
42+
</test>
43+
</tests>

‎dev/tests/verification/TestModuleMerged/Test/MergeFunctionalTest.xml

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,4 +26,11 @@
2626
<click stepKey="step10" selector="#step10MergedInResult"/>
2727
<actionGroup ref="FunctionalActionGroupWithData" stepKey="step8Merge" after="step7Merge"/>
2828
</test>
29+
<test name="MergeSkip">
30+
<annotations>
31+
<skip>
32+
<issueId value="Issue5"/>
33+
</skip>
34+
</annotations>
35+
</test>
2936
</tests>

‎dev/tests/verification/Tests/MergedGenerationTest.php

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -62,4 +62,15 @@ public function testMergeMassViaInsertAfter()
6262
{
6363
$this->generateAndCompareTest('MergeMassViaInsertAfter');
6464
}
65+
66+
/**
67+
* Tests generation of a test skipped in merge.
68+
*
69+
* @throws \Exception
70+
* @throws \Magento\FunctionalTestingFramework\Exceptions\TestReferenceException
71+
*/
72+
public function testMergeSkipGeneration()
73+
{
74+
$this->generateAndCompareTest('MergeSkip');
75+
}
6576
}

‎dev/tests/verification/Tests/SkippedGenerationTest.php

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
<?php
2+
/**
3+
* Copyright © Magento, Inc. All rights reserved.
4+
* See COPYING.txt for license details.
5+
*/
6+
namespace tests\verification\Tests;
7+
8+
use tests\util\MftfTestCase;
9+
10+
class SkippedGenerationTest extends MftfTestCase
11+
{
12+
/**
13+
* Tests skipped test generation.
14+
*
15+
* @throws \Exception
16+
* @throws \Magento\FunctionalTestingFramework\Exceptions\TestReferenceException
17+
*/
18+
public function testSkippedGeneration()
19+
{
20+
$this->generateAndCompareTest('SkippedTest');
21+
}
22+
23+
/**
24+
* Tests skipped test with multiple issues generation.
25+
*
26+
* @throws \Exception
27+
* @throws \Magento\FunctionalTestingFramework\Exceptions\TestReferenceException
28+
*/
29+
public function testMultipleSkippedIssuesGeneration()
30+
{
31+
$this->generateAndCompareTest('SkippedTestTwoIssues');
32+
}
33+
34+
/**
35+
* Tests skipped test generation with no specified issues. Will be deprecated after MFTF 3.0.0
36+
*
37+
* @throws \Exception
38+
* @throws \Magento\FunctionalTestingFramework\Exceptions\TestReferenceException
39+
*/
40+
public function testSkippedNoIssueGeneration()
41+
{
42+
$this->generateAndCompareTest('SkippedTestNoIssues');
43+
}
44+
}

‎etc/di.xml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -260,6 +260,7 @@
260260
<item name="/tests/test/annotations/group" xsi:type="string">/tests/test/annotations/group</item>
261261
<item name="/tests/test/annotations/env" xsi:type="string">/tests/test/annotations/env</item>
262262
<item name="/tests/test/annotations/return" xsi:type="string">/tests/test/annotations/return</item>
263+
<item name="/tests/test/annotations/skip/issueId" xsi:type="string">/tests/test/annotations/skip/issueId</item>
263264
</argument>
264265
</arguments>
265266
</virtualType>

‎src/Magento/FunctionalTestingFramework/Test/Objects/TestObject.php

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -128,7 +128,10 @@ public function getParentName()
128128
*/
129129
public function isSkipped()
130130
{
131-
if (array_key_exists('group', $this->annotations) && (in_array("skip", $this->annotations['group']))) {
131+
// TODO deprecation|deprecate MFTF 3.0.0 remove elseif when group skip is no longer allowed
132+
if (array_key_exists('skip', $this->annotations)) {
133+
return true;
134+
} elseif (array_key_exists('group', $this->annotations) && (in_array("skip", $this->annotations['group']))) {
132135
return true;
133136
}
134137
return false;

‎src/Magento/FunctionalTestingFramework/Test/Util/AnnotationExtractor.php

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66

77
namespace Magento\FunctionalTestingFramework\Test\Util;
88

9+
use Magento\FunctionalTestingFramework\Config\MftfApplicationConfig;
910
use Magento\FunctionalTestingFramework\Exceptions\XmlException;
1011
use Magento\FunctionalTestingFramework\Util\Logger\LoggingUtil;
1112

@@ -51,6 +52,7 @@ public function __construct()
5152
* @param array $testAnnotations
5253
* @param string $filename
5354
* @return array
55+
* @throws XmlException
5456
*/
5557
public function extractAnnotations($testAnnotations, $filename)
5658
{
@@ -69,9 +71,22 @@ public function extractAnnotations($testAnnotations, $filename)
6971
continue;
7072
}
7173

74+
if ($annotationKey == "skip") {
75+
$annotationData = $annotationData['issueId'];
76+
$this->validateSkippedIssues($annotationData, $filename);
77+
}
78+
7279
foreach ($annotationData as $annotationValue) {
7380
$annotationValues[] = $annotationValue[self::ANNOTATION_VALUE];
7481
}
82+
// TODO deprecation|deprecate MFTF 3.0.0
83+
if ($annotationKey == "group" && in_array("skip", $annotationValues)) {
84+
LoggingUtil::getInstance()->getLogger(AnnotationExtractor::class)->warning(
85+
"Use of group skip will be deprecated in MFTF 3.0.0. Please update tests to use skip tags.",
86+
["test" => $filename]
87+
);
88+
}
89+
7590
$annotationObjects[$annotationKey] = $annotationValues;
7691
}
7792

@@ -147,6 +162,23 @@ public function validateStoryTitleUniqueness()
147162
}
148163
}
149164

165+
/**
166+
* Validates that all issueId tags contain a non-empty value
167+
* @param array $issues
168+
* @param string $filename
169+
* @throws XmlException
170+
* @return void
171+
*/
172+
public function validateSkippedIssues($issues, $filename)
173+
{
174+
foreach ($issues as $issueId) {
175+
if (empty($issueId['value'])) {
176+
$message = "issueId for skipped tests cannot be empty. Test: $filename";
177+
throw new XmlException($message);
178+
}
179+
}
180+
}
181+
150182
/**
151183
* This method transforms Magento severity values from Severity annotation
152184
* Returns Allure annotation value

‎src/Magento/FunctionalTestingFramework/Test/etc/mergedTestSchema.xsd

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@
2929
<xs:element type="annotationType" name="testCaseId" minOccurs="0" maxOccurs="unbounded"/>
3030
<xs:element type="annotationType" name="useCaseId" minOccurs="0" maxOccurs="unbounded"/>
3131
<xs:element type="annotationType" name="group" minOccurs="0" maxOccurs="unbounded"/>
32+
<xs:element type="skipType" name="skip" minOccurs="0" maxOccurs="unbounded"/>
3233
<xs:element type="annotationType" name="return" minOccurs="0" maxOccurs="unbounded"/>
3334
</xs:choice>
3435
</xs:complexType>
@@ -55,6 +56,18 @@
5556
</xs:extension>
5657
</xs:simpleContent>
5758
</xs:complexType>
59+
<xs:complexType name="skipType">
60+
<xs:sequence maxOccurs="unbounded">
61+
<xs:element type="issueType" name="issueId" maxOccurs="unbounded"/>
62+
</xs:sequence>
63+
</xs:complexType>
64+
<xs:complexType name="issueType">
65+
<xs:simpleContent>
66+
<xs:extension base="xs:string">
67+
<xs:attribute type="xs:string" name="value" use="required"/>
68+
</xs:extension>
69+
</xs:simpleContent>
70+
</xs:complexType>
5871
<xs:complexType name="hookType">
5972
<xs:choice minOccurs="0" maxOccurs="unbounded">
6073
<xs:group ref="testTypeTags"/>

0 commit comments

Comments
 (0)