MQE-1541: Add option to generate:tests for XSD validation on 'merged files'

dev/tests/_bootstrap.php

@@ -32,7 +32,7 @@
     true,
     \Magento\FunctionalTestingFramework\Config\MftfApplicationConfig::UNIT_TEST_PHASE,
     true,
-    false
+    \Magento\FunctionalTestingFramework\Config\MftfApplicationConfig::DEBUG_NONE
 );
 
 // Load needed framework env params

dev/tests/verification/Resources/BasicFunctionalTest.txt

@@ -60,9 +60,7 @@ class BasicFunctionalTestCest
 	{
 		$I->comment("");
 		$I->comment("");
-		$I->skipReadinessCheck(true);
-		$I->comment("skipReadiness");
-		$I->skipReadinessCheck(false);
+		$I->comment("seeComment");
 		$someVarDefinition = $I->grabValueFrom();
 		$I->acceptPopup();
 		$I->amOnPage("/test/url");

dev/tests/verification/TestModule/ActionGroup/BasicActionGroup.xml

@@ -115,10 +115,6 @@
         <see selector=".selector" userInput="{{sameStepKeyAsArg}}" stepKey="arg1" />
     </actionGroup>
 
-    <actionGroup name="actionGroupWithSkipReadinessActions">
-        <comment userInput="ActionGroupSkipReadiness" stepKey="skip" skipReadiness="true"/>
-    </actionGroup>
-
     <actionGroup name="actionGroupWithSectionAndData">
         <arguments>
             <argument name="content" type="string"/>

dev/tests/verification/TestModule/Test/BasicFunctionalTest.xml

@@ -24,7 +24,7 @@
         </after>
         <comment stepKey="basicCommentWithNoData" userInput="{{emptyData.noData}}"/>
         <comment stepKey="basicCommentWithDefinitelyNoData" userInput="{{emptyData.definitelyNoData}}"/>
-        <comment stepKey="ReadinessCheckSkipped" userInput="skipReadiness" skipReadiness="true"/>
+        <comment stepKey="basicCommentWithData" userInput="seeComment"/>
         <grabValueFrom stepKey="someVarDefinition"/>
         <acceptPopup stepKey="acceptPopupKey1"/>
         <amOnPage stepKey="amOnPageKey1" url="/test/url"/>

dev/tests/verification/Tests/ActionGroupGenerationTest.php

@@ -185,17 +185,6 @@ public function testActionGroupWithArgContainingStepKey()
         $this->generateAndCompareTest('ActionGroupContainsStepKeyInArgText');
     }
 
-    /**
-     * Test an action group with an arg containing stepKey text
-     *
-     * @throws \Exception
-     * @throws \Magento\FunctionalTestingFramework\Exceptions\TestReferenceException
-     */
-    public function testActionGroupWithSkipReadiness()
-    {
-        $this->generateAndCompareTest('ActionGroupSkipReadiness');
-    }
-
     /**
      * Test an action group with an arg containing stepKey text
      *

dev/tests/verification/Tests/SchemaValidationTest.php

@@ -19,7 +19,7 @@ class SchemaValidationTest extends MftfTestCase
      */
     public function testInvalidTestSchema()
     {
-        AspectMock::double(MftfApplicationConfig::class, ['debugEnabled' => true]);
+        AspectMock::double(MftfApplicationConfig::class, ['getDebugLevel' => MftfApplicationConfig::LEVEL_DEVELOPER]);
         $testFile = ['testFile.xml' => "<tests><test name='testName'><annotations>a</annotations></test></tests>"];
         $expectedError = TESTS_MODULE_PATH .
             DIRECTORY_SEPARATOR .

src/Magento/FunctionalTestingFramework/Config/MftfApplicationConfig.php

@@ -14,6 +14,14 @@ class MftfApplicationConfig
     const UNIT_TEST_PHASE = "testing";
     const MFTF_PHASES = [self::GENERATION_PHASE, self::EXECUTION_PHASE, self::UNIT_TEST_PHASE];
 
+    /**
+     * Mftf debug levels
+     */
+    const LEVEL_DEFAULT = "default";
+    const LEVEL_DEVELOPER = "developer";
+    const DEBUG_NONE = "none";
+    const MFTF_DEBUG_LEVEL = [self::LEVEL_DEFAULT, self::LEVEL_DEVELOPER, self::DEBUG_NONE];
+
     /**
      * Determines whether the user has specified a force option for generation
      *
@@ -36,11 +44,11 @@ class MftfApplicationConfig
     private $verboseEnabled;
 
     /**
-     * Determines whether the user would like to execute mftf in a verbose run.
+     * String which identifies the current debug level of mftf execution
      *
-     * @var boolean
+     * @var string
      */
-    private $debugEnabled;
+    private $debugLevel;
 
     /**
      * MftfApplicationConfig Singelton Instance
@@ -55,14 +63,14 @@ class MftfApplicationConfig
      * @param boolean $forceGenerate
      * @param string  $phase
      * @param boolean $verboseEnabled
-     * @param boolean $debugEnabled
+     * @param string  $debugLevel
      * @throws TestFrameworkException
      */
     private function __construct(
         $forceGenerate = false,
         $phase = self::EXECUTION_PHASE,
         $verboseEnabled = null,
-        $debugEnabled = null
+        $debugLevel = self::DEBUG_NONE
     ) {
         $this->forceGenerate = $forceGenerate;
 
@@ -72,7 +80,15 @@ private function __construct(
 
         $this->phase = $phase;
         $this->verboseEnabled = $verboseEnabled;
-        $this->debugEnabled = $debugEnabled;
+        switch ($debugLevel) {
+            case self::LEVEL_DEVELOPER:
+            case self::LEVEL_DEFAULT:
+            case self::DEBUG_NONE:
+                $this->debugLevel = $debugLevel;
+                break;
+            default:
+                $this->debugLevel = self::LEVEL_DEVELOPER;
+        }
     }
 
     /**
@@ -82,14 +98,14 @@ private function __construct(
      * @param boolean $forceGenerate
      * @param string  $phase
      * @param boolean $verboseEnabled
-     * @param boolean $debugEnabled
+     * @param string  $debugLevel
      * @return void
      */
-    public static function create($forceGenerate, $phase, $verboseEnabled, $debugEnabled)
+    public static function create($forceGenerate, $phase, $verboseEnabled, $debugLevel)
     {
         if (self::$MFTF_APPLICATION_CONTEXT == null) {
             self::$MFTF_APPLICATION_CONTEXT =
-                new MftfApplicationConfig($forceGenerate, $phase, $verboseEnabled, $debugEnabled);
+                new MftfApplicationConfig($forceGenerate, $phase, $verboseEnabled, $debugLevel);
         }
     }
 
@@ -132,14 +148,13 @@ public function verboseEnabled()
     }
 
     /**
-     * Returns a boolean indicating whether the user has indicated a debug run, which will lengthy validation
-     * with some extra error messaging to be run
+     * Returns a string which indicates the debug level of mftf execution.
      *
-     * @return boolean
+     * @return string
      */
-    public function debugEnabled()
+    public function getDebugLevel()
     {
-        return $this->debugEnabled ?? getenv('MFTF_DEBUG');
+        return $this->debugLevel ?? getenv('MFTF_DEBUG');
     }
 
     /**

src/Magento/FunctionalTestingFramework/Config/Reader/Filesystem.php

@@ -157,7 +157,7 @@ protected function readFiles($fileList)
                 } else {
                     $configMerger->merge($content);
                 }
-                if (MftfApplicationConfig::getConfig()->debugEnabled()) {
+                if (MftfApplicationConfig::getConfig()->getDebugLevel() === MftfApplicationConfig::LEVEL_DEVELOPER) {
                     $this->validateSchema($configMerger, $fileList->getFilename());
                 }
             } catch (\Magento\FunctionalTestingFramework\Config\Dom\ValidationException $e) {
@@ -231,8 +231,14 @@ protected function validateSchema($configMerger, $filename = null)
         if ($this->validationState->isValidationRequired()) {
             $errors = [];
             if ($configMerger && !$configMerger->validate($this->schemaFile, $errors)) {
-                $message = $filename ? $filename . PHP_EOL . "Invalid Document \n" : PHP_EOL . "Invalid Document \n";
-                throw new \Exception($message . implode("\n", $errors));
+                foreach ($errors as $error) {
+                    $error = str_replace(PHP_EOL, "", $error);
+                    LoggingUtil::getInstance()->getLogger(Filesystem::class)->criticalFailure(
+                        "Schema validation error ",
+                        ($filename ? [ "file"=> $filename, "error" => $error]: ["error" => $error])
+                    );
+                }
+                throw new \Exception("Schema validation errors found in xml file(s)" . $filename);
             }
         }
     }

src/Magento/FunctionalTestingFramework/Config/Reader/MftfFilesystem.php

@@ -24,6 +24,7 @@ public function readFiles($fileList)
         $exceptionCollector = new ExceptionCollector();
         /** @var \Magento\FunctionalTestingFramework\Test\Config\Dom $configMerger */
         $configMerger = null;
+        $debugLevel = MftfApplicationConfig::getConfig()->getDebugLevel();
         foreach ($fileList as $key => $content) {
             //check if file is empty and continue to next if it is
             if (!parent::verifyFileEmpty($content, $fileList->getFilename())) {
@@ -40,16 +41,19 @@ public function readFiles($fileList)
                 } else {
                     $configMerger->merge($content, $fileList->getFilename(), $exceptionCollector);
                 }
-                if (MftfApplicationConfig::getConfig()->debugEnabled()) {
+                 // run per file validation with generate:tests -d
+                if ($debugLevel == MftfApplicationConfig::LEVEL_DEVELOPER) {
                     $this->validateSchema($configMerger, $fileList->getFilename());
                 }
             } catch (\Magento\FunctionalTestingFramework\Config\Dom\ValidationException $e) {
                 throw new \Exception("Invalid XML in file " . $key . ":\n" . $e->getMessage());
             }
         }
         $exceptionCollector->throwException();
-        if ($fileList->valid()) {
-            $this->validateSchema($configMerger, $fileList->getFilename());
+
+         //run validation on merged file with generate:tests
+        if ($debugLevel == MftfApplicationConfig::LEVEL_DEFAULT) {
+            $this->validateSchema($configMerger);
         }
 
         $output = [];

src/Magento/FunctionalTestingFramework/Console/GenerateDocsCommand.php

@@ -67,7 +67,7 @@ protected function execute(InputInterface $input, OutputInterface $output)
             $force,
             MftfApplicationConfig::GENERATION_PHASE,
             false,
-            false
+            MftfApplicationConfig::DEBUG_NONE
         );
 
         $allActionGroups = ActionGroupObjectHandler::getInstance()->getAllObjects();

src/Magento/FunctionalTestingFramework/Console/GenerateTestsCommand.php

@@ -29,7 +29,7 @@ class GenerateTestsCommand extends BaseGenerateCommand
     protected function configure()
     {
         $this->setName('generate:tests')
-            ->setDescription('This command generates all test files and suites based on xml declarations')
+            ->setDescription('Run validation and generate all test files and suites based on xml declarations')
             ->addArgument(
                 'name',
                 InputArgument::OPTIONAL | InputArgument::IS_ARRAY,
@@ -39,7 +39,7 @@ protected function configure()
                 "force",
                 'f',
                 InputOption::VALUE_NONE,
-                'force generation of tests regardless of Magento Instance Configuration'
+                'Force generation of tests regardless of Magento Instance Configuration'
             )->addOption(
                 'time',
                 'i',
@@ -54,8 +54,10 @@ protected function configure()
             )->addOption(
                 'debug',
                 'd',
-                InputOption::VALUE_NONE,
-                'run extra validation when generating tests'
+                InputOption::VALUE_OPTIONAL,
+                'Run extra validation when generating tests. Use option \'none\' to turn off debugging -- 
+                 added for backward compatibility, will be removed in the next MAJOR release',
+                'default'
             );
 
         parent::configure();
@@ -78,9 +80,8 @@ protected function execute(InputInterface $input, OutputInterface $output)
         $json = $input->getOption('tests');
         $force = $input->getOption('force');
         $time = $input->getOption('time') * 60 * 1000; // convert from minutes to milliseconds
-        $debug = $input->getOption('debug');
+        $debug = $input->getOption('debug') ?? MftfApplicationConfig::LEVEL_DEVELOPER; // for backward compatibility
         $remove = $input->getOption('remove');
-
         $verbose = $output->isVerbose();
 
         if ($json !== null && !json_decode($json)) {
@@ -95,7 +96,8 @@ protected function execute(InputInterface $input, OutputInterface $output)
 
         // Remove previous GENERATED_DIR if --remove option is used
         if ($remove) {
-            $this->removeGeneratedDirectory($output, $verbose || $debug);
+            $this->removeGeneratedDirectory($output, $verbose ||
+                ($debug !== MftfApplicationConfig::DEBUG_NONE));
         }
 
         $testConfiguration = $this->createTestConfiguration($json, $tests, $force, $debug, $verbose);
@@ -124,13 +126,13 @@ protected function execute(InputInterface $input, OutputInterface $output)
      * @param string  $json
      * @param array   $tests
      * @param boolean $force
-     * @param boolean $debug
+     * @param string  $debug
      * @param boolean $verbose
      * @return array
      * @throws \Magento\FunctionalTestingFramework\Exceptions\TestReferenceException
      * @throws \Magento\FunctionalTestingFramework\Exceptions\XmlException
      */
-    private function createTestConfiguration($json, array $tests, bool $force, bool $debug, bool $verbose)
+    private function createTestConfiguration($json, array $tests, bool $force, $debug, bool $verbose)
     {
         // set our application configuration so we can references the user options in our framework
         MftfApplicationConfig::create(

src/Magento/FunctionalTestingFramework/Console/RunTestCommand.php

@@ -7,6 +7,7 @@
 
 namespace Magento\FunctionalTestingFramework\Console;
 
+use Magento\FunctionalTestingFramework\Config\MftfApplicationConfig;
 use Symfony\Component\Console\Input\ArrayInput;
 use Symfony\Component\Console\Input\InputArgument;
 use Symfony\Component\Console\Input\InputInterface;
@@ -73,7 +74,8 @@ protected function execute(InputInterface $input, OutputInterface $output): int
                     'suites' => null
                 ]),
                 '--force' => $force,
-                '--remove' => $remove
+                '--remove' => $remove,
+                '--debug' => MftfApplicationConfig::DEBUG_NONE
             ];
             $command->run(new ArrayInput($args), $output);
         }

src/Magento/FunctionalTestingFramework/Console/RunTestFailedCommand.php

@@ -72,7 +72,7 @@ protected function execute(InputInterface $input, OutputInterface $output): int
             false,
             MftfApplicationConfig::GENERATION_PHASE,
             false,
-            false
+            MftfApplicationConfig::DEBUG_NONE
         );
 
         $testConfiguration = $this->getFailedTestList();

src/Magento/FunctionalTestingFramework/Console/RunTestGroupCommand.php

@@ -77,7 +77,7 @@ protected function execute(InputInterface $input, OutputInterface $output): int
             $force,
             MftfApplicationConfig::GENERATION_PHASE,
             false,
-            false
+            MftfApplicationConfig::DEBUG_NONE
         );
 
         if (!$skipGeneration) {

src/Magento/FunctionalTestingFramework/StaticCheck/TestDependencyCheck.php

@@ -75,7 +75,7 @@ public function execute(InputInterface $input)
             true,
             MftfApplicationConfig::UNIT_TEST_PHASE,
             false,
-            false
+            MftfApplicationConfig::DEBUG_NONE
         );
 
         ModuleResolver::getInstance()->getModulesPath();

src/Magento/FunctionalTestingFramework/Test/etc/Actions/commonAttributes.xsd

@@ -31,13 +31,6 @@
             </xs:annotation>
         </xs:attribute>
 
-        <xs:attribute type="xs:boolean" name="skipReadiness">
-            <xs:annotation>
-                <xs:documentation>
-                    Flag for skipping readiness check
-                </xs:documentation>
-            </xs:annotation>
-        </xs:attribute>
     </xs:attributeGroup>
 
     <xs:attribute type="xs:string" name="userInput" id="userInput">

src/Magento/FunctionalTestingFramework/Test/etc/mergedTestSchema.xsd

@@ -129,7 +129,13 @@
                 </xs:documentation>
             </xs:annotation>
         </xs:attribute>
+        <xs:attribute type="xs:boolean" name="skipReadiness" use="prohibited">
+            <xs:annotation>
+                <xs:documentation>
+                    Flag for skipping readiness check.
+                </xs:documentation>
+            </xs:annotation>
+        </xs:attribute>
         <xs:attributeGroup ref="commonActionAttributes"/>
-        <xs:attribute type="xs:boolean" name="skipReadiness" use="prohibited"/>
     </xs:complexType>
 </xs:schema>