MQE-1541: Add option to generate:tests for XSD validation on 'merged files'

dev/tests/_bootstrap.php

@@ -32,6 +32,7 @@
     true,
     \Magento\FunctionalTestingFramework\Config\MftfApplicationConfig::UNIT_TEST_PHASE,
     true,
+    false,
     false
 );
 

etc/config/.env.example

@@ -42,8 +42,9 @@ BROWSER=chrome
 MODULE_WHITELIST=Magento_Framework,Magento_ConfigurableProductWishlist,Magento_ConfigurableProductCatalogSearch
 #CUSTOM_MODULE_PATHS=
 
-#*** Bool property which allows the user to toggle debug output during test execution
+#*** Bool properties which allows the user to toggle debug output during test execution
 #MFTF_DEBUG=
+#MFTF_FAST_DEBUG=
 
 #*** Default timeout for wait actions
 #WAIT_TIMEOUT=10

src/Magento/FunctionalTestingFramework/Config/MftfApplicationConfig.php

@@ -36,12 +36,19 @@ class MftfApplicationConfig
     private $verboseEnabled;
 
     /**
-     * Determines whether the user would like to execute mftf in a verbose run.
+     * Determines whether the user would like to execute mftf in a 'per file' debug mode
      *
      * @var boolean
      */
     private $debugEnabled;
 
+    /**
+     * Determines whether the user would like to execute mftf in a 'merged file' debug mode
+     *
+     * @var boolean
+     */
+    private $fastDebugEnabled;
+
     /**
      * MftfApplicationConfig Singelton Instance
      *
@@ -53,16 +60,18 @@ class MftfApplicationConfig
      * MftfApplicationConfig constructor.
      *
      * @param boolean $forceGenerate
-     * @param string  $phase
+     * @param string $phase
      * @param boolean $verboseEnabled
      * @param boolean $debugEnabled
+     * @param null $fastDebugEnabled
      * @throws TestFrameworkException
      */
     private function __construct(
         $forceGenerate = false,
         $phase = self::EXECUTION_PHASE,
         $verboseEnabled = null,
-        $debugEnabled = null
+        $debugEnabled = null,
+        $fastDebugEnabled = null
     ) {
         $this->forceGenerate = $forceGenerate;
 
@@ -73,6 +82,7 @@ private function __construct(
         $this->phase = $phase;
         $this->verboseEnabled = $verboseEnabled;
         $this->debugEnabled = $debugEnabled;
+        $this->fastDebugEnabled = $fastDebugEnabled;
     }
 
     /**
@@ -83,13 +93,14 @@ private function __construct(
      * @param string  $phase
      * @param boolean $verboseEnabled
      * @param boolean $debugEnabled
+     * * @param boolean $fastDebugEnabled
      * @return void
      */
-    public static function create($forceGenerate, $phase, $verboseEnabled, $debugEnabled)
+    public static function create($forceGenerate, $phase, $verboseEnabled, $debugEnabled, $fastDebugEnabled)
     {
         if (self::$MFTF_APPLICATION_CONTEXT == null) {
             self::$MFTF_APPLICATION_CONTEXT =
-                new MftfApplicationConfig($forceGenerate, $phase, $verboseEnabled, $debugEnabled);
+                new MftfApplicationConfig($forceGenerate, $phase, $verboseEnabled, $debugEnabled, $fastDebugEnabled);
         }
     }
 
@@ -128,18 +139,29 @@ public function forceGenerateEnabled()
      */
     public function verboseEnabled()
     {
-        return $this->verboseEnabled ?? getenv('MFTF_DEBUG');
+        return $this->verboseEnabled ?? (strcasecmp(getenv('MFTF_DEBUG'),'true') == 0);
     }
 
     /**
-     * Returns a boolean indicating whether the user has indicated a debug run, which will lengthy validation
+     * Returns a boolean indicating whether the user has indicated a debug run, which will run lengthy validation
      * with some extra error messaging to be run
      *
      * @return boolean
      */
     public function debugEnabled()
     {
-        return $this->debugEnabled ?? getenv('MFTF_DEBUG');
+        return $this->debugEnabled ?? (strcasecmp(getenv('MFTF_DEBUG'),'true') == 0);
+    }
+
+    /**
+     * Returns a boolean indicating whether the user has indicated a fast debug run, which will run lengthy validation
+     * on merged file instead of 'per file' with some extra error messaging to be run
+     *
+     * @return boolean
+     */
+    public function fastDebugEnabled()
+    {
+        return $this->fastDebugEnabled ?? (strcasecmp(getenv('MFTF_FAST_DEBUG'),'true') == 0);
     }
 
     /**

src/Magento/FunctionalTestingFramework/Config/Reader/Filesystem.php

@@ -231,8 +231,13 @@ protected function validateSchema($configMerger, $filename = null)
         if ($this->validationState->isValidationRequired()) {
             $errors = [];
             if ($configMerger && !$configMerger->validate($this->schemaFile, $errors)) {
-                $message = $filename ? $filename . PHP_EOL . "Invalid Document \n" : PHP_EOL . "Invalid Document \n";
-                throw new \Exception($message . implode("\n", $errors));
+                foreach ($errors as $error){
+                    LoggingUtil::getInstance()->getLogger(Filesystem::class)->buildFailure(
+                        "XSD schema error ",
+                        [ "file"=> $filename ? $filename: ":mergedFile:", "error" => $error]
+                    );
+                }
+                throw new \Exception("Error: XSD schema issues found in file(s) " . $filename . "\n");
             }
         }
     }

src/Magento/FunctionalTestingFramework/Config/Reader/MftfFilesystem.php

@@ -51,6 +51,10 @@ public function readFiles($fileList)
         if ($fileList->valid()) {
             $this->validateSchema($configMerger, $fileList->getFilename());
         }
+        if (MftfApplicationConfig::getConfig()->fastDebugEnabled() &&
+            !MftfApplicationConfig::getConfig()->debugEnabled()) {
+            $this->validateSchema($configMerger);
+        }
 
         $output = [];
         if ($configMerger) {

src/Magento/FunctionalTestingFramework/Console/GenerateDocsCommand.php

@@ -67,6 +67,7 @@ protected function execute(InputInterface $input, OutputInterface $output)
             $force,
             MftfApplicationConfig::GENERATION_PHASE,
             false,
+            false,
             false
         );
 

src/Magento/FunctionalTestingFramework/Console/GenerateTestsCommand.php

@@ -55,7 +55,12 @@ protected function configure()
                 'debug',
                 'd',
                 InputOption::VALUE_NONE,
-                'run extra validation when generating tests'
+                'run extra validation per file when generating tests'
+            )->addOption(
+                'fastdebug',
+                'a',
+                InputOption::VALUE_NONE,
+                'run extra validation on merged files when generating tests'
             );
 
         parent::configure();
@@ -79,8 +84,8 @@ protected function execute(InputInterface $input, OutputInterface $output)
         $force = $input->getOption('force');
         $time = $input->getOption('time') * 60 * 1000; // convert from minutes to milliseconds
         $debug = $input->getOption('debug');
+        $fastDebug = $input->getOption('fastdebug');
         $remove = $input->getOption('remove');
-
         $verbose = $output->isVerbose();
 
         if ($json !== null && !json_decode($json)) {
@@ -95,10 +100,10 @@ protected function execute(InputInterface $input, OutputInterface $output)
 
         // Remove previous GENERATED_DIR if --remove option is used
         if ($remove) {
-            $this->removeGeneratedDirectory($output, $verbose || $debug);
+            $this->removeGeneratedDirectory($output, $verbose || $debug || $fastDebug);
         }
 
-        $testConfiguration = $this->createTestConfiguration($json, $tests, $force, $debug, $verbose);
+        $testConfiguration = $this->createTestConfiguration($json, $tests, $force, $debug, $fastDebug, $verbose);
 
         // create our manifest file here
         $testManifest = TestManifestFactory::makeManifest($config, $testConfiguration['suites']);
@@ -125,19 +130,21 @@ protected function execute(InputInterface $input, OutputInterface $output)
      * @param array   $tests
      * @param boolean $force
      * @param boolean $debug
+     * @param boolean $fastDebug
      * @param boolean $verbose
      * @return array
      * @throws \Magento\FunctionalTestingFramework\Exceptions\TestReferenceException
      * @throws \Magento\FunctionalTestingFramework\Exceptions\XmlException
      */
-    private function createTestConfiguration($json, array $tests, bool $force, bool $debug, bool $verbose)
+    private function createTestConfiguration($json, array $tests, bool $force, bool $debug, bool $fastDebug, bool $verbose)
     {
         // set our application configuration so we can references the user options in our framework
         MftfApplicationConfig::create(
             $force,
             MftfApplicationConfig::GENERATION_PHASE,
             $verbose,
-            $debug
+            $debug,
+            $fastDebug
         );
 
         $testConfiguration = [];

src/Magento/FunctionalTestingFramework/Console/RunTestFailedCommand.php

@@ -72,6 +72,7 @@ protected function execute(InputInterface $input, OutputInterface $output): int
             false,
             MftfApplicationConfig::GENERATION_PHASE,
             false,
+            false,
             false
         );
 

src/Magento/FunctionalTestingFramework/Console/RunTestGroupCommand.php

@@ -77,6 +77,7 @@ protected function execute(InputInterface $input, OutputInterface $output): int
             $force,
             MftfApplicationConfig::GENERATION_PHASE,
             false,
+            false,
             false
         );
 

src/Magento/FunctionalTestingFramework/StaticCheck/TestDependencyCheck.php

@@ -75,6 +75,7 @@ public function execute(InputInterface $input)
             true,
             MftfApplicationConfig::UNIT_TEST_PHASE,
             false,
+            false,
             false
         );
 

src/Magento/FunctionalTestingFramework/Util/Logger/MftfLogger.php

@@ -13,7 +13,7 @@
 class MftfLogger extends Logger
 {
     /**
-     * Prints a deprecation warning, as well as adding a log at the WARNING level.
+     * Prints a deprecation warning, as well as adds a log at the WARNING level.
      *
      * @param  string $message The log message.
      * @param  array  $context The log context.
@@ -28,4 +28,21 @@ public function deprecation($message, array $context = [])
         }
         parent::warning($message, $context);
     }
+
+    /**
+     * Prints a critical failure, as well as adds a log at the CRITICAL level.
+     *
+     * @param  string $message The log message.
+     * @param  array  $context The log context.
+     * @return void
+     */
+    public function buildFailure($message, array $context = [])
+    {
+        $message = "BUILD FAILURE: " . $message;
+        // Suppress print during unit testing
+        if (MftfApplicationConfig::getConfig()->getPhase() !== MftfApplicationConfig::UNIT_TEST_PHASE) {
+            print ($message . json_encode($context) . "\n");
+        }
+        parent::critical($message, $context);
+    }
 }