diff --git a/dev/tests/_bootstrap.php b/dev/tests/_bootstrap.php
index 31b4462f6..d9a4208b2 100644
--- a/dev/tests/_bootstrap.php
+++ b/dev/tests/_bootstrap.php
@@ -32,7 +32,7 @@
true,
\Magento\FunctionalTestingFramework\Config\MftfApplicationConfig::UNIT_TEST_PHASE,
true,
- false
+ \Magento\FunctionalTestingFramework\Config\MftfApplicationConfig::LEVEL_NONE
);
// Load needed framework env params
diff --git a/dev/tests/verification/Resources/BasicFunctionalTest.txt b/dev/tests/verification/Resources/BasicFunctionalTest.txt
index 9970e81ec..fd5f0f165 100644
--- a/dev/tests/verification/Resources/BasicFunctionalTest.txt
+++ b/dev/tests/verification/Resources/BasicFunctionalTest.txt
@@ -60,9 +60,7 @@ class BasicFunctionalTestCest
{
$I->comment("");
$I->comment("");
- $I->skipReadinessCheck(true);
- $I->comment("skipReadiness");
- $I->skipReadinessCheck(false);
+ $I->comment("seeComment");
$someVarDefinition = $I->grabValueFrom();
$I->acceptPopup();
$I->amOnPage("/test/url");
diff --git a/dev/tests/verification/TestModule/ActionGroup/BasicActionGroup.xml b/dev/tests/verification/TestModule/ActionGroup/BasicActionGroup.xml
index 245447f7d..45ccdbd24 100644
--- a/dev/tests/verification/TestModule/ActionGroup/BasicActionGroup.xml
+++ b/dev/tests/verification/TestModule/ActionGroup/BasicActionGroup.xml
@@ -115,10 +115,6 @@
-
-
-
-
diff --git a/dev/tests/verification/TestModule/Test/BasicFunctionalTest.xml b/dev/tests/verification/TestModule/Test/BasicFunctionalTest.xml
index d120e5c31..bdeb4e0c6 100644
--- a/dev/tests/verification/TestModule/Test/BasicFunctionalTest.xml
+++ b/dev/tests/verification/TestModule/Test/BasicFunctionalTest.xml
@@ -24,7 +24,7 @@
-
+
diff --git a/dev/tests/verification/Tests/ActionGroupGenerationTest.php b/dev/tests/verification/Tests/ActionGroupGenerationTest.php
index dec8c6d14..613d6cb72 100644
--- a/dev/tests/verification/Tests/ActionGroupGenerationTest.php
+++ b/dev/tests/verification/Tests/ActionGroupGenerationTest.php
@@ -185,17 +185,6 @@ public function testActionGroupWithArgContainingStepKey()
$this->generateAndCompareTest('ActionGroupContainsStepKeyInArgText');
}
- /**
- * Test an action group with an arg containing stepKey text
- *
- * @throws \Exception
- * @throws \Magento\FunctionalTestingFramework\Exceptions\TestReferenceException
- */
- public function testActionGroupWithSkipReadiness()
- {
- $this->generateAndCompareTest('ActionGroupSkipReadiness');
- }
-
/**
* Test an action group with an arg containing stepKey text
*
diff --git a/dev/tests/verification/Tests/SchemaValidationTest.php b/dev/tests/verification/Tests/SchemaValidationTest.php
index 86828e9a5..92442bd37 100644
--- a/dev/tests/verification/Tests/SchemaValidationTest.php
+++ b/dev/tests/verification/Tests/SchemaValidationTest.php
@@ -19,7 +19,7 @@ class SchemaValidationTest extends MftfTestCase
*/
public function testInvalidTestSchema()
{
- AspectMock::double(MftfApplicationConfig::class, ['debugEnabled' => true]);
+ AspectMock::double(MftfApplicationConfig::class, ['getDebugLevel' => MftfApplicationConfig::LEVEL_DEVELOPER]);
$testFile = ['testFile.xml' => "a"];
$expectedError = TESTS_MODULE_PATH .
DIRECTORY_SEPARATOR .
diff --git a/src/Magento/FunctionalTestingFramework/Config/MftfApplicationConfig.php b/src/Magento/FunctionalTestingFramework/Config/MftfApplicationConfig.php
index eeabdf117..29278f761 100644
--- a/src/Magento/FunctionalTestingFramework/Config/MftfApplicationConfig.php
+++ b/src/Magento/FunctionalTestingFramework/Config/MftfApplicationConfig.php
@@ -14,6 +14,14 @@ class MftfApplicationConfig
const UNIT_TEST_PHASE = "testing";
const MFTF_PHASES = [self::GENERATION_PHASE, self::EXECUTION_PHASE, self::UNIT_TEST_PHASE];
+ /**
+ * Mftf debug levels
+ */
+ const LEVEL_DEFAULT = "default";
+ const LEVEL_DEVELOPER = "developer";
+ const LEVEL_NONE = "none";
+ const MFTF_DEBUG_LEVEL = [self::LEVEL_DEFAULT, self::LEVEL_DEVELOPER, self::LEVEL_NONE];
+
/**
* Determines whether the user has specified a force option for generation
*
@@ -36,11 +44,11 @@ class MftfApplicationConfig
private $verboseEnabled;
/**
- * Determines whether the user would like to execute mftf in a verbose run.
+ * String which identifies the current debug level of mftf execution
*
- * @var boolean
+ * @var string
*/
- private $debugEnabled;
+ private $debugLevel;
/**
* MftfApplicationConfig Singelton Instance
@@ -55,14 +63,14 @@ class MftfApplicationConfig
* @param boolean $forceGenerate
* @param string $phase
* @param boolean $verboseEnabled
- * @param boolean $debugEnabled
+ * @param string $debugLevel
* @throws TestFrameworkException
*/
private function __construct(
$forceGenerate = false,
$phase = self::EXECUTION_PHASE,
$verboseEnabled = null,
- $debugEnabled = null
+ $debugLevel = self::LEVEL_NONE
) {
$this->forceGenerate = $forceGenerate;
@@ -72,7 +80,15 @@ private function __construct(
$this->phase = $phase;
$this->verboseEnabled = $verboseEnabled;
- $this->debugEnabled = $debugEnabled;
+ switch ($debugLevel) {
+ case self::LEVEL_DEVELOPER:
+ case self::LEVEL_DEFAULT:
+ case self::LEVEL_NONE:
+ $this->debugLevel = $debugLevel;
+ break;
+ default:
+ $this->debugLevel = self::LEVEL_DEVELOPER;
+ }
}
/**
@@ -82,14 +98,14 @@ private function __construct(
* @param boolean $forceGenerate
* @param string $phase
* @param boolean $verboseEnabled
- * @param boolean $debugEnabled
+ * @param string $debugLevel
* @return void
*/
- public static function create($forceGenerate, $phase, $verboseEnabled, $debugEnabled)
+ public static function create($forceGenerate, $phase, $verboseEnabled, $debugLevel)
{
if (self::$MFTF_APPLICATION_CONTEXT == null) {
self::$MFTF_APPLICATION_CONTEXT =
- new MftfApplicationConfig($forceGenerate, $phase, $verboseEnabled, $debugEnabled);
+ new MftfApplicationConfig($forceGenerate, $phase, $verboseEnabled, $debugLevel);
}
}
@@ -132,14 +148,13 @@ public function verboseEnabled()
}
/**
- * Returns a boolean indicating whether the user has indicated a debug run, which will lengthy validation
- * with some extra error messaging to be run
+ * Returns a string which indicates the debug level of mftf execution.
*
- * @return boolean
+ * @return string
*/
- public function debugEnabled()
+ public function getDebugLevel()
{
- return $this->debugEnabled ?? getenv('MFTF_DEBUG');
+ return $this->debugLevel ?? getenv('MFTF_DEBUG');
}
/**
diff --git a/src/Magento/FunctionalTestingFramework/Config/Reader/Filesystem.php b/src/Magento/FunctionalTestingFramework/Config/Reader/Filesystem.php
index 4c3aa581b..2f2cb8ad3 100644
--- a/src/Magento/FunctionalTestingFramework/Config/Reader/Filesystem.php
+++ b/src/Magento/FunctionalTestingFramework/Config/Reader/Filesystem.php
@@ -157,7 +157,7 @@ protected function readFiles($fileList)
} else {
$configMerger->merge($content);
}
- if (MftfApplicationConfig::getConfig()->debugEnabled()) {
+ if (MftfApplicationConfig::getConfig()->getDebugLevel() === MftfApplicationConfig::LEVEL_DEVELOPER) {
$this->validateSchema($configMerger, $fileList->getFilename());
}
} catch (\Magento\FunctionalTestingFramework\Config\Dom\ValidationException $e) {
@@ -231,8 +231,14 @@ protected function validateSchema($configMerger, $filename = null)
if ($this->validationState->isValidationRequired()) {
$errors = [];
if ($configMerger && !$configMerger->validate($this->schemaFile, $errors)) {
- $message = $filename ? $filename . PHP_EOL . "Invalid Document \n" : PHP_EOL . "Invalid Document \n";
- throw new \Exception($message . implode("\n", $errors));
+ foreach ($errors as $error) {
+ $error = str_replace(PHP_EOL, "", $error);
+ LoggingUtil::getInstance()->getLogger(Filesystem::class)->criticalFailure(
+ "Schema validation error ",
+ ($filename ? [ "file"=> $filename, "error" => $error]: ["error" => $error])
+ );
+ }
+ throw new \Exception("Schema validation errors found in xml file(s)" . $filename);
}
}
}
diff --git a/src/Magento/FunctionalTestingFramework/Config/Reader/MftfFilesystem.php b/src/Magento/FunctionalTestingFramework/Config/Reader/MftfFilesystem.php
index 728c4cb3a..6fbc2fdfc 100644
--- a/src/Magento/FunctionalTestingFramework/Config/Reader/MftfFilesystem.php
+++ b/src/Magento/FunctionalTestingFramework/Config/Reader/MftfFilesystem.php
@@ -24,6 +24,7 @@ public function readFiles($fileList)
$exceptionCollector = new ExceptionCollector();
/** @var \Magento\FunctionalTestingFramework\Test\Config\Dom $configMerger */
$configMerger = null;
+ $debugLevel = MftfApplicationConfig::getConfig()->getDebugLevel();
foreach ($fileList as $key => $content) {
//check if file is empty and continue to next if it is
if (!parent::verifyFileEmpty($content, $fileList->getFilename())) {
@@ -40,7 +41,8 @@ public function readFiles($fileList)
} else {
$configMerger->merge($content, $fileList->getFilename(), $exceptionCollector);
}
- if (MftfApplicationConfig::getConfig()->debugEnabled()) {
+ // run per file validation with generate:tests -d
+ if ($debugLevel == MftfApplicationConfig::LEVEL_DEVELOPER) {
$this->validateSchema($configMerger, $fileList->getFilename());
}
} catch (\Magento\FunctionalTestingFramework\Config\Dom\ValidationException $e) {
@@ -48,8 +50,10 @@ public function readFiles($fileList)
}
}
$exceptionCollector->throwException();
- if ($fileList->valid()) {
- $this->validateSchema($configMerger, $fileList->getFilename());
+
+ //run validation on merged file with generate:tests
+ if ($debugLevel == MftfApplicationConfig::LEVEL_DEFAULT) {
+ $this->validateSchema($configMerger);
}
$output = [];
diff --git a/src/Magento/FunctionalTestingFramework/Console/GenerateDocsCommand.php b/src/Magento/FunctionalTestingFramework/Console/GenerateDocsCommand.php
index db77bc74f..4ed2b6b29 100644
--- a/src/Magento/FunctionalTestingFramework/Console/GenerateDocsCommand.php
+++ b/src/Magento/FunctionalTestingFramework/Console/GenerateDocsCommand.php
@@ -67,7 +67,7 @@ protected function execute(InputInterface $input, OutputInterface $output)
$force,
MftfApplicationConfig::GENERATION_PHASE,
false,
- false
+ MftfApplicationConfig::LEVEL_NONE
);
$allActionGroups = ActionGroupObjectHandler::getInstance()->getAllObjects();
diff --git a/src/Magento/FunctionalTestingFramework/Console/GenerateTestsCommand.php b/src/Magento/FunctionalTestingFramework/Console/GenerateTestsCommand.php
index fcd977060..bc8c76142 100644
--- a/src/Magento/FunctionalTestingFramework/Console/GenerateTestsCommand.php
+++ b/src/Magento/FunctionalTestingFramework/Console/GenerateTestsCommand.php
@@ -29,7 +29,7 @@ class GenerateTestsCommand extends BaseGenerateCommand
protected function configure()
{
$this->setName('generate:tests')
- ->setDescription('This command generates all test files and suites based on xml declarations')
+ ->setDescription('Run validation and generate all test files and suites based on xml declarations')
->addArgument(
'name',
InputArgument::OPTIONAL | InputArgument::IS_ARRAY,
@@ -39,7 +39,7 @@ protected function configure()
"force",
'f',
InputOption::VALUE_NONE,
- 'force generation of tests regardless of Magento Instance Configuration'
+ 'Force generation of tests regardless of Magento Instance Configuration'
)->addOption(
'time',
'i',
@@ -54,8 +54,10 @@ protected function configure()
)->addOption(
'debug',
'd',
- InputOption::VALUE_NONE,
- 'run extra validation when generating tests'
+ InputOption::VALUE_OPTIONAL,
+ 'Run extra validation when generating tests. Use option \'none\' to turn off debugging --
+ added for backward compatibility, will be removed in the next MAJOR release',
+ 'default'
);
parent::configure();
@@ -78,9 +80,8 @@ protected function execute(InputInterface $input, OutputInterface $output)
$json = $input->getOption('tests');
$force = $input->getOption('force');
$time = $input->getOption('time') * 60 * 1000; // convert from minutes to milliseconds
- $debug = $input->getOption('debug');
+ $debug = $input->getOption('debug') ?? MftfApplicationConfig::LEVEL_DEVELOPER; // for backward compatibility
$remove = $input->getOption('remove');
-
$verbose = $output->isVerbose();
if ($json !== null && !json_decode($json)) {
@@ -95,7 +96,8 @@ protected function execute(InputInterface $input, OutputInterface $output)
// Remove previous GENERATED_DIR if --remove option is used
if ($remove) {
- $this->removeGeneratedDirectory($output, $verbose || $debug);
+ $this->removeGeneratedDirectory($output, $verbose ||
+ ($debug !== MftfApplicationConfig::LEVEL_NONE));
}
$testConfiguration = $this->createTestConfiguration($json, $tests, $force, $debug, $verbose);
@@ -124,13 +126,13 @@ protected function execute(InputInterface $input, OutputInterface $output)
* @param string $json
* @param array $tests
* @param boolean $force
- * @param boolean $debug
+ * @param string $debug
* @param boolean $verbose
* @return array
* @throws \Magento\FunctionalTestingFramework\Exceptions\TestReferenceException
* @throws \Magento\FunctionalTestingFramework\Exceptions\XmlException
*/
- private function createTestConfiguration($json, array $tests, bool $force, bool $debug, bool $verbose)
+ private function createTestConfiguration($json, array $tests, bool $force, $debug, bool $verbose)
{
// set our application configuration so we can references the user options in our framework
MftfApplicationConfig::create(
diff --git a/src/Magento/FunctionalTestingFramework/Console/RunTestCommand.php b/src/Magento/FunctionalTestingFramework/Console/RunTestCommand.php
index 33a29b2e8..bd9b4b99b 100644
--- a/src/Magento/FunctionalTestingFramework/Console/RunTestCommand.php
+++ b/src/Magento/FunctionalTestingFramework/Console/RunTestCommand.php
@@ -7,6 +7,7 @@
namespace Magento\FunctionalTestingFramework\Console;
+use Magento\FunctionalTestingFramework\Config\MftfApplicationConfig;
use Symfony\Component\Console\Input\ArrayInput;
use Symfony\Component\Console\Input\InputArgument;
use Symfony\Component\Console\Input\InputInterface;
@@ -73,7 +74,8 @@ protected function execute(InputInterface $input, OutputInterface $output): int
'suites' => null
]),
'--force' => $force,
- '--remove' => $remove
+ '--remove' => $remove,
+ '--debug' => MftfApplicationConfig::LEVEL_NONE
];
$command->run(new ArrayInput($args), $output);
}
diff --git a/src/Magento/FunctionalTestingFramework/Console/RunTestFailedCommand.php b/src/Magento/FunctionalTestingFramework/Console/RunTestFailedCommand.php
index 8257162b9..899fa8312 100644
--- a/src/Magento/FunctionalTestingFramework/Console/RunTestFailedCommand.php
+++ b/src/Magento/FunctionalTestingFramework/Console/RunTestFailedCommand.php
@@ -72,7 +72,7 @@ protected function execute(InputInterface $input, OutputInterface $output): int
false,
MftfApplicationConfig::GENERATION_PHASE,
false,
- false
+ MftfApplicationConfig::LEVEL_NONE
);
$testConfiguration = $this->getFailedTestList();
diff --git a/src/Magento/FunctionalTestingFramework/Console/RunTestGroupCommand.php b/src/Magento/FunctionalTestingFramework/Console/RunTestGroupCommand.php
index e895fb66a..174fd8130 100644
--- a/src/Magento/FunctionalTestingFramework/Console/RunTestGroupCommand.php
+++ b/src/Magento/FunctionalTestingFramework/Console/RunTestGroupCommand.php
@@ -77,7 +77,7 @@ protected function execute(InputInterface $input, OutputInterface $output): int
$force,
MftfApplicationConfig::GENERATION_PHASE,
false,
- false
+ MftfApplicationConfig::LEVEL_NONE
);
if (!$skipGeneration) {
diff --git a/src/Magento/FunctionalTestingFramework/StaticCheck/TestDependencyCheck.php b/src/Magento/FunctionalTestingFramework/StaticCheck/TestDependencyCheck.php
index e0b3cefd6..1a149a657 100644
--- a/src/Magento/FunctionalTestingFramework/StaticCheck/TestDependencyCheck.php
+++ b/src/Magento/FunctionalTestingFramework/StaticCheck/TestDependencyCheck.php
@@ -75,7 +75,7 @@ public function execute(InputInterface $input)
true,
MftfApplicationConfig::UNIT_TEST_PHASE,
false,
- false
+ MftfApplicationConfig::LEVEL_NONE
);
ModuleResolver::getInstance()->getModulesPath();
diff --git a/src/Magento/FunctionalTestingFramework/Test/etc/Actions/commonAttributes.xsd b/src/Magento/FunctionalTestingFramework/Test/etc/Actions/commonAttributes.xsd
index 22c05ea7d..c39d779ed 100644
--- a/src/Magento/FunctionalTestingFramework/Test/etc/Actions/commonAttributes.xsd
+++ b/src/Magento/FunctionalTestingFramework/Test/etc/Actions/commonAttributes.xsd
@@ -31,13 +31,6 @@
-
-
-
- Flag for skipping readiness check
-
-
-
diff --git a/src/Magento/FunctionalTestingFramework/Test/etc/mergedTestSchema.xsd b/src/Magento/FunctionalTestingFramework/Test/etc/mergedTestSchema.xsd
index b663bb99d..fe45945ee 100644
--- a/src/Magento/FunctionalTestingFramework/Test/etc/mergedTestSchema.xsd
+++ b/src/Magento/FunctionalTestingFramework/Test/etc/mergedTestSchema.xsd
@@ -129,7 +129,13 @@
+
+
+
+ Flag for skipping readiness check.
+
+
+
-
diff --git a/src/Magento/FunctionalTestingFramework/Util/Logger/MftfLogger.php b/src/Magento/FunctionalTestingFramework/Util/Logger/MftfLogger.php
index 5844858a0..0a52f6b6b 100644
--- a/src/Magento/FunctionalTestingFramework/Util/Logger/MftfLogger.php
+++ b/src/Magento/FunctionalTestingFramework/Util/Logger/MftfLogger.php
@@ -13,7 +13,7 @@
class MftfLogger extends Logger
{
/**
- * Prints a deprecation warning, as well as adding a log at the WARNING level.
+ * Prints a deprecation warning, as well as adds a log at the WARNING level.
*
* @param string $message The log message.
* @param array $context The log context.
@@ -28,4 +28,21 @@ public function deprecation($message, array $context = [])
}
parent::warning($message, $context);
}
+
+ /**
+ * Prints a critical failure, as well as adds a log at the CRITICAL level.
+ *
+ * @param string $message The log message.
+ * @param array $context The log context.
+ * @return void
+ */
+ public function criticalFailure($message, array $context = [])
+ {
+ $message = "FAILURE: " . $message;
+ // Suppress print during unit testing
+ if (MftfApplicationConfig::getConfig()->getPhase() !== MftfApplicationConfig::UNIT_TEST_PHASE) {
+ print ($message . implode("\n", $context) . "\n");
+ }
+ parent::critical($message, $context);
+ }
}