From 81c40a58493ff83fb0dfe0c378c4c4635b75d377 Mon Sep 17 00:00:00 2001 From: Ihor Sviziev Date: Thu, 17 Sep 2020 16:15:44 +0300 Subject: [PATCH 1/3] Speedup static content deploy for specific theme --- .../Deploy/Console/DeployStaticOptions.php | 15 ++++- .../Magento/Deploy/Console/InputValidator.php | 60 +++++++++++++------ .../Magento/Deploy/Strategy/QuickDeploy.php | 46 ++++++++++++-- 3 files changed, 96 insertions(+), 25 deletions(-) diff --git a/app/code/Magento/Deploy/Console/DeployStaticOptions.php b/app/code/Magento/Deploy/Console/DeployStaticOptions.php index 06887fc0206fc..e3abb5c6b0c54 100644 --- a/app/code/Magento/Deploy/Console/DeployStaticOptions.php +++ b/app/code/Magento/Deploy/Console/DeployStaticOptions.php @@ -7,8 +7,8 @@ namespace Magento\Deploy\Console; use Magento\Deploy\Process\Queue; -use Symfony\Component\Console\Input\InputOption; use Symfony\Component\Console\Input\InputArgument; +use Symfony\Component\Console\Input\InputOption; /** * Static Content Deployment Options helper @@ -127,6 +127,11 @@ class DeployStaticOptions */ const NO_LESS = 'no-less'; + /** + * Key for not compiling parent themes + */ + const NO_PARENT = 'no-parent'; + const DEFAULT_JOBS_AMOUNT = 0; /** @@ -324,7 +329,13 @@ private function getSkipOptions() null, InputOption::VALUE_NONE, 'Do not minify HTML files.' - ) + ), + new InputOption( + self::NO_PARENT, + null, + InputOption::VALUE_NONE, + 'Do not compile parent themes. Supported only in quick strategy.' + ), ]; } } diff --git a/app/code/Magento/Deploy/Console/InputValidator.php b/app/code/Magento/Deploy/Console/InputValidator.php index 772410d58a461..2c25269f88538 100644 --- a/app/code/Magento/Deploy/Console/InputValidator.php +++ b/app/code/Magento/Deploy/Console/InputValidator.php @@ -3,14 +3,16 @@ * Copyright © Magento, Inc. All rights reserved. * See COPYING.txt for license details. */ + namespace Magento\Deploy\Console; -use Magento\Setup\Console\Command\DeployStaticContentCommand; +use InvalidArgumentException; use Magento\Deploy\Console\DeployStaticOptions as Options; -use Magento\Framework\Validator\Locale; -use Symfony\Component\Console\Input\InputInterface; use Magento\Framework\App\ObjectManager; +use Magento\Framework\Validator\Locale; use Magento\Framework\Validator\RegexFactory; +use Symfony\Component\Console\Input\InputInterface; +use function array_key_exists; /** * Command input arguments validator class @@ -66,7 +68,7 @@ class InputValidator * InputValidator constructor * * @param Locale $localeValidator - * @param RegexFactory $versionValidatorFactory + * @param RegexFactory|null $versionValidatorFactory */ public function __construct( Locale $localeValidator, @@ -100,6 +102,10 @@ public function validate(InputInterface $input) $this->checkVersionInput( $input->getOption(Options::CONTENT_VERSION) ?: '' ); + $this->checkNoParentInput( + (bool)$input->getOption(Options::NO_PARENT), + (string)$input->getOption(Options::STRATEGY) + ); } /** @@ -108,12 +114,12 @@ public function validate(InputInterface $input) * @param array $areasInclude * @param array $areasExclude * @return void - * @throws \InvalidArgumentException + * @throws InvalidArgumentException */ private function checkAreasInput(array $areasInclude, array $areasExclude) { - if ($areasInclude[0] != 'all' && $areasExclude[0] != 'none') { - throw new \InvalidArgumentException( + if ($areasInclude[0] !== 'all' && $areasExclude[0] !== 'none') { + throw new InvalidArgumentException( '--area (-a) and --exclude-area cannot be used at the same time' ); } @@ -125,12 +131,12 @@ private function checkAreasInput(array $areasInclude, array $areasExclude) * @param array $themesInclude * @param array $themesExclude * @return void - * @throws \InvalidArgumentException + * @throws InvalidArgumentException */ private function checkThemesInput(array $themesInclude, array $themesExclude) { - if ($themesInclude[0] != 'all' && $themesExclude[0] != 'none') { - throw new \InvalidArgumentException( + if ($themesInclude[0] !== 'all' && $themesExclude[0] !== 'none') { + throw new InvalidArgumentException( '--theme (-t) and --exclude-theme cannot be used at the same time' ); } @@ -142,21 +148,21 @@ private function checkThemesInput(array $themesInclude, array $themesExclude) * @param array $languagesInclude * @param array $languagesExclude * @return void - * @throws \InvalidArgumentException + * @throws InvalidArgumentException */ private function checkLanguagesInput(array $languagesInclude, array $languagesExclude) { - if ($languagesInclude[0] != 'all') { + if ($languagesInclude[0] !== 'all') { foreach ($languagesInclude as $lang) { if (!$this->localeValidator->isValid($lang)) { - throw new \InvalidArgumentException( + throw new InvalidArgumentException( $lang . ' argument has invalid value, please run info:language:list for list of available locales' ); } } - if ($languagesExclude[0] != 'none') { - throw new \InvalidArgumentException( + if ($languagesExclude[0] !== 'none') { + throw new InvalidArgumentException( '--language (-l) and --exclude-language cannot be used at the same time' ); } @@ -167,7 +173,7 @@ private function checkLanguagesInput(array $languagesInclude, array $languagesEx * Version input checks * * @param string $contentVersion - * @throws \InvalidArgumentException + * @throws InvalidArgumentException */ private function checkVersionInput(string $contentVersion): void { @@ -179,7 +185,7 @@ private function checkVersionInput(string $contentVersion): void ); if (!$versionValidator->isValid($contentVersion)) { - throw new \InvalidArgumentException( + throw new InvalidArgumentException( 'Argument "' . Options::CONTENT_VERSION . '" has invalid value, content version should contain only characters, digits and dots' @@ -187,4 +193,24 @@ private function checkVersionInput(string $contentVersion): void } } } + + /** + * Validate if --no-parent flag could be used with selected strategy + * + * @param bool $noParent + * @param string $strategy + * @throws InvalidArgumentException + */ + private function checkNoParentInput(bool $noParent, string $strategy): void + { + $supportedStrategies = [ + 'quick' => true, + ]; + + if ($noParent && !array_key_exists($strategy, $supportedStrategies)) { + throw new InvalidArgumentException( + sprintf('Argument "%s" is not supported with "%s" strategy', Options::NO_PARENT, $strategy) + ); + } + } } diff --git a/app/code/Magento/Deploy/Strategy/QuickDeploy.php b/app/code/Magento/Deploy/Strategy/QuickDeploy.php index 02badd7272936..665253f6aa6cb 100644 --- a/app/code/Magento/Deploy/Strategy/QuickDeploy.php +++ b/app/code/Magento/Deploy/Strategy/QuickDeploy.php @@ -5,9 +5,11 @@ */ namespace Magento\Deploy\Strategy; -use Magento\Deploy\Package\PackagePool; +use Magento\Deploy\Console\DeployStaticOptions as Options; use Magento\Deploy\Package\Package; +use Magento\Deploy\Package\PackagePool; use Magento\Deploy\Process\Queue; +use function array_key_exists; /** * Quick deployment strategy implementation @@ -51,7 +53,6 @@ public function deploy(array $options) $groupedPackages = $deployPackages = []; $packages = $this->packagePool->getPackagesForDeployment($options); foreach ($packages as $package) { - /** @var Package $package */ if ($package->isVirtual()) { // skip packages which can not be referenced directly continue; @@ -66,10 +67,17 @@ public function deploy(array $options) $this->preparePackages($level, $levelPackages); } + $parentCompilationRequested = $options[Options::NO_PARENT] !== true; + $includeThemesMap = array_flip($options[Options::THEME] ?? []); + $excludeThemesMap = array_flip($options[Options::EXCLUDE_THEME] ?? []); + foreach ($groupedPackages as $levelPackages) { foreach ($levelPackages as $package) { - $this->queue->add($package); - $deployPackages[] = $package; + if ($parentCompilationRequested + || $this->canDeployTheme($package->getTheme(), $includeThemesMap, $excludeThemesMap)) { + $this->queue->add($package); + $deployPackages[] = $package; + } } } @@ -79,11 +87,13 @@ public function deploy(array $options) } /** + * Prepare packages before deploying + * * @param int $level * @param Package[] $levelPackages * @return void */ - private function preparePackages($level, array $levelPackages) + private function preparePackages(int $level, array $levelPackages): void { foreach ($levelPackages as $package) { $package->aggregate(); @@ -120,7 +130,7 @@ private function preparePackages($level, array $levelPackages) * @param Package $package * @return int */ - private function getInheritanceLevel(Package $package) + private function getInheritanceLevel(Package $package): int { $level = $package->getInheritanceLevel(); $packageId = $package->getArea() . '/' . $package->getTheme(); @@ -131,4 +141,28 @@ private function getInheritanceLevel(Package $package) } return $level; } + + /** + * Verify if specified theme should be deployed + * + * @param string $theme + * @param array $includedThemesMap + * @param array $excludedEntitiesMap + * @return bool + */ + private function canDeployTheme(string $theme, array $includedThemesMap, array $excludedEntitiesMap): bool + { + $includesAllThemes = array_key_exists('all', $includedThemesMap); + $excludesNoneThemes = array_key_exists('none', $excludedEntitiesMap); + + if ($includesAllThemes && $excludesNoneThemes) { + return true; + } elseif (!$excludesNoneThemes) { + return !array_key_exists($theme, $excludedEntitiesMap); + } elseif (!$includesAllThemes) { + return array_key_exists($theme, $includedThemesMap); + } + + return true; + } } From d060ce60985557d5fe270045495f9e347db0c11f Mon Sep 17 00:00:00 2001 From: Ihor Sviziev Date: Tue, 27 Oct 2020 15:39:52 +0200 Subject: [PATCH 2/3] Speedup static content deploy for specific theme Add support for standard strategy --- .../Deploy/Console/DeployStaticOptions.php | 2 +- .../Magento/Deploy/Console/InputValidator.php | 1 + .../Deploy/Strategy/StandardDeploy.php | 36 +++++++++++++++++-- 3 files changed, 36 insertions(+), 3 deletions(-) diff --git a/app/code/Magento/Deploy/Console/DeployStaticOptions.php b/app/code/Magento/Deploy/Console/DeployStaticOptions.php index e3abb5c6b0c54..cc63d34a9a280 100644 --- a/app/code/Magento/Deploy/Console/DeployStaticOptions.php +++ b/app/code/Magento/Deploy/Console/DeployStaticOptions.php @@ -334,7 +334,7 @@ private function getSkipOptions() self::NO_PARENT, null, InputOption::VALUE_NONE, - 'Do not compile parent themes. Supported only in quick strategy.' + 'Do not compile parent themes. Supported only in quick and standard strategies.' ), ]; } diff --git a/app/code/Magento/Deploy/Console/InputValidator.php b/app/code/Magento/Deploy/Console/InputValidator.php index 2c25269f88538..24541174bbabc 100644 --- a/app/code/Magento/Deploy/Console/InputValidator.php +++ b/app/code/Magento/Deploy/Console/InputValidator.php @@ -205,6 +205,7 @@ private function checkNoParentInput(bool $noParent, string $strategy): void { $supportedStrategies = [ 'quick' => true, + 'standard' => true, ]; if ($noParent && !array_key_exists($strategy, $supportedStrategies)) { diff --git a/app/code/Magento/Deploy/Strategy/StandardDeploy.php b/app/code/Magento/Deploy/Strategy/StandardDeploy.php index 8a5ec462d3f4f..86469fc262059 100644 --- a/app/code/Magento/Deploy/Strategy/StandardDeploy.php +++ b/app/code/Magento/Deploy/Strategy/StandardDeploy.php @@ -5,8 +5,9 @@ */ namespace Magento\Deploy\Strategy; -use Magento\Deploy\Package\PackagePool; +use Magento\Deploy\Console\DeployStaticOptions as Options; use Magento\Deploy\Package\Package; +use Magento\Deploy\Package\PackagePool; use Magento\Deploy\Process\Queue; /** @@ -60,12 +61,43 @@ public function deploy(array $options) $deployedPackages[] = $package; } + $parentCompilationRequested = $options[Options::NO_PARENT] !== true; + $includeThemesMap = array_flip($options[Options::THEME] ?? []); + $excludeThemesMap = array_flip($options[Options::EXCLUDE_THEME] ?? []); + foreach ($deployedPackages as $package) { - $this->queue->add($package); + if ($parentCompilationRequested + || $this->canDeployTheme($package->getTheme(), $includeThemesMap, $excludeThemesMap)) { + $this->queue->add($package); + } } $this->queue->process(); return $deployedPackages; } + + /** + * Verify if specified theme should be deployed + * + * @param string $theme + * @param array $includedThemesMap + * @param array $excludedEntitiesMap + * @return bool + */ + private function canDeployTheme(string $theme, array $includedThemesMap, array $excludedEntitiesMap): bool + { + $includesAllThemes = array_key_exists('all', $includedThemesMap); + $excludesNoneThemes = array_key_exists('none', $excludedEntitiesMap); + + if ($includesAllThemes && $excludesNoneThemes) { + return true; + } elseif (!$excludesNoneThemes) { + return !array_key_exists($theme, $excludedEntitiesMap); + } elseif (!$includesAllThemes) { + return array_key_exists($theme, $includedThemesMap); + } + + return true; + } } From fe4fd659f0e2071084202d0d7f0cec9335229f96 Mon Sep 17 00:00:00 2001 From: Ihor Sviziev Date: Wed, 28 Oct 2020 13:27:13 +0200 Subject: [PATCH 3/3] Speedup static content deploy for specific theme Fix failing integration test --- .../Framework/View/Asset/MinifierTest.php | 98 ++++++++++--------- 1 file changed, 53 insertions(+), 45 deletions(-) diff --git a/dev/tests/integration/testsuite/Magento/Framework/View/Asset/MinifierTest.php b/dev/tests/integration/testsuite/Magento/Framework/View/Asset/MinifierTest.php index 277391ba151c3..c9a6bc6eeca94 100644 --- a/dev/tests/integration/testsuite/Magento/Framework/View/Asset/MinifierTest.php +++ b/dev/tests/integration/testsuite/Magento/Framework/View/Asset/MinifierTest.php @@ -5,12 +5,29 @@ */ namespace Magento\Framework\View\Asset; +use Exception; use Magento\Deploy\Console\ConsoleLogger; -use Magento\Framework\App\Filesystem\DirectoryList; -use Magento\TestFramework\Helper\Bootstrap; -use Magento\Framework\App\State as AppState; use Magento\Deploy\Console\DeployStaticOptions as Options; +use Magento\Deploy\Service\DeployStaticContent; use Magento\Deploy\Strategy\DeployStrategyFactory; +use Magento\Framework\App\Filesystem\DirectoryList; +use Magento\Framework\App\ObjectManagerFactory; +use Magento\Framework\App\Request\Http; +use Magento\Framework\App\Response\FileInterface; +use Magento\Framework\App\State as AppState; +use Magento\Framework\App\StaticResource; +use Magento\Framework\App\Utility\Files; +use Magento\Framework\App\View\Deployment\Version\StorageInterface; +use Magento\Framework\Code\Minifier\AdapterInterface; +use Magento\Framework\Exception\LocalizedException; +use Magento\Framework\Filesystem; +use Magento\Framework\Filesystem\Directory\WriteInterface; +use Magento\TestFramework\App\State; +use Magento\TestFramework\Helper\Bootstrap; +use Magento\TestFramework\ObjectManager; +use Magento\Theme\Model\Theme\Registration; +use PHPUnit\Framework\TestCase; +use Symfony\Component\Console\Output\ConsoleOutput; /** * Tests for minifier @@ -19,15 +36,15 @@ * @magentoDbIsolation enabled * @SuppressWarnings(PHPMD.CouplingBetweenObjects) */ -class MinifierTest extends \PHPUnit\Framework\TestCase +class MinifierTest extends TestCase { /** - * @var \Magento\Framework\Filesystem\Directory\WriteInterface + * @var WriteInterface */ private $staticDir; /** - * @var \Magento\TestFramework\ObjectManager + * @var ObjectManager */ protected $objectManager; @@ -43,17 +60,15 @@ protected function setUp(): void { parent::setUp(); $this->objectManager = Bootstrap::getInstance()->getObjectManager(); - /** @var \Magento\Theme\Model\Theme\Registration $registration */ - $registration = $this->objectManager->get( - \Magento\Theme\Model\Theme\Registration::class - ); + /** @var Registration $registration */ + $registration = $this->objectManager->get(Registration::class); $registration->register(); - /** @var \Magento\TestFramework\App\State $appState */ - $appState = $this->objectManager->get(\Magento\TestFramework\App\State::class); + /** @var State $appState */ + $appState = $this->objectManager->get(State::class); $this->origMode = $appState->getMode(); $appState->setMode(AppState::MODE_DEFAULT); - /** @var \Magento\Framework\Filesystem $filesystem */ - $filesystem = Bootstrap::getObjectManager()->get(\Magento\Framework\Filesystem::class); + /** @var Filesystem $filesystem */ + $filesystem = Bootstrap::getObjectManager()->get(Filesystem::class); $this->staticDir = $filesystem->getDirectoryWrite(DirectoryList::STATIC_VIEW); } @@ -62,8 +77,8 @@ protected function setUp(): void */ protected function tearDown(): void { - /** @var \Magento\TestFramework\App\State $appState */ - $appState = $this->objectManager->get(\Magento\TestFramework\App\State::class); + /** @var State $appState */ + $appState = $this->objectManager->get(State::class); $appState->setMode($this->origMode); if ($this->staticDir->isExist('frontend/FrameworkViewMinifier')) { $this->staticDir->delete('frontend/FrameworkViewMinifier'); @@ -83,7 +98,7 @@ protected function tearDown(): void */ public function testCSSminLibrary() { - /** @var \Magento\Framework\Code\Minifier\AdapterInterface $adapter */ + /** @var AdapterInterface $adapter */ $adapter = $this->objectManager->get('cssMinificationAdapter'); $this->assertEquals( file_get_contents(dirname(__DIR__) . '/_files/static/expected/styles.magento.min.css'), @@ -101,7 +116,7 @@ public function testCSSminLibrary() */ public function testJshrinkLibrary() { - /** @var \Magento\Framework\Code\Minifier\AdapterInterface $adapter */ + /** @var AdapterInterface $adapter */ $adapter = $this->objectManager->get('jsMinificationAdapter'); $this->assertEquals( file_get_contents(dirname(__DIR__) . '/_files/static/expected/test.min.js'), @@ -117,16 +132,16 @@ public function testJshrinkLibrary() * * @param string $requestedUri * @param callable $assertionCallback - * @throws \Magento\Framework\Exception\LocalizedException + * @throws LocalizedException|Exception */ - protected function _testCssMinification($requestedUri, $assertionCallback) + private function checkCssMinification(string $requestedUri, callable $assertionCallback): void { - /** @var \Magento\Framework\App\Request\Http $request */ - $request = $this->objectManager->get(\Magento\Framework\App\Request\Http::class); + /** @var Http $request */ + $request = $this->objectManager->get(Http::class); $request->setRequestUri($requestedUri); $request->setParam('resource', $requestedUri); - $response = $this->getMockBuilder(\Magento\Framework\App\Response\FileInterface::class) + $response = $this->getMockBuilder(FileInterface::class) ->setMethods(['setFilePath']) ->getMockForAbstractClass(); $response @@ -134,9 +149,9 @@ protected function _testCssMinification($requestedUri, $assertionCallback) ->method('setFilePath') ->willReturnCallback($assertionCallback); - /** @var \Magento\Framework\App\StaticResource $staticResourceApp */ + /** @var StaticResource $staticResourceApp */ $staticResourceApp = $this->objectManager->create( - \Magento\Framework\App\StaticResource::class, + StaticResource::class, ['response' => $response] ); $staticResourceApp->launch(); @@ -148,7 +163,7 @@ protected function _testCssMinification($requestedUri, $assertionCallback) */ public function testCssMinificationOff() { - $this->_testCssMinification( + $this->checkCssMinification( '/frontend/FrameworkViewMinifier/default/en_US/css/styles.css', function ($path) { $content = file_get_contents($path); @@ -171,7 +186,7 @@ function ($path) { */ public function testCssMinification() { - $this->_testCssMinification( + $this->checkCssMinification( '/frontend/FrameworkViewMinifier/default/en_US/css/styles.min.css', function ($path) { $this->assertEquals( @@ -191,7 +206,7 @@ function ($path) { */ public function testCssMinificationForMinifiedFiles() { - $this->_testCssMinification( + $this->checkCssMinification( '/frontend/FrameworkViewMinifier/default/en_US/css/preminified-styles.min.css', function ($path) { $content = file_get_contents($path); @@ -212,12 +227,12 @@ public function testDeploymentWithMinifierEnabled() $fileToBePublished = $staticPath . '/frontend/FrameworkViewMinifier/default/en_US/css/styles.min.css'; $fileToTestPublishing = dirname(__DIR__) . '/_files/static/theme/web/css/styles.css'; - $omFactory = $this->createPartialMock(\Magento\Framework\App\ObjectManagerFactory::class, ['create']); + $omFactory = $this->createPartialMock(ObjectManagerFactory::class, ['create']); $omFactory->expects($this->any()) ->method('create') ->willReturn($this->objectManager); - $filesUtil = $this->createMock(\Magento\Framework\App\Utility\Files::class); + $filesUtil = $this->createMock(Files::class); $filesUtil->expects($this->any()) ->method('getStaticLibraryFiles') ->willReturn([]); @@ -234,25 +249,17 @@ public function testDeploymentWithMinifierEnabled() ] ); - $this->objectManager->addSharedInstance($filesUtil, \Magento\Framework\App\Utility\Files::class); + $this->objectManager->addSharedInstance($filesUtil, Files::class); - $output = $this->objectManager->create( - \Symfony\Component\Console\Output\ConsoleOutput::class - ); + $output = $this->objectManager->create(ConsoleOutput::class); - $logger = $this->objectManager->create( - ConsoleLogger::class, - ['output' => $output] - ); + $logger = $this->objectManager->create(ConsoleLogger::class, ['output' => $output]); - $versionStorage = $this->createPartialMock( - \Magento\Framework\App\View\Deployment\Version\StorageInterface::class, - ['save', 'load'] - ); + $versionStorage = $this->createPartialMock(StorageInterface::class, ['save', 'load']); - /** @var \Magento\Deploy\Service\DeployStaticContent $deployService */ + /** @var DeployStaticContent $deployService */ $deployService = $this->objectManager->create( - \Magento\Deploy\Service\DeployStaticContent::class, + DeployStaticContent::class, [ 'objectManager' => $this->objectManager, 'logger' => $logger, @@ -279,7 +286,8 @@ public function testDeploymentWithMinifierEnabled() Options::EXCLUDE_LANGUAGE => ['none'], Options::JOBS_AMOUNT => 0, Options::SYMLINK_LOCALE => false, - Options::STRATEGY => DeployStrategyFactory::DEPLOY_STRATEGY_QUICK + Options::STRATEGY => DeployStrategyFactory::DEPLOY_STRATEGY_QUICK, + Options::NO_PARENT => false, ] );