Skip to content

Commit f60612f

Browse files
RuslanKostiv1RuslanKostiv1
authored
Merge pull request #6473 from magento-performance/40-mview-patch-update-version-to-be-merged-perf
mView patch update
2 parents 05a3e7e + 49b487e commit f60612f

File tree

20 files changed

+853
-87
lines changed

20 files changed

+853
-87
lines changed

‎app/code/Magento/Eav/Model/Mview/ChangeLogBatchWalker.php

Lines changed: 120 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,120 @@
1+
<?php
2+
/**
3+
* Copyright © Magento, Inc. All rights reserved.
4+
* See COPYING.txt for license details.
5+
*/
6+
7+
namespace Magento\Eav\Model\Mview;
8+
9+
use Magento\Framework\App\ResourceConnection;
10+
use Magento\Framework\DB\Sql\Expression;
11+
use Magento\Framework\Mview\View\ChangeLogBatchWalkerInterface;
12+
use Magento\Framework\Mview\View\ChangelogInterface;
13+
14+
/**
15+
* Class BatchIterator
16+
*/
17+
class ChangeLogBatchWalker implements ChangeLogBatchWalkerInterface
18+
{
19+
private const GROUP_CONCAT_MAX_VARIABLE = 'group_concat_max_len';
20+
/** ID is defined as small int. Default size of it is 5 */
21+
private const DEFAULT_ID_SIZE = 5;
22+
23+
/**
24+
* @var ResourceConnection
25+
*/
26+
private $resourceConnection;
27+
28+
/**
29+
* @var array
30+
*/
31+
private $entityTypeCodes;
32+
33+
/**
34+
* @param ResourceConnection $resourceConnection
35+
* @param array $entityTypeCodes
36+
*/
37+
public function __construct(
38+
ResourceConnection $resourceConnection,
39+
array $entityTypeCodes = []
40+
) {
41+
$this->resourceConnection = $resourceConnection;
42+
$this->entityTypeCodes = $entityTypeCodes;
43+
}
44+
45+
/**
46+
* Calculate EAV attributes size
47+
*
48+
* @param ChangelogInterface $changelog
49+
* @return int
50+
* @throws \Exception
51+
*/
52+
private function calculateEavAttributeSize(ChangelogInterface $changelog): int
53+
{
54+
$connection = $this->resourceConnection->getConnection();
55+
56+
if (!isset($this->entityTypeCodes[$changelog->getViewId()])) {
57+
throw new \Exception('Entity type for view was not defined');
58+
}
59+
60+
$select = $connection->select();
61+
$select->from(
62+
$this->resourceConnection->getTableName('eav_attribute'),
63+
new Expression('COUNT(*)')
64+
)
65+
->joinInner(
66+
['type' => $connection->getTableName('eav_entity_type')],
67+
'type.entity_type_id=eav_attribute.entity_type_id'
68+
)
69+
->where('type.entity_type_code = ?', $this->entityTypeCodes[$changelog->getViewId()]);
70+
71+
return (int) $connection->fetchOne($select);
72+
}
73+
74+
/**
75+
* Prepare group max concat
76+
*
77+
* @param int $numberOfAttributes
78+
* @return void
79+
* @throws \Exception
80+
*/
81+
private function setGroupConcatMax(int $numberOfAttributes): void
82+
{
83+
$connection = $this->resourceConnection->getConnection();
84+
$connection->query(sprintf(
85+
'SET SESSION %s=%s',
86+
self::GROUP_CONCAT_MAX_VARIABLE,
87+
$numberOfAttributes * (self::DEFAULT_ID_SIZE + 1)
88+
));
89+
}
90+
91+
/**
92+
* @inheritdoc
93+
* @throws \Exception
94+
*/
95+
public function walk(ChangelogInterface $changelog, int $fromVersionId, int $toVersion, int $batchSize)
96+
{
97+
$connection = $this->resourceConnection->getConnection();
98+
$numberOfAttributes = $this->calculateEavAttributeSize($changelog);
99+
$this->setGroupConcatMax($numberOfAttributes);
100+
$select = $connection->select()->distinct(true)
101+
->where(
102+
'version_id > ?',
103+
(int) $fromVersionId
104+
)
105+
->where(
106+
'version_id <= ?',
107+
$toVersion
108+
)
109+
->group([$changelog->getColumnName(), 'store_id'])
110+
->limit($batchSize);
111+
112+
$columns = [
113+
$changelog->getColumnName(),
114+
'attribute_ids' => new Expression('GROUP_CONCAT(attribute_id)'),
115+
'store_id'
116+
];
117+
$select->from($changelog->getName(), $columns);
118+
return $connection->fetchAll($select);
119+
}
120+
}

‎dev/tests/integration/_files/Magento/TestModuleMview/etc/module.xml

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
<?xml version="1.0"?>
2+
<!--
3+
/**
4+
* Copyright © Magento, Inc. All rights reserved.
5+
* See COPYING.txt for license details.
6+
*/
7+
-->
8+
<config xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:noNamespaceSchemaLocation="urn:magento:framework:Module/etc/module.xsd">
9+
<module name="Magento_TestModuleMview"/>
10+
</config>

‎dev/tests/integration/_files/Magento/TestModuleMview/etc/mview.xml

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
<?xml version="1.0" encoding="UTF-8"?>
2+
<!--
3+
/**
4+
* Copyright © Magento, Inc. All rights reserved.
5+
* See COPYING.txt for license details.
6+
*/
7+
-->
8+
<config xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:noNamespaceSchemaLocation="urn:magento:framework:Mview/etc/mview.xsd">
9+
<view id="test_view_with_additional_columns" class="Magento\Framework\Indexer\Action\Dummy" group="indexer">
10+
<subscriptions>
11+
<table name="test_mview_table" entity_column="entity_id">
12+
<additionalColumns>
13+
<column name="additional_column" cl_name="test_additional_column" />
14+
</additionalColumns>
15+
</table>
16+
</subscriptions>
17+
</view>
18+
</config>

‎dev/tests/integration/_files/Magento/TestModuleMview/registration.php

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
<?php
2+
/**
3+
* Copyright © Magento, Inc. All rights reserved.
4+
* See COPYING.txt for license details.
5+
*/
6+
7+
use Magento\Framework\Component\ComponentRegistrar;
8+
9+
$registrar = new ComponentRegistrar();
10+
if ($registrar->getPath(ComponentRegistrar::MODULE, 'Magento_TestModuleMview') === null) {
11+
ComponentRegistrar::register(ComponentRegistrar::MODULE, 'Magento_TestModuleMview', __DIR__);
12+
}

‎dev/tests/integration/testsuite/Magento/Framework/Mview/View/ChangelogTest.php

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66
namespace Magento\Framework\Mview\View;
77

88
use Magento\Framework\App\ResourceConnection;
9+
use Magento\Framework\Mview\View;
910

1011
/**
1112
* Test Class for \Magento\Framework\Mview\View\Changelog
@@ -123,6 +124,54 @@ public function testClear()
123124
$this->assertEquals(1, $this->model->getVersion()); //the same that a table is empty
124125
}
125126

127+
/**
128+
* Create entity table for MView
129+
*
130+
* @param string $tableName
131+
* @return void
132+
*/
133+
private function createEntityTable(string $tableName)
134+
{
135+
$table = $this->resource->getConnection()->newTable(
136+
$tableName
137+
)->addColumn(
138+
'entity_id',
139+
\Magento\Framework\DB\Ddl\Table::TYPE_INTEGER,
140+
null,
141+
['identity' => true, 'unsigned' => true, 'nullable' => false, 'primary' => true],
142+
'Version ID'
143+
)->addColumn(
144+
'additional_column',
145+
\Magento\Framework\DB\Ddl\Table::TYPE_INTEGER,
146+
null,
147+
['unsigned' => true, 'nullable' => false, 'default' => '0'],
148+
'Entity ID'
149+
);
150+
$this->resource->getConnection()->createTable($table);
151+
}
152+
153+
public function testAdditionalColumns()
154+
{
155+
$tableName = 'test_mview_table';
156+
$this->createEntityTable($tableName);
157+
$view = $this->objectManager->create(View::class);
158+
$view->load('test_view_with_additional_columns');
159+
$view->subscribe();
160+
$this->connection->insert($tableName, ['entity_id' => 12, 'additional_column' => 13]);
161+
$select = $this->connection->select()
162+
->from($view->getChangelog()->getName(), ['entity_id', 'test_additional_column']);
163+
$actual = $this->connection->fetchAll($select);
164+
$this->assertEquals(
165+
[
166+
'entity_id' => "12",
167+
'test_additional_column' => "13"
168+
],
169+
reset($actual)
170+
);
171+
$this->connection->dropTable($tableName);
172+
$this->connection->dropTable($view->getChangelog()->getName());
173+
}
174+
126175
/**
127176
* Test for getList() method
128177
*

‎lib/internal/Magento/Framework/Mview/Config/Converter.php

Lines changed: 60 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,10 +5,34 @@
55
*/
66
namespace Magento\Framework\Mview\Config;
77

8+
use Magento\Framework\Mview\View\AdditionalColumnsProcessor\DefaultProcessor;
9+
use Magento\Framework\Mview\View\ChangeLogBatchWalker;
810
use Magento\Framework\Mview\View\SubscriptionInterface;
911

1012
class Converter implements \Magento\Framework\Config\ConverterInterface
1113
{
14+
/**
15+
* @var string
16+
*/
17+
private $defaultProcessor;
18+
19+
/**
20+
* @var string
21+
*/
22+
private $defaultIterator;
23+
24+
/**
25+
* @param string $defaultProcessor
26+
* @param string $defaultIterator
27+
*/
28+
public function __construct(
29+
string $defaultProcessor = DefaultProcessor::class,
30+
string $defaultIterator = ChangeLogBatchWalker::class
31+
) {
32+
$this->defaultProcessor = $defaultProcessor;
33+
$this->defaultIterator = $defaultIterator;
34+
}
35+
1236
/**
1337
* Convert dom node tree to array
1438
*
@@ -28,6 +52,7 @@ public function convert($source)
2852
$data['view_id'] = $viewId;
2953
$data['action_class'] = $this->getAttributeValue($viewNode, 'class');
3054
$data['group'] = $this->getAttributeValue($viewNode, 'group');
55+
$data['walker'] = $this->getAttributeValue($viewNode, 'walker') ?: $this->defaultIterator;
3156
$data['subscriptions'] = [];
3257

3358
/** @var $childNode \DOMNode */
@@ -76,6 +101,7 @@ protected function convertChild(\DOMNode $childNode, $data)
76101
$name = $this->getAttributeValue($subscription, 'name');
77102
$column = $this->getAttributeValue($subscription, 'entity_column');
78103
$subscriptionModel = $this->getAttributeValue($subscription, 'subscription_model');
104+
79105
if (!empty($subscriptionModel)
80106
&& !in_array(
81107
SubscriptionInterface::class,
@@ -89,11 +115,44 @@ class_implements(ltrim($subscriptionModel, '\\'))
89115
$data['subscriptions'][$name] = [
90116
'name' => $name,
91117
'column' => $column,
92-
'subscription_model' => $subscriptionModel
118+
'subscription_model' => $subscriptionModel,
119+
'additional_columns' => $this->getAdditionalColumns($subscription),
120+
'processor' => $this->getAttributeValue($subscription, 'processor')
121+
?: $this->defaultProcessor
93122
];
94123
}
95124
break;
96125
}
97126
return $data;
98127
}
128+
129+
/**
130+
* Retrieve additional columns of subscription table
131+
*
132+
* @param \DOMNode $subscription
133+
* @return array
134+
*/
135+
private function getAdditionalColumns(\DOMNode $subscription): array
136+
{
137+
$additionalColumns = [];
138+
foreach ($subscription->childNodes as $childNode) {
139+
if ($childNode->nodeType != XML_ELEMENT_NODE || $childNode->nodeName != 'additionalColumns') {
140+
continue;
141+
}
142+
143+
foreach ($childNode->childNodes as $columnNode) {
144+
if ($columnNode->nodeName !== 'column') {
145+
continue;
146+
}
147+
148+
$additionalColumns[$this->getAttributeValue($columnNode, 'name')] = [
149+
'name' => $this->getAttributeValue($columnNode, 'name'),
150+
'cl_name' => $this->getAttributeValue($columnNode, 'cl_name'),
151+
'constant' => $this->getAttributeValue($columnNode, 'constant'),
152+
];
153+
}
154+
}
155+
156+
return $additionalColumns;
157+
}
99158
}

‎lib/internal/Magento/Framework/Mview/Test/Unit/View/ChangelogTest.php

Lines changed: 25 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,8 @@
1111
use Magento\Framework\DB\Adapter\Pdo\Mysql;
1212
use Magento\Framework\DB\Ddl\Table;
1313
use Magento\Framework\DB\Select;
14+
use Magento\Framework\Mview\Config;
15+
use Magento\Framework\Mview\View\AdditionalColumnsProcessor\ProcessorFactory;
1416
use Magento\Framework\Mview\View\Changelog;
1517
use Magento\Framework\Mview\View\ChangelogInterface;
1618
use PHPUnit\Framework\MockObject\MockObject;
@@ -40,21 +42,41 @@ class ChangelogTest extends TestCase
4042
*/
4143
protected $resourceMock;
4244

45+
/**
46+
* @var ProcessorFactory|MockObject
47+
*/
48+
protected $processorFactory;
49+
4350
protected function setUp(): void
4451
{
4552
$this->connectionMock = $this->createMock(Mysql::class);
4653
$this->resourceMock = $this->createMock(ResourceConnection::class);
4754
$this->mockGetConnection($this->connectionMock);
55+
$this->processorFactory = $this->createMock(ProcessorFactory::class);
56+
57+
$this->model = new Changelog($this->resourceMock, $this->getMviewConfigMock(), $this->processorFactory);
58+
}
4859

49-
$this->model = new Changelog($this->resourceMock);
60+
/**
61+
* @return Config|MockObject
62+
*/
63+
private function getMviewConfigMock()
64+
{
65+
$mviewConfigMock = $this->createMock(Config::class);
66+
$mviewConfigMock->expects($this->any())
67+
->method('getView')
68+
->willReturn([
69+
'subscriptions' => []
70+
]);
71+
return $mviewConfigMock;
5072
}
5173

5274
public function testInstanceOf()
5375
{
5476
$resourceMock =
5577
$this->createMock(ResourceConnection::class);
5678
$resourceMock->expects($this->once())->method('getConnection')->willReturn(true);
57-
$model = new Changelog($resourceMock);
79+
$model = new Changelog($resourceMock, $this->getMviewConfigMock(), $this->processorFactory);
5880
$this->assertInstanceOf(ChangelogInterface::class, $model);
5981
}
6082

@@ -65,7 +87,7 @@ public function testCheckConnectionException()
6587
$resourceMock =
6688
$this->createMock(ResourceConnection::class);
6789
$resourceMock->expects($this->once())->method('getConnection')->willReturn(null);
68-
$model = new Changelog($resourceMock);
90+
$model = new Changelog($resourceMock, $this->getMviewConfigMock(), $this->processorFactory);
6991
$model->setViewId('ViewIdTest');
7092
$this->assertNull($model);
7193
}

0 commit comments

Comments
 (0)