Skip to content

Commit f87cc79

Browse files
author
Fabian Schmengler /
authored
Merge pull request #50 from integer-net/partial-index-command
Console commands for full and partial product reindex (#48)
2 parents 97bbc40 + a344f40 commit f87cc79

25 files changed

+1411
-93
lines changed

‎build/install-magento.sh

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,7 @@ composer config repositories.solr-pro vcs [email protected]:integer-net/solr-pro.gi
3131
sed -i -e 's/"psr-4": {/"psr-4": {\n "IntegerNet\\\\Solr\\\\": ["vendor\/integer-net\/solr-magento2\/main\/test\/unit", "vendor\/integer-net\/solr-magento2\/main\/test\/integration", "vendor\/integer-net\/solr-base\/test\/Solr" ],/g' composer.json
3232
composer config minimum-stability dev
3333
composer require integer-net/solr-magento2 dev-tmp --no-update
34-
composer require --dev tddwizard/magento2-fixtures 0.3.0 --no-update
34+
composer require --dev tddwizard/magento2-fixtures 0.3.1 --no-update
3535
phpunit_version="$(composer info | grep "phpunit/phpunit " | awk '{ print $2 }')"
3636
phpunit_minimum="5.7.0"
3737
if [ "$(printf "$phpunit_minimum\n$phpunit_version" | sort -V | head -n1)" == "$phpunit_version" ] && [ "$phpunit_version" != "$phpunit_minimum" ]; then
@@ -43,4 +43,5 @@ sed -i -e "s/8983/$SOLR_CI_PORT_8983_TCP_PORT/g" vendor/integer-net/solr-magent
4343
sed -i -e "s/localhost/$SOLR_CI_PORT_8983_TCP_ADDR/g" vendor/integer-net/solr-magento2/main/test/integration/_files/solr_config.dist.php
4444
sed -i -e "s/solr-magento2-tests/core0/g" vendor/integer-net/solr-magento2/main/test/integration/_files/solr_config.dist.php
4545
bin/magento module:enable IntegerNet_Solr
46-
bin/magento setup:upgrade
46+
bin/magento setup:di:compile
47+
bin/magento setup:upgrade

‎composer.json

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,15 +3,16 @@
33
"description": "",
44
"require": {
55
"php": "~5.6.0|^7.0",
6-
"integer-net/solr-base": "^3.1.0",
6+
"integer-net/solr-base": "^3.2.1",
77
"integer-net/solr-pro": "^1.2.1",
88
"integer-net/solr-magento2-autosuggest" : "^1.0.0",
9+
"symfony/console": "^2.3|^3.0|^4.0",
910
"magento/module-catalog": "^101.0.0|^102.0.0",
1011
"magento/module-search": "^100.1.0"
1112
},
1213
"require-dev": {
1314
"phpunit/phpunit": "^5.7",
14-
"tddwizard/magento2-fixtures": "0.3.0"
15+
"tddwizard/magento2-fixtures": "0.3.1"
1516
},
1617
"type": "magento2-module",
1718
"license": [

‎main/src/Console/Command/ProgressInConsole.php

Lines changed: 80 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,80 @@
1+
<?php
2+
3+
namespace IntegerNet\Solr\Console\Command;
4+
5+
use IntegerNet\Solr\Indexer\Progress\EventFinish;
6+
use IntegerNet\Solr\Indexer\Progress\EventInfo;
7+
use IntegerNet\Solr\Indexer\Progress\EventProgress;
8+
use IntegerNet\Solr\Indexer\Progress\EventStart;
9+
use IntegerNet\Solr\Indexer\Progress\ProgressHandler;
10+
use IntegerNet\Solr\Indexer\Progress\ProgressUpdate;
11+
use Symfony\Component\Console\Helper\ProgressBar;
12+
use Symfony\Component\Console\Output\OutputInterface;
13+
14+
class ProgressInConsole implements ProgressHandler
15+
{
16+
/**
17+
* @var OutputInterface
18+
*/
19+
private $output;
20+
21+
/**
22+
* @var ProgressBar
23+
*/
24+
private $progressBar;
25+
/**
26+
* @var bool
27+
*/
28+
private $useProgressBar;
29+
30+
const USE_PROGRESS_BAR = true;
31+
32+
/**
33+
* @param OutputInterface $output
34+
*/
35+
public function __construct(OutputInterface $output, $useProgressBar = false)
36+
{
37+
$this->output = $output;
38+
$this->useProgressBar = $useProgressBar;
39+
}
40+
41+
public function progress(ProgressUpdate $update)
42+
{
43+
if ($update instanceof EventStart) {
44+
$this->output->writeln('<info>' . $update->getDescription() . '...</info>');
45+
if ($update->getExpectedSteps() && $this->useProgressBar) {
46+
$this->progressBar = $this->createProgressBarForEvent($update);
47+
$this->progressBar->start();
48+
}
49+
} elseif ($update instanceof EventProgress && $this->progressBar) {
50+
$this->progressBar->advance($update->getSteps());
51+
} elseif ($update instanceof EventFinish) {
52+
if ($this->progressBar) {
53+
$this->progressBar->finish();
54+
$this->progressBar->clear();
55+
}
56+
$this->output->writeln(
57+
'<info>' . $update->getDescription() . ' finished in ' . $update->getElapsedTimeMs() . 'ms</info>'
58+
);
59+
} elseif ($update instanceof EventInfo) {
60+
$this->output->writeln(
61+
'<comment>' . $update->getDescription() . '</comment>'
62+
);
63+
}
64+
}
65+
66+
/**
67+
* @param EventStart $event
68+
* @return ProgressBar
69+
*/
70+
private function createProgressBarForEvent(EventStart $event)
71+
{
72+
$progressBar = new ProgressBar($this->output, $event->getExpectedSteps());
73+
$progressBar->setBarCharacter('<fg=blue>█</>');
74+
$progressBar->setEmptyBarCharacter(' ');
75+
$progressBar->setProgressCharacter('');
76+
$progressBar->setBarWidth(40);
77+
return $progressBar;
78+
}
79+
80+
}

‎main/src/Console/Command/ReindexCommand.php

Lines changed: 109 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,109 @@
1+
<?php
2+
3+
namespace IntegerNet\Solr\Console\Command;
4+
5+
use IntegerNet\Solr\Indexer\Slice;
6+
use IntegerNet\Solr\Model\Indexer;
7+
use Magento\Framework\App;
8+
use Symfony\Component\Console\Command\Command;
9+
use Symfony\Component\Console\Helper\ProgressBar;
10+
use Symfony\Component\Console\Input\InputInterface;
11+
use Symfony\Component\Console\Input\InputOption;
12+
use Symfony\Component\Console\Output\OutputInterface;
13+
use Symfony\Component\Console\Style\SymfonyStyle;
14+
15+
/**
16+
* solr:reindex:full command
17+
*/
18+
class ReindexCommand extends Command
19+
{
20+
const INPUT_STORES = 'stores';
21+
/**
22+
* @var Indexer\Console
23+
*/
24+
private $indexer;
25+
/**
26+
* @var App\State
27+
*/
28+
private $appState;
29+
30+
public function __construct(Indexer\Console $indexer, App\State $appState, $name = null)
31+
{
32+
parent::__construct($name);
33+
$this->indexer = $indexer;
34+
$this->appState = $appState;
35+
}
36+
37+
protected function configure()
38+
{
39+
$options = [
40+
new InputOption(
41+
'stores',
42+
null,
43+
InputOption::VALUE_OPTIONAL,
44+
'Reindex given stores (can be store id, store code, comma seperated. Or "all".) '
45+
. 'If not set, reindex all stores.'
46+
),
47+
new InputOption(
48+
'emptyindex',
49+
null,
50+
InputOption::VALUE_NONE,
51+
'Force emptying the solr index for the given store(s). If not set, configured value is used.'
52+
),
53+
new InputOption(
54+
'noemptyindex',
55+
null,
56+
InputOption::VALUE_NONE,
57+
'Force not emptying the solr index for the given store(s). If not set, configured value is used.'
58+
),
59+
new InputOption(
60+
'progress',
61+
null,
62+
InputOption::VALUE_NONE,
63+
'Show progress bar.'
64+
)
65+
];
66+
$this->setName('solr:reindex:full');
67+
$this->setHelp('Reindex Solr for given stores (see "stores" param).');
68+
$this->setDescription('Reindex Solr');
69+
$this->setDefinition($options);
70+
}
71+
72+
protected function execute(InputInterface $input, OutputInterface $output)
73+
{
74+
$styledOutput = new StyledOutput(
75+
$output,
76+
class_exists(SymfonyStyle::class) ? new SymfonyStyle($input, $output) : null
77+
);
78+
$startTime = microtime(true);
79+
$this->appState->setAreaCode(App\Area::AREA_GLOBAL);
80+
if (!$input->getOption(self::INPUT_STORES) || $input->getOption(self::INPUT_STORES) === 'all') {
81+
$stores = null;
82+
$styledOutput->title('Full reindex of Solr product index...');
83+
} else {
84+
$stores = \explode(',', $input->getOption(self::INPUT_STORES));
85+
$styledOutput->title('Reindex of Solr product index for stores ' . \implode(', ', $stores) . '...');
86+
}
87+
try {
88+
$this->indexer->addProgressHandler(
89+
new ProgressInConsole(
90+
$output,
91+
$input->getOption('progress') ? ProgressInConsole::USE_PROGRESS_BAR : false
92+
)
93+
);
94+
if ($input->getOption('emptyindex')) {
95+
$styledOutput->note('Forcing empty index.');
96+
$this->indexer->executeStoresForceEmpty($stores);
97+
} elseif ($input->getOption('noemptyindex')) {
98+
$styledOutput->note('Forcing non-empty index.');
99+
$this->indexer->executeStoresForceNotEmpty($stores);
100+
} else {
101+
$this->indexer->executeStores($stores);
102+
}
103+
$totalTime = number_format(microtime(true) - $startTime, 2);
104+
$styledOutput->success("Reindex finished in $totalTime seconds.");
105+
} catch (\Exception $e) {
106+
$styledOutput->error($e->getMessage());
107+
}
108+
}
109+
}

‎main/src/Console/Command/ReindexSliceCommand.php

Lines changed: 109 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,109 @@
1+
<?php
2+
3+
namespace IntegerNet\Solr\Console\Command;
4+
5+
use IntegerNet\Solr\Indexer\Slice;
6+
use IntegerNet\Solr\Model\Indexer;
7+
use Magento\Framework\App;
8+
use Symfony\Component\Console\Command\Command;
9+
use Symfony\Component\Console\Helper\ProgressBar;
10+
use Symfony\Component\Console\Input\InputInterface;
11+
use Symfony\Component\Console\Input\InputOption;
12+
use Symfony\Component\Console\Output\OutputInterface;
13+
use Symfony\Component\Console\Style\SymfonyStyle;
14+
15+
/**
16+
* solr:reindex:slice command
17+
*/
18+
class ReindexSliceCommand extends Command
19+
{
20+
const INPUT_STORES = 'stores';
21+
/**
22+
* @var Indexer\Console
23+
*/
24+
private $indexer;
25+
/**
26+
* @var App\State
27+
*/
28+
private $appState;
29+
30+
public function __construct(Indexer\Console $indexer, App\State $appState, $name = null)
31+
{
32+
parent::__construct($name);
33+
$this->indexer = $indexer;
34+
$this->appState = $appState;
35+
}
36+
37+
protected function configure()
38+
{
39+
$options = [
40+
new InputOption(
41+
'stores',
42+
null,
43+
InputOption::VALUE_OPTIONAL,
44+
'Reindex given stores (can be store id, store code, comma seperated. Or "all".) '
45+
. 'If not set, reindex all stores.'
46+
),
47+
new InputOption(
48+
'slice',
49+
null,
50+
InputOption::VALUE_REQUIRED,
51+
'<number>/<total_number>, i.e. "1/5" or "2/5". '
52+
),
53+
new InputOption(
54+
'useswapcore',
55+
null,
56+
InputOption::VALUE_NONE,
57+
'Use swap core for indexing instead of live solr core (only if configured correctly).'
58+
),
59+
new InputOption(
60+
'progress',
61+
null,
62+
InputOption::VALUE_NONE,
63+
'Show progress bar.'
64+
)
65+
];
66+
$this->setName('solr:reindex:slice');
67+
$this->setHelp('Partially reindex Solr for given stores (see "stores" param). Can be used for letting indexing run in parallel.');
68+
$this->setDescription('Partially reindex Solr');
69+
$this->setDefinition($options);
70+
}
71+
72+
protected function execute(InputInterface $input, OutputInterface $output)
73+
{
74+
$styledOutput = new StyledOutput(
75+
$output,
76+
class_exists(SymfonyStyle::class) ? new SymfonyStyle($input, $output) : null
77+
);
78+
$startTime = microtime(true);
79+
$this->appState->setAreaCode(App\Area::AREA_GLOBAL);
80+
if (!$input->getOption(self::INPUT_STORES) || $input->getOption(self::INPUT_STORES) === 'all') {
81+
$stores = null;
82+
$styledOutput->title('Full reindex of Solr product index...');
83+
} else {
84+
$stores = \explode(',', $input->getOption(self::INPUT_STORES));
85+
$styledOutput->title('Reindex of Solr product index for stores ' . \implode(', ', $stores) . '...');
86+
}
87+
try {
88+
$this->indexer->addProgressHandler(
89+
new ProgressInConsole(
90+
$output,
91+
$input->getOption('progress') ? ProgressInConsole::USE_PROGRESS_BAR : false
92+
)
93+
);
94+
$styledOutput->note('Processing slice ' . $input->getOption('slice') . '...');
95+
if ($input->getOption('useswapcore')) {
96+
$this->indexer->executeStoresSliceOnSwappedCore(
97+
Slice::fromExpression($input->getOption('slice')),
98+
$stores
99+
);
100+
} else {
101+
$this->indexer->executeStoresSlice(Slice::fromExpression($input->getOption('slice')), $stores);
102+
}
103+
$totalTime = number_format(microtime(true) - $startTime, 2);
104+
$styledOutput->success("Reindex finished in $totalTime seconds.");
105+
} catch (\Exception $e) {
106+
$styledOutput->error($e->getMessage());
107+
}
108+
}
109+
}

0 commit comments

Comments
 (0)