Skip to content

Commit 1e61bcb

Browse files
dfeyerdfeyer
committed
Merge branch '3.0'
2 parents dbc595d + a1b9ca5 commit 1e61bcb

10 files changed

Lines changed: 157 additions & 64 deletions

File tree

Classes/Flowpack/ElasticSearch/ContentRepositoryQueueIndexer/Command/NodeIndexQueueCommandController.php renamed to Classes/Command/NodeIndexQueueCommandController.php

Lines changed: 7 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -7,15 +7,13 @@
77
use Flowpack\ElasticSearch\ContentRepositoryQueueIndexer\IndexingJob;
88
use Flowpack\ElasticSearch\ContentRepositoryQueueIndexer\LoggerTrait;
99
use Flowpack\ElasticSearch\ContentRepositoryQueueIndexer\UpdateAliasJob;
10-
use Flowpack\ElasticSearch\Domain\Model\Mapping;
1110
use Flowpack\JobQueue\Common\Job\JobManager;
1211
use Flowpack\JobQueue\Common\Queue\QueueManager;
13-
use TYPO3\Flow\Annotations as Flow;
14-
use TYPO3\Flow\Cli\CommandController;
15-
use TYPO3\Flow\Persistence\PersistenceManagerInterface;
16-
use TYPO3\Flow\Utility\Files;
17-
use TYPO3\TYPO3CR\Domain\Repository\WorkspaceRepository;
18-
use Flowpack\JobQueue\Common\Exception as JobQueueException;
12+
use Neos\Flow\Annotations as Flow;
13+
use Neos\Flow\Cli\CommandController;
14+
use Neos\Flow\Persistence\PersistenceManagerInterface;
15+
use Neos\ContentRepository\Domain\Repository\WorkspaceRepository;
16+
use Neos\Utility\Files;
1917

2018
/**
2119
* Provides CLI features for index handling
@@ -82,6 +80,7 @@ public function buildCommand($workspace = null)
8280
$indexName = $this->createNextIndex($indexPostfix);
8381
$this->updateMapping();
8482

83+
8584
$this->outputLine();
8685
$this->outputLine('<b>Indexing on %s ...</b>', [$indexName]);
8786

@@ -211,7 +210,7 @@ protected function indexWorkspace($workspaceName, $indexPostfix)
211210
$this->outputLine('<info>++</info> Indexing %s workspace', [$workspaceName]);
212211
$nodeCounter = 0;
213212
$offset = 0;
214-
$batchSize = 250;
213+
$batchSize = 500;
215214
while (true) {
216215
$iterator = $this->nodeDataRepository->findAllBySiteAndWorkspace($workspaceName, $offset, $batchSize);
217216

Classes/Flowpack/ElasticSearch/ContentRepositoryQueueIndexer/Domain/Repository/NodeDataRepository.php renamed to Classes/Domain/Repository/NodeDataRepository.php

Lines changed: 5 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -3,18 +3,16 @@
33

44
use Doctrine\Common\Persistence\ObjectManager;
55
use Doctrine\ORM\Internal\Hydration\IterableResult;
6-
use Doctrine\ORM\Query;
76
use Doctrine\ORM\QueryBuilder;
8-
use TYPO3\Flow\Annotations as Flow;
9-
use TYPO3\Flow\Persistence\Repository;
10-
use TYPO3\TYPO3CR\Exception;
7+
use Neos\Flow\Annotations as Flow;
8+
use Neos\Flow\Persistence\Repository;
119

1210
/**
1311
* @Flow\Scope("singleton")
1412
*/
1513
class NodeDataRepository extends Repository
1614
{
17-
const ENTITY_CLASSNAME = 'TYPO3\TYPO3CR\Domain\Model\NodeData';
15+
const ENTITY_CLASSNAME = 'Neos\ContentRepository\Domain\Model\NodeData';
1816

1917
/**
2018
* @Flow\Inject
@@ -35,7 +33,7 @@ public function findAllBySiteAndWorkspace($workspaceName, $firstResult = 0, $max
3533
$queryBuilder = $this->entityManager->createQueryBuilder();
3634

3735
$queryBuilder->select('n.Persistence_Object_Identifier nodeIdentifier, n.dimensionValues dimensions')
38-
->from('TYPO3\TYPO3CR\Domain\Model\NodeData', 'n')
36+
->from('Neos\ContentRepository\Domain\Model\NodeData', 'n')
3937
->where("n.workspace = :workspace AND n.removed = :removed AND n.movedTo IS NULL")
4038
->setFirstResult((integer)$firstResult)
4139
->setMaxResults((integer)$maxResults)
@@ -50,7 +48,7 @@ public function findAllBySiteAndWorkspace($workspaceName, $firstResult = 0, $max
5048
/**
5149
* Iterator over an IterableResult and return a Generator
5250
*
53-
* This methos is useful for batch processing huge result set as it clear the object
51+
* This method is useful for batch processing huge result set as it clear the object
5452
* manager and detach the current object on each iteration.
5553
*
5654
* @param IterableResult $iterator

Classes/Flowpack/ElasticSearch/ContentRepositoryQueueIndexer/Indexer/NodeIndexer.php renamed to Classes/Indexer/NodeIndexer.php

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -5,9 +5,9 @@
55
use Flowpack\ElasticSearch\ContentRepositoryQueueIndexer\Command\NodeIndexQueueCommandController;
66
use Flowpack\ElasticSearch\ContentRepositoryQueueIndexer\IndexingJob;
77
use Flowpack\JobQueue\Common\Job\JobManager;
8-
use TYPO3\Flow\Annotations as Flow;
9-
use TYPO3\Flow\Persistence\PersistenceManagerInterface;
10-
use TYPO3\TYPO3CR\Domain\Model\NodeInterface;
8+
use Neos\Flow\Annotations as Flow;
9+
use Neos\Flow\Persistence\PersistenceManagerInterface;
10+
use Neos\ContentRepository\Domain\Model\NodeInterface;
1111

1212
/**
1313
* ElasticSearch Indexing Job Interface
@@ -34,7 +34,7 @@ class NodeIndexer extends ContentRepositoryAdaptor\Indexer\NodeIndexer
3434

3535
/**
3636
* @param NodeInterface $node
37-
* @param string|null $targetWorkspaceName
37+
* @param string|null $targetWorkspaceName In case indexing is triggered during publishing, a target workspace name will be passed in
3838
*/
3939
public function indexNode(NodeInterface $node, $targetWorkspaceName = null)
4040
{

Classes/Flowpack/ElasticSearch/ContentRepositoryQueueIndexer/IndexingJob.php renamed to Classes/IndexingJob.php

Lines changed: 13 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -2,16 +2,16 @@
22
namespace Flowpack\ElasticSearch\ContentRepositoryQueueIndexer;
33

44
use Flowpack\ElasticSearch\ContentRepositoryAdaptor\Indexer\NodeIndexer;
5+
use Flowpack\ElasticSearch\ContentRepositoryQueueIndexer\Domain\Repository\NodeDataRepository;
56
use Flowpack\JobQueue\Common\Job\JobInterface;
67
use Flowpack\JobQueue\Common\Queue\Message;
78
use Flowpack\JobQueue\Common\Queue\QueueInterface;
8-
use TYPO3\Flow\Annotations as Flow;
9-
use TYPO3\Flow\Utility\Algorithms;
10-
use TYPO3\TYPO3CR\Domain\Factory\NodeFactory;
11-
use TYPO3\TYPO3CR\Domain\Model\NodeData;
12-
use TYPO3\TYPO3CR\Domain\Model\NodeInterface;
13-
use TYPO3\TYPO3CR\Domain\Repository\NodeDataRepository;
14-
use TYPO3\TYPO3CR\Domain\Service\ContextFactory;
9+
use Neos\ContentRepository\Domain\Model\NodeData;
10+
use Neos\Flow\Annotations as Flow;
11+
use Neos\Flow\Utility\Algorithms;
12+
use Neos\ContentRepository\Domain\Factory\NodeFactory;
13+
use Neos\ContentRepository\Domain\Model\NodeInterface;
14+
use Neos\ContentRepository\Domain\Service\ContextFactoryInterface;
1515

1616
/**
1717
* ElasticSearch Indexing Job Interface
@@ -39,7 +39,7 @@ class IndexingJob implements JobInterface
3939
protected $nodeFactory;
4040

4141
/**
42-
* @var ContextFactory
42+
* @var ContextFactoryInterface
4343
* @Flow\Inject
4444
*/
4545
protected $contextFactory;
@@ -52,7 +52,7 @@ class IndexingJob implements JobInterface
5252
/**
5353
* @var string
5454
*/
55-
protected $workspaceName;
55+
protected $targetWorkspaceName;
5656

5757
/**
5858
* @var string
@@ -66,13 +66,13 @@ class IndexingJob implements JobInterface
6666

6767
/**
6868
* @param string $indexPostfix
69-
* @param string $workspaceName
69+
* @param string $targetWorkspaceName In case indexing is triggered during publishing, a target workspace name will be passed in
7070
* @param array $nodes
7171
*/
72-
public function __construct($indexPostfix, $workspaceName, array $nodes)
72+
public function __construct($indexPostfix, $targetWorkspaceName, array $nodes)
7373
{
7474
$this->identifier = Algorithms::generateRandomString(24);
75-
$this->workspaceName = $workspaceName;
75+
$this->targetWorkspaceName = $targetWorkspaceName;
7676
$this->indexPostfix = $indexPostfix;
7777
$this->nodes = $nodes;
7878
}
@@ -94,7 +94,7 @@ public function execute(QueueInterface $queue, Message $message)
9494
/** @var NodeData $nodeData */
9595
$nodeData = $this->nodeDataRepository->findByIdentifier($node['nodeIdentifier']);
9696
$context = $this->contextFactory->create([
97-
'workspaceName' => $this->workspaceName,
97+
'workspaceName' => $this->targetWorkspaceName ?: $nodeData->getWorkspace()->getName(),
9898
'invisibleContentShown' => true,
9999
'inaccessibleContentShown' => false,
100100
'dimensions' => $node['dimensions']

Classes/Flowpack/ElasticSearch/ContentRepositoryQueueIndexer/LoggerTrait.php renamed to Classes/LoggerTrait.php

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,15 @@
11
<?php
22
namespace Flowpack\ElasticSearch\ContentRepositoryQueueIndexer;
33

4-
use TYPO3\Flow\Annotations as Flow;
4+
use Neos\Flow\Annotations as Flow;
55

66
/**
77
* LoggerTrait
88
*/
99
trait LoggerTrait
1010
{
1111
/**
12-
* @var \TYPO3\Flow\Log\SystemLoggerInterface
12+
* @var \Neos\Flow\Log\SystemLoggerInterface
1313
* @Flow\Inject
1414
*/
1515
protected $_logger;

Classes/Flowpack/ElasticSearch/ContentRepositoryQueueIndexer/UpdateAliasJob.php renamed to Classes/UpdateAliasJob.php

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,11 +2,12 @@
22
namespace Flowpack\ElasticSearch\ContentRepositoryQueueIndexer;
33

44
use Flowpack\ElasticSearch\ContentRepositoryAdaptor\Indexer\NodeIndexer;
5+
use Flowpack\ElasticSearch\ContentRepositoryAdaptor\LoggerInterface;
56
use Flowpack\JobQueue\Common\Job\JobInterface;
67
use Flowpack\JobQueue\Common\Queue\Message;
78
use Flowpack\JobQueue\Common\Queue\QueueInterface;
8-
use TYPO3\Flow\Annotations as Flow;
9-
use TYPO3\Flow\Utility\Algorithms;
9+
use Neos\Flow\Annotations as Flow;
10+
use Neos\Flow\Utility\Algorithms;
1011

1112
/**
1213
* ElasticSearch Indexing Job Interface

‎Configuration/Objects.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,2 +1,2 @@
1-
TYPO3\TYPO3CR\Search\Indexer\NodeIndexerInterface:
1+
Neos\ContentRepository\Search\Indexer\NodeIndexerInterface:
22
className: Flowpack\ElasticSearch\ContentRepositoryQueueIndexer\Indexer\NodeIndexer

‎Configuration/Settings.yaml‎

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,9 @@
11
Flowpack:
2-
32
ElasticSearch:
4-
53
ContentRepositoryQueueIndexer:
6-
74
enableLiveAsyncIndexing: true
85

96
JobQueue:
10-
117
Common:
128
queues:
139
'Flowpack.ElasticSearch.ContentRepositoryQueueIndexer':

‎README.md‎

Lines changed: 50 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -1,42 +1,76 @@
1-
# Neos CMS ElasticSearch indexer based on beanstalkd (job queue)
1+
# Neos CMS Elasticsearch indexer based on a job queue
22

3-
This package can be used to index a huge amount of nodes in ElasticSearch indexes. This
4-
package use Beanstalkd and the JobQueue package to handle ES indexing asynchronously.
3+
This package can be used to index a huge amount of nodes in Elasticsearch indexes. This
4+
package use the Flowpack JobQueue packages to handle the indexing asynchronously.
55

6-
## Batch Indexing
6+
# Breaking change after an upgrade to 3.0
77

8-
### How to build indexing job
8+
Previously the Beanstalk queue package was installed by default, this is no longer
9+
the case.
10+
11+
# Install and configure your Queue package
12+
13+
You need to install the correct Queue package based on your needs.
14+
15+
Available packages:
16+
17+
- [sqlite](https://packagist.org/packages/flownative/jobqueue-sqlite)
18+
- [beanstalkd](https://packagist.org/packages/flownative/jobqueue-beanstalkd)
19+
- [doctrine](https://packagist.org/packages/flownative/jobqueue-doctrine)
20+
- [redis](https://packagist.org/packages/flownative/jobqueue-redis)
21+
22+
Please check the package documentation for specific configurations.
23+
24+
The default configuration uses Beanstalkd, but you need to install it manually:
25+
26+
composer require flowpack/jobqueue-beanstalkd
27+
28+
Check the ```Settings.yaml``` to adapt based on the Queue package, you need to adapt the ```className```:
29+
30+
Flowpack:
31+
JobQueue:
32+
Common:
33+
queues:
34+
'Flowpack.ElasticSearch.ContentRepositoryQueueIndexer':
35+
className: 'Flowpack\JobQueue\Beanstalkd\Queue\BeanstalkdQueue'
36+
37+
'Flowpack.ElasticSearch.ContentRepositoryQueueIndexer.Live':
38+
className: 'Flowpack\JobQueue\Beanstalkd\Queue\BeanstalkdQueue'
39+
40+
# Batch Indexing
41+
42+
## How to build indexing job
943

1044
flow nodeindexqueue:build --workspace live
11-
12-
### How to process indexing job
45+
46+
## How to process indexing job
1347

1448
You can use this CLI command to process indexing job:
1549

1650
flow nodeindexqueue:work --queue batch
1751

18-
## Live Indexing
52+
# Live Indexing
1953

20-
You can enable async live indexing by editing ```Settings.yaml```:
54+
You can disable async live indexing by editing ```Settings.yaml```:
2155

2256
Flowpack:
2357
ElasticSearch:
2458
ContentRepositoryQueueIndexer:
25-
enableLiveAsyncIndexing: true
26-
59+
enableLiveAsyncIndexing: false
60+
2761
You can use this CLI command to process indexing job:
2862

29-
flow nodeindexqueue:work --queue live
63+
flow nodeindexqueue:work --queue live
3064

31-
## Supervisord configuration
65+
# Supervisord configuration
3266

3367
You can use tools like ```supervisord``` to manage long runing process. Bellow you can
3468
found a basic configuration:
3569

3670
[supervisord]
37-
71+
3872
[supervisorctl]
39-
73+
4074
[program:elasticsearch_batch_indexing]
4175
command=php flow nodeindexqueue:work --queue batch
4276
stdout_logfile=AUTO
@@ -47,7 +81,7 @@ found a basic configuration:
4781
autostart=true
4882
autorestart=true
4983
stopsignal=QUIT
50-
84+
5185
[program:elasticsearch_live_indexing]
5286
command=php flow nodeindexqueue:work --queue live
5387
stdout_logfile=AUTO

0 commit comments

Comments
 (0)