Skip to content

Commit d3e83f1

Browse files
dfeyerdfeyer
committed
[TASK] Initial commit
0 parents  commit d3e83f1

5 files changed

Lines changed: 365 additions & 0 deletions

File tree

‎Classes/Flowpack/ElasticSearch/ContentRepositoryQueueIndexer/Command/NodeIndexQueueCommandController.php‎

Lines changed: 153 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,153 @@
1+
<?php
2+
namespace Flowpack\ElasticSearch\ContentRepositoryQueueIndexer\Command;
3+
4+
/* *
5+
* This script belongs to the TYPO3 Flow package "Flowpack.ElasticSearch.ContentRepositoryQueueIndexer". *
6+
* *
7+
* It is free software; you can redistribute it and/or modify it under *
8+
* the terms of the GNU Lesser General Public License, either version 3 *
9+
* of the License, or (at your option) any later version. *
10+
* *
11+
* The TYPO3 project - inspiring people to share! *
12+
* */
13+
14+
use Doctrine\Common\Persistence\ObjectManager;
15+
use Flowpack\ElasticSearch\ContentRepositoryAdaptor\Command\NodeIndexCommandController;
16+
use Flowpack\ElasticSearch\ContentRepositoryAdaptor\Indexer\NodeIndexer;
17+
use Flowpack\ElasticSearch\ContentRepositoryAdaptor\LoggerInterface;
18+
use Flowpack\ElasticSearch\ContentRepositoryQueueIndexer\Domain\Repository\NodeDataRepository;
19+
use Flowpack\ElasticSearch\ContentRepositoryQueueIndexer\IndexingJob;
20+
use TYPO3\Flow\Annotations as Flow;
21+
use TYPO3\Flow\Cli\CommandController;
22+
use Flowpack\ElasticSearch\ContentRepositoryAdaptor\Mapping\NodeTypeMappingBuilder;
23+
use TYPO3\Flow\Configuration\ConfigurationManager;
24+
use TYPO3\Flow\Core\Booting\Scripts;
25+
use TYPO3\Flow\Exception;
26+
use TYPO3\Flow\Persistence\PersistenceManagerInterface;
27+
use TYPO3\Jobqueue\Common\Job\JobManager;
28+
use TYPO3\TYPO3CR\Domain\Model\NodeData;
29+
use TYPO3\TYPO3CR\Domain\Model\NodeInterface;
30+
use TYPO3\TYPO3CR\Domain\Repository\WorkspaceRepository;
31+
use TYPO3\TYPO3CR\Search\Indexer\NodeIndexingManager;
32+
33+
/**
34+
* Provides CLI features for index handling
35+
*
36+
* @Flow\Scope("singleton")
37+
*/
38+
class NodeIndexQueueCommandController extends CommandController {
39+
40+
/**
41+
* @Flow\Inject
42+
* @var JobManager
43+
*/
44+
protected $jobManager;
45+
46+
/**
47+
* @Flow\Inject
48+
* @var NodeTypeMappingBuilder
49+
*/
50+
protected $nodeTypeMappingBuilder;
51+
52+
/**
53+
* @Flow\Inject
54+
* @var NodeDataRepository
55+
*/
56+
protected $nodeDataRepository;
57+
58+
/**
59+
* @Flow\Inject
60+
* @var WorkspaceRepository
61+
*/
62+
protected $workspaceRepository;
63+
64+
/**
65+
* @Flow\Inject
66+
* @var NodeIndexer
67+
*/
68+
protected $nodeIndexer;
69+
70+
/**
71+
* @Flow\Inject
72+
* @var LoggerInterface
73+
*/
74+
protected $logger;
75+
76+
/**
77+
* @param string $workspace
78+
*/
79+
public function buildCommand($workspace = NULL) {
80+
$indexPostfix = time();
81+
$this->createNextIndex($indexPostfix);
82+
$this->updateMapping();
83+
84+
85+
$this->outputLine(sprintf('Indexing on %s ... ', $indexPostfix));
86+
87+
if ($workspace === NULL) {
88+
foreach ($this->workspaceRepository->findAll() as $workspace) {
89+
$this->indexWorkspace($workspace->getName(), $indexPostfix);
90+
}
91+
} else {
92+
$this->indexWorkspace($workspace, $indexPostfix);
93+
}
94+
}
95+
96+
/**
97+
* @param string $indexPostfix
98+
*/
99+
public function updateAliasCommand($indexPostfix) {
100+
$this->nodeIndexer->setIndexNamePostfix($indexPostfix);
101+
$this->nodeIndexer->updateIndexAlias();
102+
}
103+
104+
/**
105+
* @param string $workspaceName
106+
* @param string $indexPostfix
107+
*/
108+
protected function indexWorkspace($workspaceName, $indexPostfix) {
109+
$offset = 0;
110+
$batchSize = 1000;
111+
while (TRUE) {
112+
$result = $this->nodeDataRepository->findAllBySiteAndWorkspace($workspaceName, $offset, 1000);
113+
if ($result === array()) {
114+
break;
115+
}
116+
foreach ($result as $data) {
117+
$indexingJob = new IndexingJob(
118+
$indexPostfix,
119+
$data['nodeIdentifier'],
120+
$workspaceName,
121+
$data['dimensions']
122+
);
123+
$this->jobManager->queue('Flowpack.ElasticSearch.ContentRepositoryQueueIndexer', $indexingJob);
124+
}
125+
$this->output('.');
126+
$offset += $batchSize;
127+
}
128+
}
129+
130+
/**
131+
* Create next index
132+
* @param string $indexPostfix
133+
*/
134+
protected function createNextIndex($indexPostfix) {
135+
$this->nodeIndexer->setIndexNamePostfix($indexPostfix);
136+
$this->nodeIndexer->getIndex()->create();
137+
$this->logger->log('Created index ' . $this->nodeIndexer->getIndexName(), LOG_INFO);
138+
}
139+
140+
/**
141+
* Update Index Mapping
142+
*/
143+
protected function updateMapping() {
144+
$nodeTypeMappingCollection = $this->nodeTypeMappingBuilder->buildMappingInformation($this->nodeIndexer->getIndex());
145+
foreach ($nodeTypeMappingCollection as $mapping) {
146+
/** @var \Flowpack\ElasticSearch\Domain\Model\Mapping $mapping */
147+
$mapping->apply();
148+
}
149+
$this->logger->log('Updated Mapping.', LOG_INFO);
150+
}
151+
152+
153+
}

‎Classes/Flowpack/ElasticSearch/ContentRepositoryQueueIndexer/Domain/Repository/NodeDataRepository.php‎

Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,58 @@
1+
<?php
2+
namespace Flowpack\ElasticSearch\ContentRepositoryQueueIndexer\Domain\Repository;
3+
4+
/* *
5+
* This script belongs to the TYPO3 Flow package "TYPO3CR". *
6+
* *
7+
* It is free software; you can redistribute it and/or modify it under *
8+
* the terms of the GNU General Public License, either version 3 of the *
9+
* License, or (at your option) any later version. *
10+
* *
11+
* The TYPO3 project - inspiring people to share! *
12+
* */
13+
14+
use Doctrine\Common\Persistence\ObjectManager;
15+
use Doctrine\ORM\Query;
16+
use Doctrine\ORM\QueryBuilder;
17+
use TYPO3\Flow\Annotations as Flow;
18+
use TYPO3\Flow\Persistence\Repository;
19+
use TYPO3\TYPO3CR\Exception;
20+
21+
/**
22+
* @Flow\Scope("singleton")
23+
*/
24+
class NodeDataRepository extends Repository {
25+
26+
const ENTITY_CLASSNAME = 'TYPO3\TYPO3CR\Domain\Model\NodeData';
27+
28+
/**
29+
* @Flow\Inject
30+
* @var ObjectManager
31+
*/
32+
protected $entityManager;
33+
34+
/**
35+
* @param string $workspaceName
36+
* @param integer $firstResult
37+
* @param integer $maxResults
38+
* @return array
39+
*/
40+
public function findAllBySiteAndWorkspace($workspaceName, $firstResult = 0, $maxResults = 1000) {
41+
42+
/** @var QueryBuilder $queryBuilder */
43+
$queryBuilder = $this->entityManager->createQueryBuilder();
44+
45+
$queryBuilder->select('n.Persistence_Object_Identifier nodeIdentifier, n.dimensionValues dimensions')
46+
->from('TYPO3\TYPO3CR\Domain\Model\NodeData', 'n')
47+
->where("n.workspace = :workspace AND n.removed = :removed AND n.movedTo IS NULL")
48+
->setFirstResult((integer)$firstResult)
49+
->setMaxResults((integer)$maxResults)
50+
->setParameters([
51+
':workspace' => $workspaceName,
52+
':removed' => FALSE,
53+
]);
54+
55+
return $queryBuilder->getQuery()->getArrayResult();
56+
}
57+
58+
}

‎Classes/Flowpack/ElasticSearch/ContentRepositoryQueueIndexer/IndexingJob.php‎

Lines changed: 134 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,134 @@
1+
<?php
2+
namespace Flowpack\ElasticSearch\ContentRepositoryQueueIndexer;
3+
4+
/* *
5+
* This script belongs to the TYPO3 Flow package "Flowpack.ElasticSearch.ContentRepositoryQueueIndexer". *
6+
* *
7+
* It is free software; you can redistribute it and/or modify it under *
8+
* the terms of the GNU Lesser General Public License, either version 3 *
9+
* of the License, or (at your option) any later version. *
10+
* *
11+
* The TYPO3 project - inspiring people to share! *
12+
* */
13+
14+
use Flowpack\ElasticSearch\ContentRepositoryAdaptor\Indexer\NodeIndexer;
15+
use Flowpack\ElasticSearch\ContentRepositoryAdaptor\LoggerInterface;
16+
use TYPO3\Flow\Annotations as Flow;
17+
use TYPO3\Jobqueue\Common\Job\JobInterface;
18+
use TYPO3\Jobqueue\Common\Queue\Message;
19+
use TYPO3\Jobqueue\Common\Queue\QueueInterface;
20+
use TYPO3\TYPO3CR\Domain\Factory\NodeFactory;
21+
use TYPO3\TYPO3CR\Domain\Model\NodeData;
22+
use TYPO3\TYPO3CR\Domain\Repository\NodeDataRepository;
23+
use TYPO3\TYPO3CR\Domain\Service\ContextFactory;
24+
25+
/**
26+
* ElasticSearch Indexing Job Interface
27+
*/
28+
class IndexingJob implements JobInterface {
29+
30+
/**
31+
* @var string
32+
*/
33+
protected $indexPostfix;
34+
35+
/**
36+
* @var string
37+
*/
38+
protected $nodeIdentifier;
39+
40+
/**
41+
* @var string
42+
*/
43+
protected $workspaceName;
44+
45+
/**
46+
* @var string
47+
*/
48+
protected $dimensions;
49+
50+
/**
51+
* @Flow\Inject
52+
* @var NodeIndexer
53+
*/
54+
protected $nodeIndexer;
55+
56+
/**
57+
* @Flow\Inject
58+
* @var NodeDataRepository
59+
*/
60+
protected $nodeDataRepository;
61+
62+
/**
63+
* @Flow\Inject
64+
* @var NodeFactory
65+
*/
66+
protected $nodeFactory;
67+
68+
/**
69+
* @Flow\Inject
70+
* @var ContextFactory
71+
*/
72+
protected $contextFactory;
73+
74+
/**
75+
* @Flow\Inject
76+
* @var LoggerInterface
77+
*/
78+
protected $logger;
79+
80+
/**
81+
* @param string $indexPostfix
82+
* @param string $nodeIdentifier
83+
* @param string $workspaceName
84+
* @param array $dimensions
85+
*/
86+
public function __construct($indexPostfix, $nodeIdentifier, $workspaceName, array $dimensions = array()) {
87+
$this->indexPostfix = $indexPostfix;
88+
$this->nodeIdentifier = $nodeIdentifier;
89+
$this->workspaceName = $workspaceName;
90+
$this->dimensions = $dimensions;
91+
}
92+
93+
/**
94+
* Execute the job
95+
* A job should finish itself after successful execution using the queue methods.
96+
*
97+
* @param QueueInterface $queue
98+
* @param Message $message The original message
99+
* @return boolean TRUE if the job was executed successfully and the message should be finished
100+
*/
101+
public function execute(QueueInterface $queue, Message $message) {
102+
/** @var NodeData $nodeData */
103+
$nodeData = $this->nodeDataRepository->findByIdentifier($this->nodeIdentifier);
104+
$context = $this->contextFactory->create([
105+
'workspaceName' => $this->workspaceName,
106+
'dimensions' => $this->dimensions
107+
]);
108+
$currentNode = $this->nodeFactory->createFromNodeData($nodeData, $context);
109+
$this->nodeIndexer->setIndexNamePostfix($this->indexPostfix);
110+
$this->nodeIndexer->indexNode($currentNode);
111+
$this->nodeIndexer->flush();
112+
113+
return TRUE;
114+
}
115+
116+
/**
117+
* Get an optional identifier for the job
118+
*
119+
* @return string A job identifier
120+
*/
121+
public function getIdentifier() {
122+
return $this->nodeIdentifier;
123+
}
124+
125+
/**
126+
* Get a readable label for the job
127+
*
128+
* @return string A label for the job
129+
*/
130+
public function getLabel() {
131+
return sprintf('ElasticSearch Indexing Job (%s)', $this->getIdentifier());
132+
}
133+
134+
}

‎Configuration/Settings.yaml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
TYPO3:
2+
Jobqueue:
3+
Common:
4+
queues:
5+
'Flowpack.ElasticSearch.ContentRepositoryQueueIndexer':
6+
className: 'TYPO3\Jobqueue\Beanstalkd\Queue\BeanstalkdQueue'

‎composer.json‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
{
2+
"name": "flowpack/elasticsearch-contentrepositoryqueueindexer",
3+
"type": "typo3-flow-package",
4+
"description": "Add description here",
5+
"require": {
6+
"typo3/flow": "*",
7+
"typo3/jobqueue-beanstalkd": "*"
8+
},
9+
"autoload": {
10+
"psr-0": {
11+
"Flowpack\\ElasticSearch\\ContentRepositoryQueueIndexer": "Classes"
12+
}
13+
}
14+
}

0 commit comments

Comments
 (0)