Skip to content

Commit 35b8c8f

Browse files
committed
FEATURE: Add support for Elasticsearch v5
A new `5.x` setting enables a driver that supports Elasticsearch v5. The documentation has been adjusted and hints on (potentially) needed changes to Elasticsearch node type mappings are documented.
1 parent f6449ec commit 35b8c8f

11 files changed

Lines changed: 507 additions & 0 deletions

File tree

.travis.yml

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,16 +5,21 @@ matrix:
55
env: ES=1
66
- php: 7.0
77
env: ES=2
8+
- php: 7.0
9+
env: ES=5
810
- php: 5.6
911
env: ES=1
1012
- php: 5.6
1113
env: ES=2
14+
- php: 5.6
15+
env: ES=5
1216
sudo: false
1317
before_install:
1418
- export NEOS_TARGET_VERSION=2.3
1519
- cd ..
1620
- if [ "$ES" = 1 ]; then wget --no-check-certificate https://download.elastic.co/elasticsearch/elasticsearch/elasticsearch-1.7.5.zip && unzip elasticsearch-1.7.5.zip && mv elasticsearch-1.7.5 elasticsearch; fi
1721
- if [ "$ES" = 2 ]; then wget --no-check-certificate https://download.elastic.co/elasticsearch/elasticsearch/elasticsearch-2.4.3.zip && unzip elasticsearch-2.4.3.zip && mv elasticsearch-2.4.3 elasticsearch; fi
22+
- if [ "$ES" = 5 ]; then wget --no-check-certificate https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-5.4.2.zip && unzip elasticsearch-5.4.2.zip && mv elasticsearch-5.4.2 elasticsearch; fi
1823
- cd elasticsearch
1924
- bin/elasticsearch -d
2025
- cd ..
@@ -31,3 +36,4 @@ script:
3136
- bin/phpunit --colors -c Build/BuildEssentials/PhpUnit/UnitTests.xml Packages/Application/Flowpack.ElasticSearch.ContentRepositoryAdaptor/Tests/Unit
3237
- if [ "$ES" = 1 ]; then FLOW_CONTEXT="Testing/ElasticVersion1" bin/phpunit --colors --stop-on-failure -c Build/BuildEssentials/PhpUnit/FunctionalTests.xml Packages/Application/Flowpack.ElasticSearch.ContentRepositoryAdaptor/Tests/Functional; fi
3338
- if [ "$ES" = 2 ]; then FLOW_CONTEXT="Testing/ElasticVersion2" bin/phpunit --colors --stop-on-failure -c Build/BuildEssentials/PhpUnit/FunctionalTests.xml Packages/Application/Flowpack.ElasticSearch.ContentRepositoryAdaptor/Tests/Functional; fi
39+
- if [ "$ES" = 5 ]; then FLOW_CONTEXT="Testing/ElasticVersion5" bin/phpunit --colors --stop-on-failure -c Build/BuildEssentials/PhpUnit/FunctionalTests.xml Packages/Application/Flowpack.ElasticSearch.ContentRepositoryAdaptor/Tests/Functional; fi
Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
<?php
2+
namespace Flowpack\ElasticSearch\ContentRepositoryAdaptor\Driver\Version5;
3+
4+
/*
5+
* This file is part of the Flowpack.ElasticSearch.ContentRepositoryAdaptor package.
6+
*
7+
* (c) Contributors of the Neos Project - www.neos.io
8+
*
9+
* This package is Open Source Software. For the full copyright and license
10+
* information, please view the LICENSE file which was distributed with this
11+
* source code.
12+
*/
13+
14+
use Flowpack\ElasticSearch\ContentRepositoryAdaptor\Driver\Version2;
15+
use TYPO3\Flow\Annotations as Flow;
16+
17+
/**
18+
* Document driver for Elasticsearch version 5.x
19+
*
20+
* @Flow\Scope("singleton")
21+
*/
22+
class DocumentDriver extends Version2\DocumentDriver
23+
{
24+
}
Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
<?php
2+
namespace Flowpack\ElasticSearch\ContentRepositoryAdaptor\Driver\Version5;
3+
4+
/*
5+
* This file is part of the Flowpack.ElasticSearch.ContentRepositoryAdaptor package.
6+
*
7+
* (c) Contributors of the Neos Project - www.neos.io
8+
*
9+
* This package is Open Source Software. For the full copyright and license
10+
* information, please view the LICENSE file which was distributed with this
11+
* source code.
12+
*/
13+
14+
use Flowpack\ElasticSearch\ContentRepositoryAdaptor\Driver\Version1;
15+
use TYPO3\Flow\Annotations as Flow;
16+
17+
/**
18+
* Index management driver for Elasticsearch version 5.x
19+
*
20+
* @Flow\Scope("singleton")
21+
*/
22+
class IndexDriver extends Version1\IndexDriver
23+
{
24+
}
Lines changed: 132 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,132 @@
1+
<?php
2+
namespace Flowpack\ElasticSearch\ContentRepositoryAdaptor\Driver\Version5;
3+
4+
/*
5+
* This file is part of the Flowpack.ElasticSearch.ContentRepositoryAdaptor package.
6+
*
7+
* (c) Contributors of the Neos Project - www.neos.io
8+
*
9+
* This package is Open Source Software. For the full copyright and license
10+
* information, please view the LICENSE file which was distributed with this
11+
* source code.
12+
*/
13+
14+
use Flowpack\ElasticSearch\ContentRepositoryAdaptor\Driver\AbstractNodeTypeMappingBuilder;
15+
use Flowpack\ElasticSearch\ContentRepositoryAdaptor\Driver\Version2;
16+
use Flowpack\ElasticSearch\Domain\Model\Document as ElasticSearchDocument;
17+
use TYPO3\Flow\Annotations as Flow;
18+
use TYPO3\TYPO3CR\Domain\Model\NodeInterface;
19+
20+
/**
21+
* Indexer driver for Elasticsearch version 5.x
22+
*
23+
* @Flow\Scope("singleton")
24+
*/
25+
class IndexerDriver extends Version2\IndexerDriver
26+
{
27+
/**
28+
* {@inheritdoc}
29+
*/
30+
public function document($indexName, NodeInterface $node, ElasticSearchDocument $document, array $documentData)
31+
{
32+
if ($this->isFulltextRoot($node)) {
33+
// for fulltext root documents, we need to preserve the "__fulltext" field. That's why we use the
34+
// "update" API instead of the "index" API, with a custom script internally; as we
35+
// shall not delete the "__fulltext" part of the document if it has any.
36+
return [
37+
[
38+
'update' => [
39+
'_type' => $document->getType()->getName(),
40+
'_id' => $document->getId(),
41+
'_index' => $indexName,
42+
'_retry_on_conflict' => 3
43+
]
44+
],
45+
// http://www.elasticsearch.org/guide/en/elasticsearch/reference/2.4/docs-update.html
46+
[
47+
'script' => [
48+
'stored' => 'updateFulltextParts',
49+
'params' => [
50+
'newData' => $documentData
51+
]
52+
],
53+
'upsert' => $documentData
54+
]
55+
];
56+
}
57+
58+
// non-fulltext-root documents can be indexed as-they-are
59+
return [
60+
[
61+
'index' => [
62+
'_type' => $document->getType()->getName(),
63+
'_id' => $document->getId()
64+
]
65+
],
66+
$documentData
67+
];
68+
}
69+
70+
/**
71+
* {@inheritdoc}
72+
*/
73+
public function fulltext(NodeInterface $node, array $fulltextIndexOfNode, $targetWorkspaceName = null)
74+
{
75+
$closestFulltextNode = $node;
76+
while (!$this->isFulltextRoot($closestFulltextNode)) {
77+
$closestFulltextNode = $closestFulltextNode->getParent();
78+
if ($closestFulltextNode === null) {
79+
// root of hierarchy, no fulltext root found anymore, abort silently...
80+
$this->logger->log(sprintf('NodeIndexer: No fulltext root found for node %s (%)', $node->getPath(), $node->getIdentifier()), LOG_WARNING, null, 'ElasticSearch (CR)');
81+
82+
return null;
83+
}
84+
}
85+
86+
$closestFulltextNodeContextPath = $closestFulltextNode->getContextPath();
87+
if ($targetWorkspaceName !== null) {
88+
$closestFulltextNodeContextPath = str_replace($node->getContext()->getWorkspace()->getName(), $targetWorkspaceName, $closestFulltextNodeContextPath);
89+
}
90+
$closestFulltextNodeDocumentIdentifier = sha1($closestFulltextNodeContextPath);
91+
92+
if ($closestFulltextNode->isRemoved()) {
93+
// fulltext root is removed, abort silently...
94+
$this->logger->log(sprintf('NodeIndexer (%s): Fulltext root found for %s (%s) not updated, it is removed', $closestFulltextNodeDocumentIdentifier, $node->getPath(), $node->getIdentifier()), LOG_DEBUG, null, 'ElasticSearch (CR)');
95+
96+
return null;
97+
}
98+
99+
$this->logger->log(sprintf('NodeIndexer (%s): Updated fulltext index for %s (%s)', $closestFulltextNodeDocumentIdentifier, $closestFulltextNodeContextPath, $closestFulltextNode->getIdentifier()), LOG_DEBUG, null, 'ElasticSearch (CR)');
100+
101+
$upsertFulltextParts = [];
102+
if (!empty($fulltextIndexOfNode)) {
103+
$upsertFulltextParts[$node->getIdentifier()] = $fulltextIndexOfNode;
104+
}
105+
106+
return [
107+
[
108+
'update' => [
109+
'_type' => AbstractNodeTypeMappingBuilder::convertNodeTypeNameToMappingName($closestFulltextNode->getNodeType()->getName()),
110+
'_id' => $closestFulltextNodeDocumentIdentifier
111+
]
112+
],
113+
// http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/docs-update.html
114+
[
115+
// first, update the __fulltextParts, then re-generate the __fulltext from all __fulltextParts
116+
'script' => [
117+
'stored' => 'regenerateFulltext',
118+
'params' => [
119+
'identifier' => $node->getIdentifier(),
120+
'nodeIsRemoved' => $node->isRemoved(),
121+
'nodeIsHidden' => $node->isHidden(),
122+
'fulltext' => $fulltextIndexOfNode
123+
],
124+
],
125+
'upsert' => [
126+
'__fulltext' => $fulltextIndexOfNode,
127+
'__fulltextParts' => $upsertFulltextParts
128+
]
129+
]
130+
];
131+
}
132+
}

0 commit comments

Comments
 (0)