Skip to content

Commit aaafca4

Browse files
daniellienertdaniellienert
authored
Merge pull request #220 from kdambekalns/es-5-support
FEATURE: Support for Elasticsearch v5
2 parents d473e43 + 7b5b607 commit aaafca4

29 files changed

Lines changed: 887 additions & 97 deletions

‎.travis.yml‎

Lines changed: 16 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,20 +1,33 @@
11
language: php
2+
sudo: false
3+
git:
4+
depth: 5
5+
submodules: false
6+
addons:
7+
apt:
8+
packages:
9+
- oracle-java8-set-default
210
matrix:
311
include:
412
- php: 7.0
513
env: ES=1
614
- php: 7.0
715
env: ES=2
16+
- php: 7.0
17+
env: ES=5
818
- php: 5.6
919
env: ES=1
1020
- php: 5.6
1121
env: ES=2
12-
sudo: false
22+
- php: 5.6
23+
env: ES=5
1324
before_install:
25+
- export JAVA_HOME=/usr/lib/jvm/java-8-oracle
1426
- export NEOS_TARGET_VERSION=2.3
1527
- cd ..
1628
- if [ "$ES" = 1 ]; then wget --no-check-certificate https://download.elastic.co/elasticsearch/elasticsearch/elasticsearch-1.7.6.zip && unzip elasticsearch-1.7.6.zip && mv elasticsearch-1.7.6 elasticsearch; fi
17-
- if [ "$ES" = 2 ]; then wget --no-check-certificate https://download.elastic.co/elasticsearch/elasticsearch/elasticsearch-2.4.5.zip && unzip elasticsearch-2.4.5.zip && mv elasticsearch-2.4.5 elasticsearch; fi
29+
- if [ "$ES" = 2 ]; then wget --no-check-certificate https://download.elastic.co/elasticsearch/elasticsearch/elasticsearch-2.4.6.zip && unzip elasticsearch-2.4.6.zip && mv elasticsearch-2.4.6 elasticsearch; fi
30+
- if [ "$ES" = 5 ]; then wget --no-check-certificate https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-5.6.8.zip && unzip elasticsearch-5.6.8.zip && mv elasticsearch-5.6.8 elasticsearch; fi
1831
- cd elasticsearch
1932
- bin/elasticsearch -d
2033
- cd ..
@@ -31,3 +44,4 @@ script:
3144
- bin/phpunit --colors -c Build/BuildEssentials/PhpUnit/UnitTests.xml Packages/Application/Flowpack.ElasticSearch.ContentRepositoryAdaptor/Tests/Unit
3245
- if [ "$ES" = 1 ]; then FLOW_CONTEXT="Testing/ElasticVersion1" bin/phpunit --colors --stop-on-failure -c Build/BuildEssentials/PhpUnit/FunctionalTests.xml Packages/Application/Flowpack.ElasticSearch.ContentRepositoryAdaptor/Tests/Functional; fi
3346
- if [ "$ES" = 2 ]; then FLOW_CONTEXT="Testing/ElasticVersion2" bin/phpunit --colors --stop-on-failure -c Build/BuildEssentials/PhpUnit/FunctionalTests.xml Packages/Application/Flowpack.ElasticSearch.ContentRepositoryAdaptor/Tests/Functional; fi
47+
- if [ "$ES" = 5 ]; then FLOW_CONTEXT="Testing/ElasticVersion5" bin/phpunit --colors --stop-on-failure -c Build/BuildEssentials/PhpUnit/FunctionalTests.xml Packages/Application/Flowpack.ElasticSearch.ContentRepositoryAdaptor/Tests/Functional; fi

‎Classes/Command/NodeIndexCommandController.php‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -11,8 +11,8 @@
1111
* source code.
1212
*/
1313

14+
use Flowpack\ElasticSearch\ContentRepositoryAdaptor\Driver\NodeTypeMappingBuilderInterface;
1415
use Flowpack\ElasticSearch\ContentRepositoryAdaptor\LoggerInterface;
15-
use Flowpack\ElasticSearch\ContentRepositoryAdaptor\Mapping\NodeTypeMappingBuilder;
1616
use Flowpack\ElasticSearch\ContentRepositoryAdaptor\Service\IndexWorkspaceTrait;
1717
use Flowpack\ElasticSearch\Domain\Model\Mapping;
1818
use Flowpack\ElasticSearch\Transfer\Exception\ApiException;
@@ -70,7 +70,7 @@ class NodeIndexCommandController extends CommandController
7070

7171
/**
7272
* @Flow\Inject
73-
* @var NodeTypeMappingBuilder
73+
* @var NodeTypeMappingBuilderInterface
7474
*/
7575
protected $nodeTypeMappingBuilder;
7676

‎Classes/Driver/AbstractNodeTypeMappingBuilder.php‎

Lines changed: 81 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,81 @@
1+
<?php
2+
namespace Flowpack\ElasticSearch\ContentRepositoryAdaptor\Driver;
3+
4+
/*
5+
* This file is part of the Flowpack.ElasticSearch.ContentRepositoryAdaptor package.
6+
*
7+
* (c) Contributors of the Neos Project - www.neos.io
8+
*
9+
* This package is Open Source Software. For the full copyright and license
10+
* information, please view the LICENSE file which was distributed with this
11+
* source code.
12+
*/
13+
14+
use TYPO3\Flow\Annotations as Flow;
15+
use TYPO3\Flow\Configuration\ConfigurationManager;
16+
use TYPO3\Flow\Object\ObjectManagerInterface;
17+
use TYPO3\TYPO3CR\Domain\Service\NodeTypeManager;
18+
19+
/**
20+
* Builds the mapping information for Content Repository Node Types in Elasticsearch
21+
*
22+
* @Flow\Scope("singleton")
23+
*/
24+
abstract class AbstractNodeTypeMappingBuilder implements NodeTypeMappingBuilderInterface
25+
{
26+
/**
27+
* The default configuration for a given property type in NodeTypes.yaml, if no explicit elasticSearch section defined there.
28+
*
29+
* @var array
30+
*/
31+
protected $defaultConfigurationPerType;
32+
33+
/**
34+
* @Flow\Inject
35+
* @var NodeTypeManager
36+
*/
37+
protected $nodeTypeManager;
38+
39+
/**
40+
* @var \TYPO3\Flow\Error\Result
41+
*/
42+
protected $lastMappingErrors;
43+
44+
/**
45+
* @Flow\Inject
46+
* @var ConfigurationManager
47+
*/
48+
protected $configurationManager;
49+
50+
/**
51+
* Called by the Flow object framework after creating the object and resolving all dependencies.
52+
*
53+
* @param integer $cause Creation cause
54+
*/
55+
public function initializeObject($cause)
56+
{
57+
if ($cause === ObjectManagerInterface::INITIALIZATIONCAUSE_CREATED) {
58+
$settings = $this->configurationManager->getConfiguration(ConfigurationManager::CONFIGURATION_TYPE_SETTINGS, 'TYPO3.TYPO3CR.Search');
59+
$this->defaultConfigurationPerType = $settings['defaultConfigurationPerType'];
60+
}
61+
}
62+
63+
/**
64+
* Converts a Content Repository Node Type name into a name which can be used for an Elasticsearch Mapping
65+
*
66+
* @param string $nodeTypeName
67+
* @return string
68+
*/
69+
public static function convertNodeTypeNameToMappingName($nodeTypeName)
70+
{
71+
return str_replace('.', '-', $nodeTypeName);
72+
}
73+
74+
/**
75+
* @return \TYPO3\Flow\Error\Result
76+
*/
77+
public function getLastMappingErrors()
78+
{
79+
return $this->lastMappingErrors;
80+
}
81+
}

‎Classes/Driver/NodeTypeMappingBuilderInterface.php‎

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
<?php
2+
namespace Flowpack\ElasticSearch\ContentRepositoryAdaptor\Driver;
3+
4+
/*
5+
* This file is part of the Flowpack.ElasticSearch.ContentRepositoryAdaptor package.
6+
*
7+
* (c) Contributors of the Neos Project - www.neos.io
8+
*
9+
* This package is Open Source Software. For the full copyright and license
10+
* information, please view the LICENSE file which was distributed with this
11+
* source code.
12+
*/
13+
14+
use Flowpack\ElasticSearch\Domain\Model\Index;
15+
use Flowpack\ElasticSearch\Mapping\MappingCollection;
16+
use TYPO3\Flow\Error\Result;
17+
18+
/**
19+
* NodeTypeMappingBuilder Interface
20+
*/
21+
interface NodeTypeMappingBuilderInterface
22+
{
23+
24+
/**
25+
* Converts a Content Repository Node Type name into a name which can be used for an Elasticsearch Mapping
26+
*
27+
* @param string $nodeTypeName
28+
* @return string
29+
*/
30+
public static function convertNodeTypeNameToMappingName($nodeTypeName);
31+
32+
/**
33+
* Builds a Mapping Collection from the configured node types
34+
*
35+
* @param Index $index
36+
* @return MappingCollection<\Flowpack\ElasticSearch\Domain\Model\Mapping>
37+
*/
38+
public function buildMappingInformation(Index $index);
39+
40+
/**
41+
* @return Result
42+
*/
43+
public function getLastMappingErrors();
44+
}

‎Classes/Driver/Version1/DocumentDriver.php‎

Lines changed: 13 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,8 @@
1313

1414
use Flowpack\ElasticSearch\ContentRepositoryAdaptor\Driver\AbstractDriver;
1515
use Flowpack\ElasticSearch\ContentRepositoryAdaptor\Driver\DocumentDriverInterface;
16-
use Flowpack\ElasticSearch\ContentRepositoryAdaptor\Mapping\NodeTypeMappingBuilder;
16+
use Flowpack\ElasticSearch\ContentRepositoryAdaptor\Driver\DriverInterface;
17+
use Flowpack\ElasticSearch\ContentRepositoryAdaptor\Driver\NodeTypeMappingBuilderInterface;
1718
use Flowpack\ElasticSearch\Domain\Model\Index;
1819
use TYPO3\Flow\Annotations as Flow;
1920
use TYPO3\TYPO3CR\Domain\Model\NodeInterface;
@@ -26,15 +27,22 @@
2627
*/
2728
class DocumentDriver extends AbstractDriver implements DocumentDriverInterface
2829
{
30+
/**
31+
* @Flow\Inject
32+
* @var NodeTypeMappingBuilderInterface
33+
*/
34+
protected $nodeTypeMappingBuilder;
35+
2936
/**
3037
* {@inheritdoc}
3138
*/
3239
public function delete(NodeInterface $node, $identifier)
3340
{
41+
$nodeTypeMappingBuilder = $this->nodeTypeMappingBuilder;
3442
return [
3543
[
3644
'delete' => [
37-
'_type' => NodeTypeMappingBuilder::convertNodeTypeNameToMappingName($node->getNodeType()),
45+
'_type' => $nodeTypeMappingBuilder::convertNodeTypeNameToMappingName($node->getNodeType()),
3846
'_id' => $identifier
3947
]
4048
]
@@ -46,6 +54,8 @@ public function delete(NodeInterface $node, $identifier)
4654
*/
4755
public function deleteDuplicateDocumentNotMatchingType(Index $index, $documentIdentifier, NodeType $nodeType)
4856
{
57+
$nodeTypeMappingBuilder = $this->nodeTypeMappingBuilder;
58+
4959
$index->request('DELETE', '/_query', [], json_encode([
5060
'query' => [
5161
'bool' => [
@@ -56,7 +66,7 @@ public function deleteDuplicateDocumentNotMatchingType(Index $index, $documentId
5666
],
5767
'must_not' => [
5868
'term' => [
59-
'_type' => NodeTypeMappingBuilder::convertNodeTypeNameToMappingName($nodeType->getName())
69+
'_type' => $nodeTypeMappingBuilder::convertNodeTypeNameToMappingName($nodeType->getName())
6070
]
6171
],
6272
]

‎Classes/Driver/Version1/IndexerDriver.php‎

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@
1313

1414
use Flowpack\ElasticSearch\ContentRepositoryAdaptor\Driver\AbstractIndexerDriver;
1515
use Flowpack\ElasticSearch\ContentRepositoryAdaptor\Driver\IndexerDriverInterface;
16-
use Flowpack\ElasticSearch\ContentRepositoryAdaptor\Mapping\NodeTypeMappingBuilder;
16+
use Flowpack\ElasticSearch\ContentRepositoryAdaptor\Driver\NodeTypeMappingBuilderInterface;
1717
use Flowpack\ElasticSearch\Domain\Model\Document as ElasticSearchDocument;
1818
use TYPO3\Flow\Annotations as Flow;
1919
use TYPO3\TYPO3CR\Domain\Model\NodeInterface;
@@ -25,6 +25,12 @@
2525
*/
2626
class IndexerDriver extends AbstractIndexerDriver implements IndexerDriverInterface
2727
{
28+
/**
29+
* @Flow\Inject
30+
* @var NodeTypeMappingBuilderInterface
31+
*/
32+
protected $nodeTypeMappingBuilder;
33+
2834
/**
2935
* {@inheritdoc}
3036
*/
@@ -102,10 +108,11 @@ public function fulltext(NodeInterface $node, array $fulltextIndexOfNode, $targe
102108

103109
$this->logger->log(sprintf('NodeIndexer (%s): Updated fulltext index for %s (%s)', $closestFulltextNodeDocumentIdentifier, $closestFulltextNodeContextPath, $closestFulltextNode->getIdentifier()), LOG_DEBUG, null, 'ElasticSearch (CR)');
104110

111+
$nodeTypeMappingBuilder = $this->nodeTypeMappingBuilder;
105112
return [
106113
[
107114
'update' => [
108-
'_type' => NodeTypeMappingBuilder::convertNodeTypeNameToMappingName($closestFulltextNode->getNodeType()->getName()),
115+
'_type' => $nodeTypeMappingBuilder::convertNodeTypeNameToMappingName($closestFulltextNode->getNodeType()->getName()),
109116
'_id' => $closestFulltextNodeDocumentIdentifier
110117
]
111118
],

Classes/Mapping/NodeTypeMappingBuilder.php renamed to Classes/Driver/Version1/Mapping/NodeTypeMappingBuilder.php

Lines changed: 11 additions & 66 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
<?php
2-
namespace Flowpack\ElasticSearch\ContentRepositoryAdaptor\Mapping;
2+
namespace Flowpack\ElasticSearch\ContentRepositoryAdaptor\Driver\Version1\Mapping;
33

44
/*
55
* This file is part of the Flowpack.ElasticSearch.ContentRepositoryAdaptor package.
@@ -11,78 +11,31 @@
1111
* source code.
1212
*/
1313

14+
use Flowpack\ElasticSearch\ContentRepositoryAdaptor\Driver\AbstractNodeTypeMappingBuilder;
1415
use Flowpack\ElasticSearch\Domain\Model\Index;
1516
use Flowpack\ElasticSearch\Domain\Model\Mapping;
1617
use Flowpack\ElasticSearch\Mapping\MappingCollection;
1718
use TYPO3\Flow\Annotations as Flow;
18-
use TYPO3\Flow\Configuration\ConfigurationManager;
19+
use TYPO3\Flow\Error\Result;
20+
use TYPO3\Flow\Error\Warning;
1921
use TYPO3\TYPO3CR\Domain\Model\NodeType;
20-
use TYPO3\TYPO3CR\Domain\Service\NodeTypeManager;
2122

2223
/**
23-
* Builds the mapping information for TYPO3CR Node Types in Elastic Search
24+
* NodeTypeMappingBuilder for Elasticsearch version 1.x
2425
*
2526
* @Flow\Scope("singleton")
2627
*/
27-
class NodeTypeMappingBuilder
28+
class NodeTypeMappingBuilder extends AbstractNodeTypeMappingBuilder
2829
{
29-
/**
30-
* The default configuration for a given property type in NodeTypes.yaml, if no explicit elasticSearch section defined there.
31-
*
32-
* @var array
33-
*/
34-
protected $defaultConfigurationPerType;
35-
36-
/**
37-
* @Flow\Inject
38-
* @var NodeTypeManager
39-
*/
40-
protected $nodeTypeManager;
41-
42-
/**
43-
* @var \TYPO3\Flow\Error\Result
44-
*/
45-
protected $lastMappingErrors;
46-
47-
/**
48-
* @Flow\Inject
49-
* @var ConfigurationManager
50-
*/
51-
protected $configurationManager;
52-
53-
/**
54-
* Called by the Flow object framework after creating the object and resolving all dependencies.
55-
*
56-
* @param integer $cause Creation cause
57-
*/
58-
public function initializeObject($cause)
59-
{
60-
if ($cause === \TYPO3\Flow\Object\ObjectManagerInterface::INITIALIZATIONCAUSE_CREATED) {
61-
$settings = $this->configurationManager->getConfiguration(\TYPO3\Flow\Configuration\ConfigurationManager::CONFIGURATION_TYPE_SETTINGS, 'TYPO3.TYPO3CR.Search');
62-
$this->defaultConfigurationPerType = $settings['defaultConfigurationPerType'];
63-
}
64-
}
65-
66-
/**
67-
* Converts a TYPO3CR Node Type name into a name which can be used for an Elastic Search Mapping
68-
*
69-
* @param string $nodeTypeName
70-
* @return string
71-
*/
72-
public static function convertNodeTypeNameToMappingName($nodeTypeName)
73-
{
74-
return str_replace('.', '-', $nodeTypeName);
75-
}
76-
7730
/**
7831
* Builds a Mapping Collection from the configured node types
7932
*
80-
* @param \Flowpack\ElasticSearch\Domain\Model\Index $index
81-
* @return \Flowpack\ElasticSearch\Mapping\MappingCollection<\Flowpack\ElasticSearch\Domain\Model\Mapping>
33+
* @param Index $index
34+
* @return MappingCollection<\Flowpack\ElasticSearch\Domain\Model\Mapping>
8235
*/
8336
public function buildMappingInformation(Index $index)
8437
{
85-
$this->lastMappingErrors = new \TYPO3\Flow\Error\Result();
38+
$this->lastMappingErrors = new Result();
8639

8740
$mappings = new MappingCollection(MappingCollection::TYPE_ENTITY);
8841

@@ -99,7 +52,7 @@ public function buildMappingInformation(Index $index)
9952
$mapping->setFullMapping($fullConfiguration['search']['elasticSearchMapping']);
10053
}
10154

102-
// http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/mapping-root-object-type.html#_dynamic_templates
55+
// https://www.elastic.co/guide/en/elasticsearch/reference/2.4/dynamic-templates.html
10356
// 'not_analyzed' is necessary
10457
$mapping->addDynamicTemplate('dimensions', [
10558
'path_match' => '__dimensionCombinations.*',
@@ -124,7 +77,7 @@ public function buildMappingInformation(Index $index)
12477
$mapping->setPropertyByPath($propertyName, $this->defaultConfigurationPerType[$propertyConfiguration['type']]['elasticSearchMapping']);
12578
}
12679
} else {
127-
$this->lastMappingErrors->addWarning(new \TYPO3\Flow\Error\Warning('Node Type "' . $nodeTypeName . '" - property "' . $propertyName . '": No ElasticSearch Mapping found.'));
80+
$this->lastMappingErrors->addWarning(new Warning('Node Type "' . $nodeTypeName . '" - property "' . $propertyName . '": No ElasticSearch Mapping found.'));
12881
}
12982
}
13083

@@ -133,12 +86,4 @@ public function buildMappingInformation(Index $index)
13386

13487
return $mappings;
13588
}
136-
137-
/**
138-
* @return \TYPO3\Flow\Error\Result
139-
*/
140-
public function getLastMappingErrors()
141-
{
142-
return $this->lastMappingErrors;
143-
}
14489
}

0 commit comments

Comments
 (0)