Skip to content

Commit df421e1

Browse files
daniellienertdaniellienert
authored
Merge pull request #316 from daniellienert/feature/es-6-compatibility
!!! FEATURE: Elasticsearch 6 and Elasticsearch 7 compatibility
2 parents 5c64eb9 + 8b2c234 commit df421e1

19 files changed

Lines changed: 203 additions & 67 deletions

‎.travis.yml‎

Lines changed: 13 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -10,18 +10,27 @@ addons:
1010
matrix:
1111
include:
1212
- php: 7.3
13-
env: ES=5
13+
env: ES=6
14+
- php: 7.3
15+
env: ES=7
16+
17+
cache:
18+
directories:
19+
- $HOME/.composer/cache
1420

1521
before_install:
1622
- export NEOS_TARGET_VERSION=5.1
1723
- cd ..
18-
- if [ "$ES" = 5 ]; then wget --no-check-certificate https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-5.6.15.zip && unzip elasticsearch-5.6.15.zip && mv elasticsearch-5.6.15 elasticsearch; fi
24+
- if [ "$ES" = 6 ]; then wget --no-check-certificate https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-6.8.6.tar.gz && tar xvfz elasticsearch-6.8.6.tar.gz && mv elasticsearch-6.8.6 elasticsearch; fi
25+
- if [ "$ES" = 7 ]; then wget --no-check-certificate https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-7.6.0-linux-x86_64.tar.gz && tar xvfz elasticsearch-7.6.0-linux-x86_64.tar.gz && mv elasticsearch-7.6.0 elasticsearch; fi
1926
- cd elasticsearch
2027
- bin/elasticsearch -d
2128
- cd ..
2229
- git clone https://github.com/neos/neos-base-distribution.git -b ${NEOS_TARGET_VERSION}
2330
- cd neos-base-distribution
31+
- composer config repositories.flowpack-elasticsearch git https://github.com/daniellienert/Flowpack.ElasticSearch.git
2432
- composer require --no-update --no-interaction neos/content-repository-search:dev-master
33+
- composer require --no-update --no-interaction flowpack/elasticsearch:"dev-feature/es-6-compatibility as 4.1"
2534
- composer require --no-update --no-interaction flowpack/elasticsearch-contentrepositoryadaptor:dev-master
2635
- composer require --no-update --no-interaction neos/nodetypes
2736
- composer remove --no-update --no-interaction neos/site-kickstarter
@@ -33,4 +42,5 @@ install:
3342
- cd neos-base-distribution
3443
script:
3544
- bin/phpunit --colors -c Build/BuildEssentials/PhpUnit/UnitTests.xml Packages/Application/Flowpack.ElasticSearch.ContentRepositoryAdaptor/Tests/Unit
36-
- if [ "$ES" = 5 ]; then FLOW_CONTEXT="Testing/ElasticVersion5" bin/phpunit --colors --stop-on-failure -c Build/BuildEssentials/PhpUnit/FunctionalTests.xml Packages/Application/Flowpack.ElasticSearch.ContentRepositoryAdaptor/Tests/Functional; fi
45+
- if [ "$ES" = 6 ]; then FLOW_CONTEXT="Testing/ElasticVersion6" bin/phpunit --colors --stop-on-failure -c Build/BuildEssentials/PhpUnit/FunctionalTests.xml Packages/Application/Flowpack.ElasticSearch.ContentRepositoryAdaptor/Tests/Functional; fi
46+
- if [ "$ES" = 7 ]; then FLOW_CONTEXT="Testing/ElasticVersion6" bin/phpunit --colors --stop-on-failure -c Build/BuildEssentials/PhpUnit/FunctionalTests.xml Packages/Application/Flowpack.ElasticSearch.ContentRepositoryAdaptor/Tests/Functional; fi

‎Classes/Command/NodeIndexCommandController.php‎

Lines changed: 13 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@
1515

1616
use Doctrine\Common\Collections\ArrayCollection;
1717
use Flowpack\ElasticSearch\ContentRepositoryAdaptor\Driver\NodeTypeMappingBuilderInterface;
18+
use Flowpack\ElasticSearch\ContentRepositoryAdaptor\Exception\ConfigurationException;
1819
use Flowpack\ElasticSearch\ContentRepositoryAdaptor\Exception\RuntimeException;
1920
use Flowpack\ElasticSearch\ContentRepositoryAdaptor\Indexer\Error\ErrorInterface;
2021
use Flowpack\ElasticSearch\ContentRepositoryAdaptor\Indexer\NodeIndexer;
@@ -213,6 +214,7 @@ public function indexNodeCommand(string $identifier, string $workspace = null, s
213214
* @param string $postfix Index postfix, index with the same postfix will be deleted if exist
214215
* @return void
215216
* @throws StopCommandException
217+
* @throws \Flowpack\ElasticSearch\ContentRepositoryAdaptor\Exception
216218
*/
217219
public function buildCommand(int $limit = null, bool $update = false, string $workspace = null, string $postfix = null): void
218220
{
@@ -224,7 +226,7 @@ public function buildCommand(int $limit = null, bool $update = false, string $wo
224226
}
225227

226228
$postfix = (string)($postfix ?: time());
227-
$this->nodeIndexer->setIndexNamePostfix((string)$postfix);
229+
$this->nodeIndexer->setIndexNamePostfix($postfix);
228230

229231
$createMapping = function (array $dimensionsValues) use ($update, $postfix) {
230232
$this->executeInternalCommand('createInternal', [
@@ -394,6 +396,7 @@ public function buildWorkspaceInternalCommand(string $workspace, string $dimensi
394396
* @throws \Flowpack\ElasticSearch\ContentRepositoryAdaptor\Exception
395397
* @throws \Flowpack\ElasticSearch\Exception
396398
* @throws \Neos\Flow\Http\Exception
399+
* @throws ConfigurationException
397400
* @Flow\Internal
398401
*/
399402
public function refreshInternalCommand(string $dimensionsValues, string $postfix): void
@@ -413,6 +416,7 @@ public function refreshInternalCommand(string $dimensionsValues, string $postfix
413416
* @throws \Flowpack\ElasticSearch\ContentRepositoryAdaptor\Exception
414417
* @throws \Flowpack\ElasticSearch\Exception
415418
* @throws ApiException
419+
* @throws ConfigurationException
416420
* @Flow\Internal
417421
*/
418422
public function aliasInternalCommand(string $dimensionsValues, string $postfix, bool $update = false): void
@@ -452,15 +456,15 @@ public function cleanupCommand(): void
452456
foreach ($combinations as $dimensionsValues) {
453457
try {
454458
$this->nodeIndexer->setDimensions($dimensionsValues);
455-
$indicesToBeRemoved = $this->nodeIndexer->removeOldIndices();
456-
if (count($indicesToBeRemoved) > 0) {
457-
foreach ($indicesToBeRemoved as $indexToBeRemoved) {
458-
$removed = true;
459-
$this->logger->info('Removing old index ' . $indexToBeRemoved, LogEnvironment::fromMethodName(__METHOD__));
460-
}
459+
$removedIndices = $this->nodeIndexer->removeOldIndices();
460+
461+
foreach ($removedIndices as $indexToBeRemoved) {
462+
$removed = true;
463+
$this->logger->info('Removing old index ' . $indexToBeRemoved, LogEnvironment::fromMethodName(__METHOD__));
461464
}
462465
} catch (ApiException $exception) {
463-
$response = json_decode($exception->getResponse(), false);
466+
$exception->getResponse()->getBody()->rewind();
467+
$response = json_decode($exception->getResponse()->getBody()->getContents(), false);
464468
$message = sprintf('Nothing removed. ElasticSearch responded with status %s', $response->status);
465469

466470
if (isset($response->error->type)) {
@@ -548,6 +552,7 @@ private function createContentContext(string $workspaceName, array $dimensions =
548552
* @return void
549553
* @throws \Flowpack\ElasticSearch\ContentRepositoryAdaptor\Exception
550554
* @throws \Flowpack\ElasticSearch\Exception
555+
* @throws ConfigurationException
551556
*/
552557
private function applyMapping(): void
553558
{

Classes/Driver/Version5/DocumentDriver.php renamed to Classes/Driver/Version6/DocumentDriver.php

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,7 @@
11
<?php
2-
32
declare(strict_types=1);
43

5-
namespace Flowpack\ElasticSearch\ContentRepositoryAdaptor\Driver\Version5;
4+
namespace Flowpack\ElasticSearch\ContentRepositoryAdaptor\Driver\Version6;
65

76
/*
87
* This file is part of the Flowpack.ElasticSearch.ContentRepositoryAdaptor package.
@@ -18,13 +17,14 @@
1817
use Flowpack\ElasticSearch\ContentRepositoryAdaptor\Driver\DocumentDriverInterface;
1918
use Flowpack\ElasticSearch\ContentRepositoryAdaptor\Driver\NodeTypeMappingBuilderInterface;
2019
use Flowpack\ElasticSearch\Domain\Model\Index;
20+
use Flowpack\ElasticSearch\Domain\Model\Mapping;
2121
use Neos\ContentRepository\Domain\Model\NodeInterface;
2222
use Neos\ContentRepository\Domain\Model\NodeType;
2323
use Neos\Flow\Annotations as Flow;
2424
use Neos\Flow\Log\Utility\LogEnvironment;
2525

2626
/**
27-
* Document driver for Elasticsearch version 5.x
27+
* Document driver for Elasticsearch version 6.x
2828
*
2929
* @Flow\Scope("singleton")
3030
*/
@@ -54,6 +54,7 @@ public function delete(NodeInterface $node, string $identifier): array
5454
/**
5555
* {@inheritdoc}
5656
* @throws \Flowpack\ElasticSearch\Exception
57+
* @throws \Neos\Flow\Http\Exception
5758
*/
5859
public function deleteDuplicateDocumentNotMatchingType(Index $index, string $documentIdentifier, NodeType $nodeType): void
5960
{
@@ -68,18 +69,17 @@ public function deleteDuplicateDocumentNotMatchingType(Index $index, string $doc
6869
],
6970
'must_not' => [
7071
'term' => [
71-
'_type' => $this->nodeTypeMappingBuilder->convertNodeTypeNameToMappingName($nodeType->getName())
72+
Mapping::NEOS_TYPE_FIELD => $this->nodeTypeMappingBuilder->convertNodeTypeNameToMappingName($nodeType->getName())
7273
]
7374
]
7475
]
7576
]
7677
]));
7778
$treatedContent = $result->getTreatedContent();
7879
$scrollId = $treatedContent['_scroll_id'];
79-
$mapHitToDeleteRequest = function ($hit) {
80+
$mapHitToDeleteRequest = static function ($hit) {
8081
return json_encode([
8182
'delete' => [
82-
'_type' => $hit['_type'],
8383
'_id' => $hit['_id']
8484
]
8585
]);

Classes/Driver/Version5/IndexDriver.php renamed to Classes/Driver/Version6/IndexDriver.php

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
<?php
22
declare(strict_types=1);
33

4-
namespace Flowpack\ElasticSearch\ContentRepositoryAdaptor\Driver\Version5;
4+
namespace Flowpack\ElasticSearch\ContentRepositoryAdaptor\Driver\Version6;
55

66
/*
77
* This file is part of the Flowpack.ElasticSearch.ContentRepositoryAdaptor package.
@@ -22,7 +22,7 @@
2222
use Neos\Flow\Annotations as Flow;
2323

2424
/**
25-
* Index management driver for Elasticsearch version 5.x
25+
* Index management driver for Elasticsearch version 6.x
2626
*
2727
* @Flow\Scope("singleton")
2828
*/

Classes/Driver/Version5/IndexerDriver.php renamed to Classes/Driver/Version6/IndexerDriver.php

Lines changed: 8 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
<?php
22
declare(strict_types=1);
33

4-
namespace Flowpack\ElasticSearch\ContentRepositoryAdaptor\Driver\Version5;
4+
namespace Flowpack\ElasticSearch\ContentRepositoryAdaptor\Driver\Version6;
55

66
/*
77
* This file is part of the Flowpack.ElasticSearch.ContentRepositoryAdaptor package.
@@ -22,7 +22,7 @@
2222
use Neos\Flow\Log\Utility\LogEnvironment;
2323

2424
/**
25-
* Indexer driver for Elasticsearch version 5.x
25+
* Indexer driver for Elasticsearch version 6.x
2626
*
2727
* @Flow\Scope("singleton")
2828
*/
@@ -41,10 +41,10 @@ public function document(string $indexName, NodeInterface $node, ElasticSearchDo
4141
return [
4242
[
4343
'update' => [
44-
'_type' => $document->getType()->getName(),
44+
'_type' => '_doc',
4545
'_id' => $document->getId(),
4646
'_index' => $indexName,
47-
'_retry_on_conflict' => 3
47+
'retry_on_conflict' => 3
4848
]
4949
],
5050
// http://www.elasticsearch.org/guide/en/elasticsearch/reference/5.0/docs-update.html
@@ -70,8 +70,9 @@ public function document(string $indexName, NodeInterface $node, ElasticSearchDo
7070
return [
7171
[
7272
'index' => [
73-
'_type' => $document->getType()->getName(),
74-
'_id' => $document->getId()
73+
'_type' => '_doc',
74+
'_id' => $document->getId(),
75+
'_index' => $indexName,
7576
]
7677
],
7778
$documentData
@@ -112,7 +113,7 @@ public function fulltext(NodeInterface $node, array $fulltextIndexOfNode, string
112113
return [
113114
[
114115
'update' => [
115-
'_type' => $this->nodeTypeMappingBuilder->convertNodeTypeNameToMappingName($closestFulltextNode->getNodeType()->getName()),
116+
'_type' => '_doc',
116117
'_id' => $closestFulltextNodeDocumentIdentifier
117118
]
118119
],

Classes/Driver/Version5/Mapping/NodeTypeMappingBuilder.php renamed to Classes/Driver/Version6/Mapping/NodeTypeMappingBuilder.php

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22

33
declare(strict_types=1);
44

5-
namespace Flowpack\ElasticSearch\ContentRepositoryAdaptor\Driver\Version5\Mapping;
5+
namespace Flowpack\ElasticSearch\ContentRepositoryAdaptor\Driver\Version6\Mapping;
66

77
/*
88
* This file is part of the Flowpack.ElasticSearch.ContentRepositoryAdaptor package.
@@ -50,7 +50,7 @@ public function initializeObject($cause): void
5050
parent::initializeObject($cause);
5151

5252
if ($cause === ObjectManagerInterface::INITIALIZATIONCAUSE_CREATED) {
53-
$this->migrateConfigurationForElasticVersion5($this->defaultConfigurationPerType);
53+
$this->migrateConfigurationForElasticVersion6($this->defaultConfigurationPerType);
5454
}
5555
}
5656

@@ -82,15 +82,15 @@ public function buildMappingInformation(Index $index): MappingCollection
8282
$fullConfiguration = $nodeType->getFullConfiguration();
8383
if (isset($fullConfiguration['search']['elasticSearchMapping'])) {
8484
$fullMapping = $fullConfiguration['search']['elasticSearchMapping'];
85-
$this->migrateConfigurationForElasticVersion5($fullMapping);
85+
$this->migrateConfigurationForElasticVersion6($fullMapping);
8686
$mapping->setFullMapping($fullMapping);
8787
}
8888

8989
foreach ($nodeType->getProperties() as $propertyName => $propertyConfiguration) {
9090
if (isset($propertyConfiguration['search']['elasticSearchMapping'])) {
9191
if (is_array($propertyConfiguration['search']['elasticSearchMapping'])) {
9292
$propertyMapping = $propertyConfiguration['search']['elasticSearchMapping'];
93-
$this->migrateConfigurationForElasticVersion5($propertyMapping);
93+
$this->migrateConfigurationForElasticVersion6($propertyMapping);
9494
$mapping->setPropertyByPath($propertyName, $propertyMapping);
9595
}
9696
} elseif (isset($propertyConfiguration['type'], $this->defaultConfigurationPerType[$propertyConfiguration['type']]['elasticSearchMapping'])) {
@@ -112,7 +112,7 @@ public function buildMappingInformation(Index $index): MappingCollection
112112
* @param array $mapping
113113
* @return void
114114
*/
115-
protected function migrateConfigurationForElasticVersion5(array &$mapping): void
115+
protected function migrateConfigurationForElasticVersion6(array &$mapping): void
116116
{
117117
$this->adjustStringTypeMapping($mapping);
118118
}

Classes/Driver/Version5/Query/FilteredQuery.php renamed to Classes/Driver/Version6/Query/FilteredQuery.php

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,7 @@
11
<?php
2-
32
declare(strict_types=1);
43

5-
namespace Flowpack\ElasticSearch\ContentRepositoryAdaptor\Driver\Version5\Query;
4+
namespace Flowpack\ElasticSearch\ContentRepositoryAdaptor\Driver\Version6\Query;
65

76
/*
87
* This file is part of the Flowpack.ElasticSearch.ContentRepositoryAdaptor package.
@@ -18,7 +17,7 @@
1817
use Flowpack\ElasticSearch\ContentRepositoryAdaptor\Exception\QueryBuildingException;
1918

2019
/**
21-
* Filtered query for elastic version 5
20+
* Filtered query for Elasticsearch version 6.x
2221
*/
2322
class FilteredQuery extends AbstractQuery
2423
{

Classes/Driver/Version5/Query/FunctionScoreQuery.php renamed to Classes/Driver/Version6/Query/FunctionScoreQuery.php

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,7 @@
11
<?php
2-
32
declare(strict_types=1);
43

5-
namespace Flowpack\ElasticSearch\ContentRepositoryAdaptor\Driver\Version5\Query;
4+
namespace Flowpack\ElasticSearch\ContentRepositoryAdaptor\Driver\Version6\Query;
65

76
/*
87
* This file is part of the Flowpack.ElasticSearch.ContentRepositoryAdaptor package.

Classes/Driver/Version5/RequestDriver.php renamed to Classes/Driver/Version6/RequestDriver.php

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
<?php
22
declare(strict_types=1);
33

4-
namespace Flowpack\ElasticSearch\ContentRepositoryAdaptor\Driver\Version5;
4+
namespace Flowpack\ElasticSearch\ContentRepositoryAdaptor\Driver\Version6;
55

66
/*
77
* This file is part of the Flowpack.ElasticSearch.ContentRepositoryAdaptor package.
@@ -19,7 +19,7 @@
1919
use Neos\Flow\Annotations as Flow;
2020

2121
/**
22-
* Request driver for Elasticsearch version 5.x
22+
* Request driver for Elasticsearch version 6.x
2323
*
2424
* @Flow\Scope("singleton")
2525
*/
@@ -38,7 +38,6 @@ public function bulk(Index $index, $request): array
3838

3939
// Bulk request MUST end with line return
4040
$request = trim($request) . "\n";
41-
4241
$response = $index->request('POST', '/_bulk', [], $request)->getOriginalResponse()->getBody()->getContents();
4342

4443
return array_map('json_decode', explode("\n", $response));

Classes/Driver/Version5/SystemDriver.php renamed to Classes/Driver/Version6/SystemDriver.php

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,7 @@
11
<?php
2-
32
declare(strict_types=1);
43

5-
namespace Flowpack\ElasticSearch\ContentRepositoryAdaptor\Driver\Version5;
4+
namespace Flowpack\ElasticSearch\ContentRepositoryAdaptor\Driver\Version6;
65

76
/*
87
* This file is part of the Flowpack.ElasticSearch.ContentRepositoryAdaptor package.
@@ -19,7 +18,7 @@
1918
use Neos\Flow\Annotations as Flow;
2019

2120
/**
22-
* System driver for Elasticsearch version 5.x
21+
* System driver for Elasticsearch version 6.x
2322
*
2423
* @Flow\Scope("singleton")
2524
*/

0 commit comments

Comments
 (0)