Skip to content

Commit 7fefe12

Browse files
kdambekalnskdambekalns
committed
TASK: Move scripts from inline to indexed
This replaces the use of inline groovy scripts with indexed scripts.
1 parent 40a02db commit 7fefe12

5 files changed

Lines changed: 147 additions & 95 deletions

File tree

‎Classes/Driver/Version1/IndexerDriver.php‎

Lines changed: 16 additions & 52 deletions
Original file line numberDiff line numberDiff line change
@@ -49,20 +49,14 @@ public function document($indexName, NodeInterface $node, ElasticSearchDocument
4949
]
5050
],
5151
// http://www.elasticsearch.org/guide/en/elasticsearch/reference/1.7/docs-update.html
52-
[
53-
'script' => '
54-
fulltext = (ctx._source.containsKey("__fulltext") ? ctx._source.__fulltext : new LinkedHashMap());
55-
fulltextParts = (ctx._source.containsKey("__fulltextParts") ? ctx._source.__fulltextParts : new LinkedHashMap());
56-
ctx._source = newData;
57-
ctx._source.__fulltext = fulltext;
58-
ctx._source.__fulltextParts = fulltextParts
59-
',
52+
'script' => [
53+
'script_id' => 'updateFulltextParts',
54+
'lang' => 'groovy',
6055
'params' => [
6156
'newData' => $documentData
62-
],
63-
'upsert' => $documentData,
64-
'lang' => 'groovy'
65-
]
57+
]
58+
],
59+
'upsert' => $documentData
6660
];
6761
} else {
6862
// non-fulltext-root documents can be indexed as-they-are
@@ -118,52 +112,22 @@ public function fulltext(NodeInterface $node, array $fulltextIndexOfNode, $targe
118112
]
119113
],
120114
// http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/docs-update.html
121-
[
122-
// first, update the __fulltextParts, then re-generate the __fulltext from all __fulltextParts
123-
'script' => '
124-
if (!(ctx._source.containsKey("__fulltextParts") && ctx._source.__fulltextParts instanceof Map)) {
125-
ctx._source.__fulltextParts = new LinkedHashMap();
126-
}
127-
128-
if (nodeIsRemoved || nodeIsHidden || fulltext.size() == 0) {
129-
if (ctx._source.__fulltextParts.containsKey(identifier)) {
130-
ctx._source.__fulltextParts.remove(identifier);
131-
}
132-
} else {
133-
ctx._source.__fulltextParts[identifier] = fulltext;
134-
}
135-
ctx._source.__fulltext = new LinkedHashMap();
136-
137-
Iterator<LinkedHashMap.Entry<String, LinkedHashMap>> fulltextByNode = ctx._source.__fulltextParts.entrySet().iterator();
138-
for (fulltextByNode; fulltextByNode.hasNext();) {
139-
Iterator<LinkedHashMap.Entry<String, String>> elementIterator = fulltextByNode.next().getValue().entrySet().iterator();
140-
for (elementIterator; elementIterator.hasNext();) {
141-
Map.Entry<String, String> element = elementIterator.next();
142-
String value;
143-
144-
if (ctx._source.__fulltext.containsKey(element.key)) {
145-
value = ctx._source.__fulltext[element.key] + " " + element.value.trim();
146-
} else {
147-
value = element.value.trim();
148-
}
149-
150-
ctx._source.__fulltext[element.key] = value;
151-
}
152-
}
153-
',
115+
// first, update the __fulltextParts, then re-generate the __fulltext from all __fulltextParts
116+
'script' => [
117+
'script_id' => 'regenerateFulltext',
118+
'lang' => 'groovy',
154119
'params' => [
155120
'identifier' => $node->getIdentifier(),
156121
'nodeIsRemoved' => $node->isRemoved(),
157122
'nodeIsHidden' => $node->isHidden(),
158123
'fulltext' => $fulltextIndexOfNode
159124
],
160-
'upsert' => [
161-
'__fulltext' => $fulltextIndexOfNode,
162-
'__fulltextParts' => [
163-
$node->getIdentifier() => $fulltextIndexOfNode
164-
]
165-
],
166-
'lang' => 'groovy'
125+
],
126+
'upsert' => [
127+
'__fulltext' => $fulltextIndexOfNode,
128+
'__fulltextParts' => [
129+
$node->getIdentifier() => $fulltextIndexOfNode
130+
]
167131
]
168132
];
169133
}

‎Classes/Driver/Version1/Mapping/NodeTypeMappingBuilder.php‎

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@
1212
*/
1313

1414
use Flowpack\ElasticSearch\ContentRepositoryAdaptor\Driver\AbstractNodeTypeMappingBuilder;
15+
use Flowpack\ElasticSearch\ContentRepositoryAdaptor\ElasticSearchClient;
1516
use Flowpack\ElasticSearch\Domain\Model\Index;
1617
use Flowpack\ElasticSearch\Domain\Model\Mapping;
1718
use Flowpack\ElasticSearch\Mapping\MappingCollection;
@@ -27,6 +28,12 @@
2728
*/
2829
class NodeTypeMappingBuilder extends AbstractNodeTypeMappingBuilder
2930
{
31+
/**
32+
* @Flow\Inject
33+
* @var ElasticSearchClient
34+
*/
35+
protected $client;
36+
3037
/**
3138
* Builds a Mapping Collection from the configured node types
3239
*
@@ -39,6 +46,8 @@ public function buildMappingInformation(Index $index)
3946

4047
$mappings = new MappingCollection(MappingCollection::TYPE_ENTITY);
4148

49+
$this->setupStoredScripts($index);
50+
4251
/** @var NodeType $nodeType */
4352
foreach ($this->nodeTypeManager->getNodeTypes() as $nodeTypeName => $nodeType) {
4453
if ($nodeTypeName === 'unstructured' || $nodeType->isAbstract()) {
@@ -86,4 +95,58 @@ public function buildMappingInformation(Index $index)
8695

8796
return $mappings;
8897
}
98+
99+
/**
100+
* Store scripts used during indexing in Elasticsearch.
101+
*
102+
* @param Index $index
103+
* @return void
104+
*/
105+
protected function setupStoredScripts(Index $index)
106+
{
107+
$this->client->request(
108+
'POST',
109+
'/_scripts/groovy/updateFulltextParts',
110+
[],
111+
json_encode(['script' => '
112+
fulltext = (ctx._source.containsKey("__fulltext") ? ctx._source.__fulltext : new HashMap());
113+
fulltextParts = (ctx._source.containsKey("__fulltextParts") ? ctx._source.__fulltextParts : new HashMap());
114+
ctx._source = newData;
115+
ctx._source.__fulltext = fulltext;
116+
ctx._source.__fulltextParts = fulltextParts'
117+
])
118+
);
119+
120+
$this->client->request(
121+
'POST',
122+
'/_scripts/groovy/regenerateFulltext',
123+
[],
124+
json_encode(['script' => '
125+
ctx._source.__fulltext = new HashMap();
126+
if (!ctx._source.containsKey("__fulltextParts")) {
127+
ctx._source.__fulltextParts = new HashMap();
128+
}
129+
130+
if (nodeIsRemoved || nodeIsHidden || fulltext.size() == 0) {
131+
if (ctx._source.__fulltextParts.containsKey(identifier)) {
132+
ctx._source.__fulltextParts.remove(identifier);
133+
}
134+
} else {
135+
ctx._source.__fulltextParts.put(identifier, fulltext);
136+
}
137+
138+
ctx._source.__fulltextParts.each {
139+
originNodeIdentifier, partContent -> partContent.each {
140+
bucketKey, content ->
141+
if (ctx._source.__fulltext.containsKey(bucketKey)) {
142+
value = ctx._source.__fulltext[bucketKey] + " " + content.trim();
143+
} else {
144+
value = content.trim();
145+
}
146+
ctx._source.__fulltext[bucketKey] = value;
147+
}
148+
}'
149+
])
150+
);
151+
}
89152
}

‎Classes/Driver/Version2/IndexerDriver.php‎

Lines changed: 10 additions & 42 deletions
Original file line numberDiff line numberDiff line change
@@ -49,22 +49,14 @@ public function document($indexName, NodeInterface $node, ElasticSearchDocument
4949
]
5050
],
5151
// http://www.elasticsearch.org/guide/en/elasticsearch/reference/2.4/docs-update.html
52-
[
53-
'script' => [
54-
'inline' => '
55-
fulltext = (ctx._source.containsKey("__fulltext") ? ctx._source.__fulltext : new HashMap());
56-
fulltextParts = (ctx._source.containsKey("__fulltextParts") ? ctx._source.__fulltextParts : new HashMap());
57-
ctx._source = newData;
58-
ctx._source.__fulltext = fulltext;
59-
ctx._source.__fulltextParts = fulltextParts
60-
',
61-
'params' => [
62-
'newData' => $documentData
63-
]
64-
],
65-
'upsert' => $documentData,
66-
'lang' => 'groovy'
67-
]
52+
'script' => [
53+
'script_id' => 'updateFulltextParts',
54+
'lang' => 'groovy',
55+
'params' => [
56+
'newData' => $documentData
57+
]
58+
],
59+
'upsert' => $documentData
6860
];
6961
}
7062

@@ -128,31 +120,8 @@ public function fulltext(NodeInterface $node, array $fulltextIndexOfNode, $targe
128120
[
129121
// first, update the __fulltextParts, then re-generate the __fulltext from all __fulltextParts
130122
'script' => [
131-
'inline' => '
132-
ctx._source.__fulltext = new HashMap();
133-
134-
if (!(ctx._source.containsKey("__fulltextParts") && ctx._source.__fulltextParts instanceof Map)) {
135-
ctx._source.__fulltextParts = new HashMap();
136-
}
137-
138-
if (nodeIsRemoved || nodeIsHidden || fulltext.size() == 0) {
139-
if (ctx._source.__fulltextParts.containsKey(identifier)) {
140-
ctx._source.__fulltextParts.remove(identifier);
141-
}
142-
} else {
143-
ctx._source.__fulltextParts.put(identifier, fulltext);
144-
}
145-
146-
ctx._source.__fulltextParts.each { originNodeIdentifier, partContent -> partContent.each { bucketKey, content ->
147-
if (ctx._source.__fulltext.containsKey(bucketKey)) {
148-
value = ctx._source.__fulltext[bucketKey] + " " + content.trim();
149-
} else {
150-
value = content.trim();
151-
}
152-
ctx._source.__fulltext[bucketKey] = value;
153-
}
154-
}
155-
',
123+
'script_id' => 'regenerateFulltext',
124+
'lang' => 'groovy',
156125
'params' => [
157126
'identifier' => $node->getIdentifier(),
158127
'nodeIsRemoved' => $node->isRemoved(),
@@ -164,7 +133,6 @@ public function fulltext(NodeInterface $node, array $fulltextIndexOfNode, $targe
164133
'__fulltext' => $fulltextIndexOfNode,
165134
'__fulltextParts' => $upsertFulltextParts
166135
],
167-
'lang' => 'groovy'
168136
]
169137
];
170138
}

‎Classes/Driver/Version2/Mapping/NodeTypeMappingBuilder.php‎

Lines changed: 57 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,8 +11,9 @@
1111
* source code.
1212
*/
1313

14-
use TYPO3\Flow\Annotations as Flow;
14+
use Flowpack\ElasticSearch\Domain\Model\Index;
1515
use Flowpack\ElasticSearch\ContentRepositoryAdaptor\Driver\Version1;
16+
use TYPO3\Flow\Annotations as Flow;
1617

1718
/**
1819
* NodeTypeMappingBuilder for Elasticsearch version 2.x
@@ -21,4 +22,59 @@
2122
*/
2223
class NodeTypeMappingBuilder extends Version1\Mapping\NodeTypeMappingBuilder
2324
{
25+
/**
26+
* Store scripts used during indexing in Elasticsearch.
27+
*
28+
* @param Index $index
29+
* @return void
30+
*/
31+
protected function setupStoredScripts(Index $index)
32+
{
33+
$this->client->request(
34+
'POST',
35+
'/_scripts/groovy/updateFulltextParts',
36+
[],
37+
json_encode(['script' => '
38+
fulltext = (ctx._source.containsKey("__fulltext") ? ctx._source.__fulltext : new HashMap());
39+
fulltextParts = (ctx._source.containsKey("__fulltextParts") ? ctx._source.__fulltextParts : new HashMap());
40+
41+
ctx._source = newData;
42+
ctx._source.__fulltext = fulltext;
43+
ctx._source.__fulltextParts = fulltextParts'
44+
])
45+
);
46+
47+
$this->client->request(
48+
'POST',
49+
'/_scripts/groovy/regenerateFulltext',
50+
[],
51+
json_encode(['script' => '
52+
ctx._source.__fulltext = new HashMap();
53+
54+
if (!(ctx._source.containsKey("__fulltextParts") && ctx._source.__fulltextParts instanceof Map)) {
55+
ctx._source.__fulltextParts = new HashMap();
56+
}
57+
58+
if (nodeIsRemoved || nodeIsHidden || fulltext.size() == 0) {
59+
if (ctx._source.__fulltextParts.containsKey(identifier)) {
60+
ctx._source.__fulltextParts.remove(identifier);
61+
}
62+
} else {
63+
ctx._source.__fulltextParts.put(identifier, fulltext);
64+
}
65+
66+
ctx._source.__fulltextParts.each {
67+
originNodeIdentifier, partContent -> partContent.each {
68+
bucketKey, content ->
69+
if (ctx._source.__fulltext.containsKey(bucketKey)) {
70+
value = ctx._source.__fulltext[bucketKey] + " " + content.trim();
71+
} else {
72+
value = content.trim();
73+
}
74+
ctx._source.__fulltext[bucketKey] = value;
75+
}
76+
}'
77+
])
78+
);
79+
}
2480
}

‎Documentation/ElasticConfiguration-2.x.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ Since version 2.0 the fine-grained script settings are in place as described in
66
```
77
# The following settings are absolutely required for the CR adaptor to work
88
script.inline: true
9+
script.indexed: true
910
1011
# the following settings secure your cluster
1112
cluster.name: [PUT_YOUR_CUSTOM_NAME_HERE]

0 commit comments

Comments
 (0)