Skip to content

Commit 24bf0cb

Browse files
hlubekhlubek
committed
Merge branch 'main' into php-8-compat
2 parents 0cb59bc + 45375a4 commit 24bf0cb

18 files changed

Lines changed: 188 additions & 32 deletions

‎Classes/BackendUi/BackendUiDataService.php‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -105,7 +105,9 @@ private function calculateReleaseSize(RedisInstanceIdentifier $redisInstanceIden
105105
$size = 0;
106106

107107
foreach ($allKeys as $key) {
108-
$size += $redis->rawCommand('memory', 'usage', $key);
108+
// We need to set the `samples` option to 0 here, as the default value is 5 and specifies the number of
109+
// sampled nested values. With 0 all nested values are sampled.
110+
$size += $redis->rawCommand('memory', 'usage', $key, 'samples', '0');
109111
}
110112

111113
// bytes are returned, convert to megabytes

‎Classes/Command/ContentReleasePrepareCommandController.php‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33

44
namespace Flowpack\DecoupledContentStore\Command;
55

6+
use Flowpack\DecoupledContentStore\Core\ConcurrentBuildLockService;
67
use Flowpack\DecoupledContentStore\Core\Domain\ValueObject\PrunnerJobId;
78
use Flowpack\DecoupledContentStore\PrepareContentRelease\Infrastructure\RedisContentReleaseService;
89
use Neos\Flow\Annotations as Flow;
@@ -21,6 +22,12 @@ class ContentReleasePrepareCommandController extends CommandController
2122
*/
2223
protected $redisContentReleaseService;
2324

25+
/**
26+
* @Flow\Inject
27+
* @var ConcurrentBuildLockService
28+
*/
29+
protected $concurrentBuildLock;
30+
2431
public function createContentReleaseCommand(string $contentReleaseIdentifier, string $prunnerJobId)
2532
{
2633
$contentReleaseIdentifier = ContentReleaseIdentifier::fromString($contentReleaseIdentifier);
@@ -30,6 +37,13 @@ public function createContentReleaseCommand(string $contentReleaseIdentifier, st
3037
$this->redisContentReleaseService->createContentRelease($contentReleaseIdentifier, $prunnerJobId, $logger);
3138
}
3239

40+
public function ensureAllOtherInProgressContentReleasesWillBeTerminatedCommand(string $contentReleaseIdentifier)
41+
{
42+
$contentReleaseIdentifier = ContentReleaseIdentifier::fromString($contentReleaseIdentifier);
43+
44+
$this->concurrentBuildLock->ensureAllOtherInProgressContentReleasesWillBeTerminated($contentReleaseIdentifier);
45+
}
46+
3347
public function registerManualTransferJobCommand(string $contentReleaseIdentifier, string $prunnerJobId)
3448
{
3549
$contentReleaseIdentifier = ContentReleaseIdentifier::fromString($contentReleaseIdentifier);

‎Classes/Core/ConcurrentBuildLockService.php‎

Lines changed: 64 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,64 @@
1+
<?php
2+
3+
namespace Flowpack\DecoupledContentStore\Core;
4+
5+
use Flowpack\DecoupledContentStore\Core\Domain\ValueObject\ContentReleaseIdentifier;
6+
use Flowpack\DecoupledContentStore\Core\Infrastructure\RedisClientManager;
7+
use Neos\Flow\Annotations as Flow;
8+
9+
/**
10+
* We usually rely on prunner to ensure that only one build is running at any given time.
11+
*
12+
* However, when running in a cloud environment with no shared storage, the prunner data folder is not shared between
13+
* instances. In this case, during a deployment, two containers run concurrently, with two separate prunner instances
14+
* (the old and the new one), which do not see each other.
15+
*
16+
* We could fix this in prunner itself, but this would be a bigger undertaking (different storage backends for prunner),
17+
* or we can work around this in DecoupledContentStore. This is what this class does.
18+
*
19+
* ## Main Idea
20+
*
21+
* - We use a special redis key "contentStore:concurrentBuildLock" which is set to the current being-built release ID in
22+
* `./flow contentReleasePrepare:ensureAllOtherInProgressContentReleasesWillBeTerminated`
23+
* - In the "Enumerate" and "Render" phases, we periodically check whether the concurrentBuildLock is set to the currently
24+
* in-progress content release. If NO, we abort.
25+
*
26+
* @Flow\Scope("singleton")
27+
*/
28+
class ConcurrentBuildLockService
29+
{
30+
31+
/**
32+
* @Flow\Inject
33+
* @var RedisClientManager
34+
*/
35+
protected $redisClientManager;
36+
37+
public function ensureAllOtherInProgressContentReleasesWillBeTerminated(ContentReleaseIdentifier $contentReleaseIdentifier)
38+
{
39+
$this->redisClientManager->getPrimaryRedis()->set('contentStore:concurrentBuildLock', $contentReleaseIdentifier->getIdentifier());
40+
}
41+
42+
public function assertNoOtherContentReleaseWasStarted(ContentReleaseIdentifier $contentReleaseIdentifier)
43+
{
44+
$concurrentBuildLockString = $this->redisClientManager->getPrimaryRedis()->get('contentStore:concurrentBuildLock');
45+
46+
if (empty($concurrentBuildLockString)) {
47+
echo '!!!!! Hard-aborting the current job ' . $contentReleaseIdentifier->getIdentifier() . ' because the concurrentBuildLock does not exist.' . "\n\n";
48+
echo "This should never happen for correctly configured jobs (that run after prepare_finished).\n\n";
49+
exit(1);
50+
}
51+
52+
$concurrentBuildLock = ContentReleaseIdentifier::fromString($concurrentBuildLockString);
53+
54+
if (!$contentReleaseIdentifier->equals($concurrentBuildLock)) {
55+
// the concurrent build lock is different (i.e. newer) than our currently-running content release.
56+
// Thus, we abort the in-progress content release as quickly as we can - by DYING.
57+
58+
echo '!!!!! Hard-aborting the current job ' . $contentReleaseIdentifier->getIdentifier() . ' because the concurrentBuildLock contains ' . $concurrentBuildLock->getIdentifier() . "\n\n";
59+
echo "This is no error during deployment, but should never happen outside a deployment.\n\n It can only happen if two prunner instances run concurrently.\n\n";
60+
exit(1);
61+
}
62+
}
63+
64+
}

‎Classes/Exception/InvalidTransferConfigException.php‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
<?php
2+
namespace Flowpack\DecoupledContentStore\Exception;
3+
4+
class InvalidTransferConfigException extends \Flowpack\DecoupledContentStore\Exception
5+
{
6+
7+
}

‎Classes/NodeEnumeration/NodeEnumerator.php‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
namespace Flowpack\DecoupledContentStore\NodeEnumeration;
44

55

6+
use Flowpack\DecoupledContentStore\Core\ConcurrentBuildLockService;
67
use Flowpack\DecoupledContentStore\Core\Domain\ValueObject\ContentReleaseIdentifier;
78
use Flowpack\DecoupledContentStore\Core\Domain\ValueObject\RedisInstanceIdentifier;
89
use Flowpack\DecoupledContentStore\Core\Infrastructure\ContentReleaseLogger;
@@ -37,6 +38,12 @@ class NodeEnumerator
3738
*/
3839
protected $redisContentReleaseService;
3940

41+
/**
42+
* @Flow\Inject
43+
* @var ConcurrentBuildLockService
44+
*/
45+
protected $concurrentBuildLockService;
46+
4047
/**
4148
* @Flow\InjectConfiguration("nodeRendering.nodeTypeWhitelist")
4249
* @var string
@@ -55,6 +62,7 @@ public function enumerateAndStoreInRedis(?Site $site, ContentReleaseLogger $cont
5562

5663
$this->redisEnumerationRepository->clearDocumentNodesEnumeration($releaseIdentifier);
5764
foreach (GeneratorUtility::createArrayBatch($this->enumerateAll($site, $contentReleaseLogger), 100) as $enumeration) {
65+
$this->concurrentBuildLockService->assertNoOtherContentReleaseWasStarted($releaseIdentifier);
5866
// $enumeration is an array of EnumeratedNode, with at most 100 elements in it.
5967
// TODO: EXTENSION POINT HERE, TO ADD ADDITIONAL ENUMERATIONS (.metadata.json f.e.)
6068
// TODO: not yet fully sure how to handle Enumeration

‎Classes/NodeRendering/Dto/DocumentNodeCacheKey.php‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -57,8 +57,8 @@ public function redisKeyName(): string
5757
return preg_replace('/[^a-zA-Z0-9-]/', '_', sprintf('doc--%s-%s-%s', $this->nodeIdentifier, json_encode($this->dimensions), json_encode($this->arguments)));
5858
}
5959

60-
public function fullyQualifiedRedisKeyName(): string
60+
public function fullyQualifiedRedisKeyName(string $identifierPrefix): string
6161
{
62-
return 'Neos_Fusion_Content:entry:' . $this->redisKeyName();
62+
return $identifierPrefix . 'Neos_Fusion_Content:entry:' . $this->redisKeyName();
6363
}
64-
}
64+
}

‎Classes/NodeRendering/Infrastructure/RedisContentCacheReader.php‎

Lines changed: 25 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
use Flowpack\DecoupledContentStore\NodeRendering\Dto\DocumentNodeCacheKey;
66
use Flowpack\DecoupledContentStore\NodeRendering\Dto\DocumentNodeCacheValues;
77
use Flowpack\DecoupledContentStore\NodeRendering\Dto\RenderedDocumentFromContentCache;
8+
use Neos\Cache\Backend\AbstractBackend;
89
use Neos\Cache\Frontend\StringFrontend;
910
use Neos\Flow\Annotations as Flow;
1011
use Flowpack\DecoupledContentStore\Core\Infrastructure\RedisClientManager;
@@ -29,12 +30,22 @@ class RedisContentCacheReader
2930
*/
3031
protected $contentCache;
3132

33+
/**
34+
* @Flow\InjectConfiguration(path="cache.applicationIdentifier", package="Neos.Flow")
35+
* @var string
36+
*/
37+
protected $applicationIdentifier;
38+
3239
public function tryToExtractRenderingForEnumeratedNodeFromContentCache(DocumentNodeCacheKey $documentNodeCacheKey): RenderedDocumentFromContentCache
3340
{
3441
$maxNestLevel = ContentCache::MAXIMUM_NESTING_LEVEL;
3542
$contentCacheStartToken = ContentCache::CACHE_SEGMENT_START_TOKEN;
3643
$contentCacheEndToken = ContentCache::CACHE_SEGMENT_END_TOKEN;
3744
$contentCacheMarker = ContentCache::CACHE_SEGMENT_MARKER;
45+
/**
46+
* @see AbstractBackend::setCache()
47+
*/
48+
$identifierPrefix = md5($this->applicationIdentifier) . ':';
3849

3950
$redis = null;
4051
$backend = $this->contentCache->getBackend();
@@ -45,15 +56,16 @@ public function tryToExtractRenderingForEnumeratedNodeFromContentCache(DocumentN
4556
} else {
4657
throw new \RuntimeException('TODO: Cache backend must be OptimizedRedisCacheBackend.');
4758
}
48-
$serializedCacheValues = $redis->get($documentNodeCacheKey->fullyQualifiedRedisKeyName());
59+
$serializedCacheValues = $redis->get($documentNodeCacheKey->fullyQualifiedRedisKeyName($identifierPrefix));
4960
if ($serializedCacheValues === false) {
5061
return RenderedDocumentFromContentCache::createIncomplete('No Redis Key "' . $documentNodeCacheKey->redisKeyName() . '" found.');
5162
}
5263
$documentNodeCacheValues = DocumentNodeCacheValues::fromJsonString($serializedCacheValues);
5364

5465
$script = "
5566
local rootIdentifier = ARGV[1]
56-
67+
local identifierPrefix = ARGV[2]
68+
5769
local function readContentCacheRecursively(identifier, depth)
5870
depth = depth or 1
5971
@@ -62,45 +74,45 @@ public function tryToExtractRenderingForEnumeratedNodeFromContentCache(DocumentN
6274
return '', 'Maximum Nesting Level Reached'
6375
end
6476
65-
local content = redis.call('GET', 'Neos_Fusion_Content:entry:' .. identifier)
77+
local content = redis.call('GET', identifierPrefix .. 'Neos_Fusion_Content:entry:' .. identifier)
6678
if not content then
67-
return '', 'Neos_Fusion_Content:entry:' .. identifier .. ' not found'
79+
return '', identifierPrefix .. 'Neos_Fusion_Content:entry:' .. identifier .. ' not found'
6880
end
6981
70-
local error = nil
82+
local error = nil
7183
content = string.gsub(content, '${contentCacheStartToken}${contentCacheMarker}([a-z0-9]+)${contentCacheEndToken}${contentCacheMarker}', function(id)
7284
local str
7385
local errMsg
7486
str, errMsg = readContentCacheRecursively(id, depth + 1)
75-
87+
7688
if errMsg then
7789
error = errMsg
7890
end
79-
91+
8092
return str
8193
end
8294
)
83-
95+
8496
if error then
8597
return nil, error
8698
else
8799
return content, nil
88100
end
89101
end
90-
102+
91103
local content, error = readContentCacheRecursively(rootIdentifier)
92104
if not error then
93105
error = ''
94106
end
95-
107+
96108
if not content then
97109
content = ''
98110
end
99-
111+
100112
return {content, error}
101113
";
102114
// starting with Lua 7, eval_ro can be used.
103-
$res = $redis->eval($script, [$documentNodeCacheValues->getRootIdentifier()], 0);
115+
$res = $redis->eval($script, [$documentNodeCacheValues->getRootIdentifier(), $identifierPrefix], 0);
104116
$error = $redis->getLastError();
105117
if ($error !== null) {
106118
throw new \RuntimeException('Redis Error: ' . $error);
@@ -117,4 +129,4 @@ public function tryToExtractRenderingForEnumeratedNodeFromContentCache(DocumentN
117129
}
118130
return RenderedDocumentFromContentCache::createWithFullContent($content, $documentNodeCacheValues);
119131
}
120-
}
132+
}

‎Classes/NodeRendering/NodeRenderOrchestrator.php‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44

55
namespace Flowpack\DecoupledContentStore\NodeRendering;
66

7+
use Flowpack\DecoupledContentStore\Core\ConcurrentBuildLockService;
78
use Flowpack\DecoupledContentStore\Core\Domain\ValueObject\RedisInstanceIdentifier;
89
use Flowpack\DecoupledContentStore\NodeRendering\Dto\DocumentNodeCacheKey;
910
use Flowpack\DecoupledContentStore\NodeRendering\Dto\RenderingStatistics;
@@ -93,6 +94,12 @@ class NodeRenderOrchestrator
9394
*/
9495
protected $redisContentReleaseService;
9596

97+
/**
98+
* @Flow\Inject
99+
* @var ConcurrentBuildLockService
100+
*/
101+
protected $concurrentBuildLockService;
102+
96103
private const EXIT_ERRORSTATUSCODE_RELEASE_ALREADY_COMPLETED = 1;
97104
private const EXIT_ERRORSTATUSCODE_EMPTY_ENUMERATION = 2;
98105
private const EXIT_ERRORSTATUSCODE_RETRY_LIMIT_REACHED = 3;
@@ -140,6 +147,7 @@ public function renderContentRelease(ContentReleaseIdentifier $contentReleaseIde
140147
}
141148

142149
$contentReleaseLogger->info('Starting iteration ' . $i);
150+
$this->concurrentBuildLockService->assertNoOtherContentReleaseWasStarted($contentReleaseIdentifier);
143151

144152
$this->redisRenderingStatisticsStore->addStatisticsIteration($contentReleaseIdentifier, RenderingStatistics::create(0, 0, []));
145153

@@ -200,11 +208,12 @@ public function renderContentRelease(ContentReleaseIdentifier $contentReleaseIde
200208
$jobsWorkedThroughOverLastTenSeconds = $previousRemainingJobs - $remainingJobsCount;
201209
$renderingsPerSecondDataPoints[] = $jobsWorkedThroughOverLastTenSeconds / 10;
202210

203-
204211
$contentReleaseLogger->debug('Waiting... ', [
205212
'numberOfQueuedJobs' => $remainingJobsCount,
206213
'numberOfRenderingsInProgress' => $this->redisRenderingQueue->numberOfRenderingsInProgress($contentReleaseIdentifier),
207214
]);
215+
216+
$this->concurrentBuildLockService->assertNoOtherContentReleaseWasStarted($contentReleaseIdentifier);
208217
}
209218
}
210219

‎Classes/NodeRendering/NodeRenderer.php‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
namespace Flowpack\DecoupledContentStore\NodeRendering;
66

77
use Flowpack\DecoupledContentStore\ContentReleaseManager;
8+
use Flowpack\DecoupledContentStore\Core\ConcurrentBuildLockService;
89
use Flowpack\DecoupledContentStore\NodeRendering\ProcessEvents\DocumentRenderedEvent;
910
use Flowpack\DecoupledContentStore\NodeRendering\ProcessEvents\ExitEvent;
1011
use Flowpack\DecoupledContentStore\NodeRendering\ProcessEvents\QueueEmptyEvent;
@@ -112,6 +113,12 @@ class NodeRenderer
112113
*/
113114
protected $persistenceManager;
114115

116+
/**
117+
* @Flow\Inject
118+
* @var ConcurrentBuildLockService
119+
*/
120+
protected $concurrentBuildLockService;
121+
115122

116123
public function render(ContentReleaseIdentifier $contentReleaseIdentifier, ContentReleaseLogger $contentReleaseLogger, RendererIdentifier $rendererIdentifier)
117124
{
@@ -133,6 +140,7 @@ public function render(ContentReleaseIdentifier $contentReleaseIdentifier, Conte
133140
// determining what needs to be done. We just need to wait a bit and retry.
134141
$contentReleaseLogger->debug('Rendering queue currently empty; we wait a bit see if there is work for us.');
135142
sleep(2);
143+
$this->concurrentBuildLockService->assertNoOtherContentReleaseWasStarted($contentReleaseIdentifier);
136144
continue;
137145
}
138146

@@ -149,6 +157,11 @@ public function render(ContentReleaseIdentifier $contentReleaseIdentifier, Conte
149157
yield DocumentRenderedEvent::create();
150158

151159
$i++;
160+
161+
if ($i % 5 === 0) {
162+
$this->concurrentBuildLockService->assertNoOtherContentReleaseWasStarted($contentReleaseIdentifier);
163+
}
164+
152165
if ($i % 20 === 0) {
153166
$contentReleaseLogger->info('Restarting after 20 renders.');
154167
yield ExitEvent::createWithStatusCode(193);

‎Classes/NodeRendering/Render/DocumentRenderer.php‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,10 +10,12 @@
1010
use Neos\Flow\Http\BaseUriProvider;
1111
use Neos\Flow\Http\Helper\RequestInformationHelper;
1212
use Neos\Flow\Http\Helper\ResponseInformationHelper;
13+
use Neos\Flow\Http\ServerRequestAttributes;
1314
use Neos\Flow\Mvc\ActionRequest;
1415
use Neos\Flow\Mvc\ActionResponse;
1516
use Neos\Flow\Mvc\Controller\Arguments;
1617
use Neos\Flow\Mvc\Controller\ControllerContext;
18+
use Neos\Flow\Mvc\Routing\Dto\RouteParameters;
1719
use Neos\Flow\Mvc\Routing\UriBuilder;
1820
use Neos\Neos\Domain\Model\Site;
1921
use Neos\Utility\ObjectAccess;
@@ -123,7 +125,7 @@ public function renderDocumentNodeVariant(NodeInterface $node, array $arguments,
123125
} catch (\Exception $exception) {
124126
throw new Exception\RenderingException('Error rendering document view', $node, $nodeUri, 1491378709, $exception);
125127
} finally {
126-
$this->cacheUrlMappingAspect->afterDocumentRendering ();
128+
$this->cacheUrlMappingAspect->afterDocumentRendering();
127129
}
128130
}
129131

@@ -185,6 +187,8 @@ protected function getRequest($uri, NodeInterface $node)
185187
$_SERVER['FLOW_REWRITEURLS'] = '1';
186188

187189
$httpRequest = new ServerRequest('GET', $uri);
190+
$routingParameters = RouteParameters::createEmpty()->withParameter('requestUriHost', $httpRequest->getUri()->getHost());
191+
$httpRequest = $httpRequest->withAttribute(ServerRequestAttributes::ROUTING_PARAMETERS, $routingParameters);
188192

189193
$request = ActionRequest::fromHttpRequest($httpRequest);
190194
$request->setControllerObjectName('Neos\Neos\Controller\Frontend\NodeController');

0 commit comments

Comments
 (0)